From aadf4c81b0f8fc6868710873baa4120271ae2f79 Mon Sep 17 00:00:00 2001 From: Devin Smith Date: Thu, 14 Dec 2023 13:16:03 -0800 Subject: [PATCH] Add concrete JsPlugin (#4925) This PR gives a concrete structure to JsPlugin; name, version, main, root, and paths. All existing means of configuration for JS plugins delegates to JsPlugin. Java jars can now provide JS plugins with this change as well. This is also a _partial_ improvement for #4817 (JS Plugins development is slow), in that JsPlugin paths is a mean to reduce the scope of files that needs to be copied. This is _not_ a full solution (which may involve routing paths in Jetty that lead directly to the resources as configured via the JS plugins). For NPM packages, this PR takes a practical approach as opposed to a "perfect" approach; we are requiring that all "real" files be included under the top-level directory as specified by package.json/main. For example, if main is "build/index.js", we'll include everything under "build/". If main is "dist/bundle/index.js", we'll include everything under "dist/". Adds testing, fixes #4893 --- docker/registry/server-base/gradle.properties | 2 +- .../src/main/server-jetty/requirements.txt | 2 +- .../src/main/server-netty/requirements.txt | 2 +- .../main/java/io/deephaven/plugin/Plugin.java | 1 + .../java/io/deephaven/plugin/js/JsPlugin.java | 141 ++++++++++++++- .../io/deephaven/plugin/js/JsPluginBase.java | 15 -- .../plugin/js/JsPluginManifestPath.java | 58 ------ .../plugin/js/JsPluginPackagePath.java | 46 ----- .../java/io/deephaven/plugin/js/Paths.java | 53 ++++++ .../java/io/deephaven/plugin/js/PathsAll.java | 15 ++ .../io/deephaven/plugin/js/PathsInternal.java | 10 ++ .../io/deephaven/plugin/js/PathsPrefixes.java | 53 ++++++ .../io/deephaven/plugin/js/package-info.java | 18 ++ .../deephaven_internal/plugin/js/__init__.py | 29 +++ .../deephaven_internal/plugin/register.py | 9 +- py/server/setup.py | 2 +- .../io/deephaven/server/jetty/CopyHelper.java | 53 +++++- .../server/jetty/JsPluginManifest.java | 28 --- .../io/deephaven/server/jetty/JsPlugins.java | 56 +----- .../server/jetty/JsPluginsZipFilesystem.java | 49 ++--- .../jetty/JettyFlightRoundTripTest.java | 169 ++++++++++++++++-- .../jetty/js/Example123Registration.java | 68 +++++++ .../deephaven/server/jetty/js/Sentinel.java | 8 + .../js/JsPluginsManifestRegistration.java | 35 ++++ .../js/JsPluginsNpmPackageRegistration.java | 32 ++++ .../@deephaven_test/example1/dist/index.js | 1 + .../@deephaven_test/example1/dist/index2.js | 1 + .../@deephaven_test/example1/package.json | 1 + .../@deephaven_test/example2/dist/index.js | 1 + .../@deephaven_test/example2/dist/index2.js | 1 + .../@deephaven_test/example2/package.json | 1 + .../@deephaven_test/example3/index.js | 1 + .../server/jetty/js/examples/manifest.json | 19 ++ .../netty/NettyFlightRoundTripTest.java | 16 -- .../plugin/PluginRegistrationVisitor.java | 1 - .../deephaven/server/plugin/js/Jackson.java | 12 ++ .../js/JsPluginConfigDirRegistration.java | 65 +++++++ .../plugin/js/JsPluginFromNpmPackage.java | 40 +++++ .../server/plugin/js/JsPluginManifest.java | 48 +++++ .../plugin/js}/JsPluginManifestEntry.java | 9 +- .../js/JsPluginManifestRegistration.java | 56 ++++++ .../server/plugin/js/JsPluginModule.java | 113 +----------- .../js/JsPluginNpmPackageRegistration.java | 130 ++++++++++++++ .../plugin/js/JsPluginsFromManifest.java | 33 ++++ .../server/plugin/js/NpmPackage.java | 49 +++++ .../server/plugin/python/CallbackAdapter.java | 6 + .../test/FlightMessageRoundTripTest.java | 31 +++- 47 files changed, 1200 insertions(+), 389 deletions(-) delete mode 100644 plugin/src/main/java/io/deephaven/plugin/js/JsPluginBase.java delete mode 100644 plugin/src/main/java/io/deephaven/plugin/js/JsPluginManifestPath.java delete mode 100644 plugin/src/main/java/io/deephaven/plugin/js/JsPluginPackagePath.java create mode 100644 plugin/src/main/java/io/deephaven/plugin/js/Paths.java create mode 100644 plugin/src/main/java/io/deephaven/plugin/js/PathsAll.java create mode 100644 plugin/src/main/java/io/deephaven/plugin/js/PathsInternal.java create mode 100644 plugin/src/main/java/io/deephaven/plugin/js/PathsPrefixes.java create mode 100644 plugin/src/main/java/io/deephaven/plugin/js/package-info.java create mode 100644 py/server/deephaven_internal/plugin/js/__init__.py delete mode 100644 server/jetty/src/main/java/io/deephaven/server/jetty/JsPluginManifest.java create mode 100644 server/jetty/src/test/java/io/deephaven/server/jetty/js/Example123Registration.java create mode 100644 server/jetty/src/test/java/io/deephaven/server/jetty/js/Sentinel.java create mode 100644 server/jetty/src/test/java/io/deephaven/server/plugin/js/JsPluginsManifestRegistration.java create mode 100644 server/jetty/src/test/java/io/deephaven/server/plugin/js/JsPluginsNpmPackageRegistration.java create mode 100644 server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example1/dist/index.js create mode 100644 server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example1/dist/index2.js create mode 100644 server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example1/package.json create mode 100644 server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example2/dist/index.js create mode 100644 server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example2/dist/index2.js create mode 100644 server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example2/package.json create mode 100644 server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example3/index.js create mode 100644 server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/manifest.json create mode 100644 server/src/main/java/io/deephaven/server/plugin/js/Jackson.java create mode 100644 server/src/main/java/io/deephaven/server/plugin/js/JsPluginConfigDirRegistration.java create mode 100644 server/src/main/java/io/deephaven/server/plugin/js/JsPluginFromNpmPackage.java create mode 100644 server/src/main/java/io/deephaven/server/plugin/js/JsPluginManifest.java rename server/{jetty/src/main/java/io/deephaven/server/jetty => src/main/java/io/deephaven/server/plugin/js}/JsPluginManifestEntry.java (82%) create mode 100644 server/src/main/java/io/deephaven/server/plugin/js/JsPluginManifestRegistration.java create mode 100644 server/src/main/java/io/deephaven/server/plugin/js/JsPluginNpmPackageRegistration.java create mode 100644 server/src/main/java/io/deephaven/server/plugin/js/JsPluginsFromManifest.java create mode 100644 server/src/main/java/io/deephaven/server/plugin/js/NpmPackage.java diff --git a/docker/registry/server-base/gradle.properties b/docker/registry/server-base/gradle.properties index 5195ebe303f..4367ce45c0d 100644 --- a/docker/registry/server-base/gradle.properties +++ b/docker/registry/server-base/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=ghcr.io/deephaven/server-base:edge -deephaven.registry.imageId=ghcr.io/deephaven/server-base@sha256:bd59f63831db8ff86c3ebab21ffecd1804e58dcfaaf08a67bf0c2b320a2ce179 +deephaven.registry.imageId=ghcr.io/deephaven/server-base@sha256:8f9b993a4ce7c78b50b869be840241a0a9d19d3f4f35601f20cd05475abd5753 diff --git a/docker/server-jetty/src/main/server-jetty/requirements.txt b/docker/server-jetty/src/main/server-jetty/requirements.txt index 5688518b55c..a33870876ee 100644 --- a/docker/server-jetty/src/main/server-jetty/requirements.txt +++ b/docker/server-jetty/src/main/server-jetty/requirements.txt @@ -1,7 +1,7 @@ adbc-driver-manager==0.8.0 adbc-driver-postgresql==0.8.0 connectorx==0.3.2; platform.machine == 'x86_64' -deephaven-plugin==0.5.0 +deephaven-plugin==0.6.0 java-utilities==0.2.0 jedi==0.18.2 jpy==0.14.0 diff --git a/docker/server/src/main/server-netty/requirements.txt b/docker/server/src/main/server-netty/requirements.txt index 5688518b55c..a33870876ee 100644 --- a/docker/server/src/main/server-netty/requirements.txt +++ b/docker/server/src/main/server-netty/requirements.txt @@ -1,7 +1,7 @@ adbc-driver-manager==0.8.0 adbc-driver-postgresql==0.8.0 connectorx==0.3.2; platform.machine == 'x86_64' -deephaven-plugin==0.5.0 +deephaven-plugin==0.6.0 java-utilities==0.2.0 jedi==0.18.2 jpy==0.14.0 diff --git a/plugin/src/main/java/io/deephaven/plugin/Plugin.java b/plugin/src/main/java/io/deephaven/plugin/Plugin.java index 0acff4743f3..3dbfcd2409a 100644 --- a/plugin/src/main/java/io/deephaven/plugin/Plugin.java +++ b/plugin/src/main/java/io/deephaven/plugin/Plugin.java @@ -10,6 +10,7 @@ * A plugin is a structured extension point for user-definable behavior. * * @see ObjectType + * @see JsPlugin */ public interface Plugin extends Registration { diff --git a/plugin/src/main/java/io/deephaven/plugin/js/JsPlugin.java b/plugin/src/main/java/io/deephaven/plugin/js/JsPlugin.java index be4a5a1a86f..278ce7f079e 100644 --- a/plugin/src/main/java/io/deephaven/plugin/js/JsPlugin.java +++ b/plugin/src/main/java/io/deephaven/plugin/js/JsPlugin.java @@ -3,16 +3,145 @@ */ package io.deephaven.plugin.js; +import io.deephaven.annotations.BuildableStyle; import io.deephaven.plugin.Plugin; +import io.deephaven.plugin.PluginBase; +import org.immutables.value.Value.Check; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.nio.file.Files; +import java.nio.file.Path; /** - * A js plugin is a {@link Plugin} that allows adding javascript code under the server's URL path "js-plugins/". See - * deephaven-plugins#js-plugins for more details - * about the underlying construction for js plugins. + * A JS plugin is a {@link Plugin} that allows for custom javascript and related content to be served, see + * {@link io.deephaven.plugin.js}. + * + *

+ * For example, if the following JS plugin was the only JS plugin installed + * + * 

+ * JsPlugin.builder()
+ *         .name("foo")
+ *         .version("1.0.0")
+ *         .main(Path.of("dist/index.js"))
+ *         .path(Path.of("/path-to/my-plugin"))
+ *         .build()
+ *
+ * + * the manifest served at "js-plugins/manifest.json" would be equivalent to + * + * 
+ * {
+ *   "plugins": [
+ *     {
+ *       "name": "foo",
+ *       "version": "1.0.0",
+ *       "main": "dist/index.js"
+ *     }
+ *   ]
+ * }
+ *
* - * @see JsPluginPackagePath - * @see JsPluginManifestPath + * and the file "/path-to/my-plugin/dist/index.js" would be served at "js-plugins/foo/dist/index.js". All other files of + * the form "/path-to/my-plugin/{somePath}" will be served at "js-plugins/foo/{somePath}". */ -public interface JsPlugin extends Plugin { +@Immutable +@BuildableStyle +public abstract class JsPlugin extends PluginBase { + + public static Builder builder() { + return ImmutableJsPlugin.builder(); + } + + /** + * The JS plugin name. The JS plugin contents will be served via the URL path "js-plugins/{name}/", as well as + * included as the "name" field for the manifest entry in "js-plugins/manifest.json". + * + * @return the name + */ + public abstract String name(); + + /** + * The JS plugin version. Will be included as the "version" field for the manifest entry in + * "js-plugins/manifest.json". + * + * @return the version + */ + public abstract String version(); + + /** + * The main JS file path, specified relative to {@link #path()}. The main JS file must exist + * ({@code Files.isRegularFile(root().resolve(main()))}) and must be included in {@link #paths()}. Will be included + * as the "main" field for the manifest entry in "js-plugins/manifest.json". + * + * @return the main JS file path + */ + public abstract Path main(); + + /** + * The directory path of the resources to serve. The resources will be served via the URL path + * "js-plugins/{name}/{relativeToPath}". The path must exist ({@code Files.isDirectory(path())}). + * + * @return the path + */ + public abstract Path path(); + + /** + * The subset of resources from {@link #path()} to serve. Production installations should preferably be packaged + * with the exact resources necessary (and thus served with {@link Paths#all()}). During development, other subsets + * may be useful if {@link #path()} contains content unrelated to the JS content. By default, is + * {@link Paths#all()}. + * + * @return the paths + */ + @Default + public Paths paths() { + return Paths.all(); + } + + @Override + public final > T walk(V visitor) { + return visitor.visit(this); + } + + @Check + final void checkPath() { + if (!Files.isDirectory(path())) { + throw new IllegalArgumentException(String.format("path ('%s') must exist and be a directory", path())); + } + } + + @Check + final void checkMain() { + final Path mainPath = path().resolve(main()); + if (!Files.isRegularFile(mainPath)) { + throw new IllegalArgumentException(String.format("main ('%s') must exist and be a regular file", mainPath)); + } + } + + @Check + final void checkPaths() { + if (!(paths() instanceof PathsInternal)) { + throw new IllegalArgumentException("Must construct one of the approved Paths"); + } + final Path relativeMain = path().relativize(path().resolve(main())); + if (!((PathsInternal) paths()).matches(relativeMain)) { + throw new IllegalArgumentException(String.format("main ('%s') is not in paths", relativeMain)); + } + } + + public interface Builder { + Builder name(String name); + + Builder version(String version); + + Builder main(Path main); + + Builder path(Path path); + + Builder paths(Paths paths); + JsPlugin build(); + } } diff --git a/plugin/src/main/java/io/deephaven/plugin/js/JsPluginBase.java b/plugin/src/main/java/io/deephaven/plugin/js/JsPluginBase.java deleted file mode 100644 index 93c0ef604c3..00000000000 --- a/plugin/src/main/java/io/deephaven/plugin/js/JsPluginBase.java +++ /dev/null @@ -1,15 +0,0 @@ -/** - * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.plugin.js; - -import io.deephaven.plugin.Plugin; -import io.deephaven.plugin.PluginBase; - -public abstract class JsPluginBase extends PluginBase implements JsPlugin { - - @Override - public final > T walk(V visitor) { - return visitor.visit(this); - } -} diff --git a/plugin/src/main/java/io/deephaven/plugin/js/JsPluginManifestPath.java b/plugin/src/main/java/io/deephaven/plugin/js/JsPluginManifestPath.java deleted file mode 100644 index 78673a94e51..00000000000 --- a/plugin/src/main/java/io/deephaven/plugin/js/JsPluginManifestPath.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.plugin.js; - -import io.deephaven.annotations.SimpleStyle; -import org.immutables.value.Value.Immutable; -import org.immutables.value.Value.Parameter; - -import java.nio.file.Path; - -/** - * A manifest-based js plugin sourced from a {@value MANIFEST_JSON} file. - */ -@Immutable -@SimpleStyle -public abstract class JsPluginManifestPath extends JsPluginBase { - - public static final String MANIFEST_JSON = "manifest.json"; - - /** - * Creates a manifest-based js plugin from {@code manifestRoot}. - * - * @param manifestRoot the manifest root directory path - * @return the manifest-based js plugin - */ - public static JsPluginManifestPath of(Path manifestRoot) { - return ImmutableJsPluginManifestPath.of(manifestRoot); - } - - /** - * The manifest root path directory path. - * - * @return the manifest root directory path - */ - @Parameter - public abstract Path path(); - - /** - * The {@value MANIFEST_JSON} file path, relative to {@link #path()}. Equivalent to - * {@code path().resolve(MANIFEST_JSON)}. - * - * @return the manifest json file path - */ - public final Path manifestJson() { - return path().resolve(MANIFEST_JSON); - } - - /** - * Equivalent to {@code JsPluginPackagePath.of(path().resolve(name))}. - * - * @param name the package name - * @return the package path - */ - public final JsPluginPackagePath packagePath(String name) { - return JsPluginPackagePath.of(path().resolve(name)); - } -} diff --git a/plugin/src/main/java/io/deephaven/plugin/js/JsPluginPackagePath.java b/plugin/src/main/java/io/deephaven/plugin/js/JsPluginPackagePath.java deleted file mode 100644 index 079f186d76b..00000000000 --- a/plugin/src/main/java/io/deephaven/plugin/js/JsPluginPackagePath.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.plugin.js; - -import io.deephaven.annotations.SimpleStyle; -import org.immutables.value.Value.Immutable; -import org.immutables.value.Value.Parameter; - -import java.nio.file.Path; - -/** - * A package-based js plugin sourced from a {@value PACKAGE_JSON} file. - */ -@Immutable -@SimpleStyle -public abstract class JsPluginPackagePath extends JsPluginBase { - public static final String PACKAGE_JSON = "package.json"; - - /** - * Creates a package-based js plugin from {@code packageRoot}. - * - * @param packageRoot the package root directory path - * @return the package-based js plugin - */ - public static JsPluginPackagePath of(Path packageRoot) { - return ImmutableJsPluginPackagePath.of(packageRoot); - } - - /** - * The package root directory path. - * - * @return the package root directory path - */ - @Parameter - public abstract Path path(); - - /** - * The {@value PACKAGE_JSON} file path. Equivalent to {@code path().resolve(PACKAGE_JSON)}. - * - * @return the package json file path - */ - public final Path packageJson() { - return path().resolve(PACKAGE_JSON); - } -} diff --git a/plugin/src/main/java/io/deephaven/plugin/js/Paths.java b/plugin/src/main/java/io/deephaven/plugin/js/Paths.java new file mode 100644 index 00000000000..9bee0545ce6 --- /dev/null +++ b/plugin/src/main/java/io/deephaven/plugin/js/Paths.java @@ -0,0 +1,53 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.plugin.js; + +import java.nio.file.Path; + +/** + * The subset of paths to serve, see {@link JsPlugin#paths()}. + */ +public interface Paths { + + /** + * Includes all paths. + * + * @return the paths + */ + static Paths all() { + return PathsAll.ALL; + } + + /** + * Includes only the paths that are prefixed by {@code prefix}. + * + * @param prefix the prefix + * @return the paths + */ + static Paths ofPrefixes(Path prefix) { + // Note: we have specific overload for single element to explicitly differentiate from Iterable overload since + // Path extends Iterable. + return PathsPrefixes.builder().addPrefixes(prefix).build(); + } + + /** + * Includes only the paths that are prefixed by one of {@code prefixes}. + * + * @param prefixes the prefixes + * @return the paths + */ + static Paths ofPrefixes(Path... prefixes) { + return PathsPrefixes.builder().addPrefixes(prefixes).build(); + } + + /** + * Includes only the paths that are prefixed by one of {@code prefixes}. + * + * @param prefixes the prefixes + * @return the paths + */ + static Paths ofPrefixes(Iterable prefixes) { + return PathsPrefixes.builder().addAllPrefixes(prefixes).build(); + } +} diff --git a/plugin/src/main/java/io/deephaven/plugin/js/PathsAll.java b/plugin/src/main/java/io/deephaven/plugin/js/PathsAll.java new file mode 100644 index 00000000000..dc9909923b3 --- /dev/null +++ b/plugin/src/main/java/io/deephaven/plugin/js/PathsAll.java @@ -0,0 +1,15 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.plugin.js; + +import java.nio.file.Path; + +enum PathsAll implements PathsInternal { + ALL; + + @Override + public boolean matches(Path path) { + return true; + } +} diff --git a/plugin/src/main/java/io/deephaven/plugin/js/PathsInternal.java b/plugin/src/main/java/io/deephaven/plugin/js/PathsInternal.java new file mode 100644 index 00000000000..70b777aef01 --- /dev/null +++ b/plugin/src/main/java/io/deephaven/plugin/js/PathsInternal.java @@ -0,0 +1,10 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.plugin.js; + +import java.nio.file.PathMatcher; + +interface PathsInternal extends Paths, PathMatcher { + +} diff --git a/plugin/src/main/java/io/deephaven/plugin/js/PathsPrefixes.java b/plugin/src/main/java/io/deephaven/plugin/js/PathsPrefixes.java new file mode 100644 index 00000000000..affb0331054 --- /dev/null +++ b/plugin/src/main/java/io/deephaven/plugin/js/PathsPrefixes.java @@ -0,0 +1,53 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.plugin.js; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Check; +import org.immutables.value.Value.Immutable; + +import java.nio.file.Path; +import java.util.Set; + +@Immutable +@BuildableStyle +abstract class PathsPrefixes implements PathsInternal { + + public static Builder builder() { + return ImmutablePathsPrefixes.builder(); + } + + public abstract Set prefixes(); + + @Override + public final boolean matches(Path path) { + if (prefixes().contains(path)) { + return true; + } + // Note: we could make a more efficient impl w/ a tree-based approach based on the names + for (Path prefix : prefixes()) { + if (path.startsWith(prefix)) { + return true; + } + } + return false; + } + + @Check + final void checkPrefixesNonEmpty() { + if (prefixes().isEmpty()) { + throw new IllegalArgumentException("prefixes must be non-empty"); + } + } + + interface Builder { + Builder addPrefixes(Path element); + + Builder addPrefixes(Path... elements); + + Builder addAllPrefixes(Iterable elements); + + PathsPrefixes build(); + } +} diff --git a/plugin/src/main/java/io/deephaven/plugin/js/package-info.java b/plugin/src/main/java/io/deephaven/plugin/js/package-info.java new file mode 100644 index 00000000000..3f3ee3f3101 --- /dev/null +++ b/plugin/src/main/java/io/deephaven/plugin/js/package-info.java @@ -0,0 +1,18 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ + +/** + * The Deephaven server supports {@link io.deephaven.plugin.js.JsPlugin JS plugins} which allow custom javascript (and + * related content) to be served under the HTTP path "js-plugins/". + * + *

+ * A "js-plugins/manifest.json" is served that allows clients to discover what JS plugins are installed. This will be a + * JSON object, and will have a "plugins" array, with object elements that have a "name", "version", and "main". All + * files served via a specific plugin will be accessed under "js-plugins/{name}/". The main entry file for a plugin will + * be accessed at "js-plugins/{name}/{main}". The "version" is currently for informational purposes only. + * + * @see deephaven-plugins for Deephaven-maintained JS + * plugins + */ +package io.deephaven.plugin.js; diff --git a/py/server/deephaven_internal/plugin/js/__init__.py b/py/server/deephaven_internal/plugin/js/__init__.py new file mode 100644 index 00000000000..b0f18f3a8da --- /dev/null +++ b/py/server/deephaven_internal/plugin/js/__init__.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending +# + +import jpy +import pathlib + +from deephaven.plugin.js import JsPlugin + +_JJsPlugin = jpy.get_type("io.deephaven.plugin.js.JsPlugin") +_JPath = jpy.get_type("java.nio.file.Path") + + +def to_j_js_plugin(js_plugin: JsPlugin) -> jpy.JType: + path = js_plugin.path() + if not isinstance(path, pathlib.Path): + # Adding a little bit of extra safety for this version of the server. + # There's potential that the return type of JsPlugin.path expands in the future. + raise Exception( + f"Expecting pathlib.Path, is type(js_plugin.path())={type(path)}, js_plugin={js_plugin}" + ) + j_path = _JPath.of(str(path)) + main_path = j_path.relativize(j_path.resolve(js_plugin.main)) + builder = _JJsPlugin.builder() + builder.name(js_plugin.name) + builder.version(js_plugin.version) + builder.main(main_path) + builder.path(j_path) + return builder.build() diff --git a/py/server/deephaven_internal/plugin/register.py b/py/server/deephaven_internal/plugin/register.py index a35d152c91c..91ec3c20e4e 100644 --- a/py/server/deephaven_internal/plugin/register.py +++ b/py/server/deephaven_internal/plugin/register.py @@ -8,9 +8,11 @@ from typing import Union, Type from deephaven.plugin import Plugin, Registration, Callback from deephaven.plugin.object_type import ObjectType +from deephaven.plugin.js import JsPlugin from .object import ObjectTypeAdapter +from .js import to_j_js_plugin -_JCallbackAdapter = jpy.get_type('io.deephaven.server.plugin.python.CallbackAdapter') +_JCallbackAdapter = jpy.get_type("io.deephaven.server.plugin.python.CallbackAdapter") def initialize_all_and_register_into(callback: _JCallbackAdapter): @@ -20,6 +22,7 @@ def initialize_all_and_register_into(callback: _JCallbackAdapter): class RegistrationAdapter(Callback): """Python implementation of Callback that delegates to its Java counterpart.""" + def __init__(self, callback: _JCallbackAdapter): self._callback = callback @@ -29,8 +32,10 @@ def register(self, plugin: Union[Plugin, Type[Plugin]]): plugin = plugin() if isinstance(plugin, ObjectType): self._callback.registerObjectType(plugin.name, ObjectTypeAdapter(plugin)) + elif isinstance(plugin, JsPlugin): + self._callback.registerJsPlugin(to_j_js_plugin(plugin)) else: - raise NotImplementedError + raise NotImplementedError(f"Unexpected type: {type(plugin)}") def __str__(self): return str(self._callback) diff --git a/py/server/setup.py b/py/server/setup.py index df83ea7e498..4161000e0d4 100644 --- a/py/server/setup.py +++ b/py/server/setup.py @@ -56,7 +56,7 @@ def _compute_version(): python_requires='>=3.8', install_requires=[ 'jpy>=0.14.0', - 'deephaven-plugin==0.5.0', + 'deephaven-plugin>=0.6.0', 'numpy', 'pandas>=1.5.0', 'pyarrow', diff --git a/server/jetty/src/main/java/io/deephaven/server/jetty/CopyHelper.java b/server/jetty/src/main/java/io/deephaven/server/jetty/CopyHelper.java index 8b13de748a6..ccd6ab2f802 100644 --- a/server/jetty/src/main/java/io/deephaven/server/jetty/CopyHelper.java +++ b/server/jetty/src/main/java/io/deephaven/server/jetty/CopyHelper.java @@ -4,40 +4,77 @@ package io.deephaven.server.jetty; import java.io.IOException; +import java.nio.file.DirectoryNotEmptyException; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.PathMatcher; import java.nio.file.SimpleFileVisitor; import java.nio.file.StandardCopyOption; import java.nio.file.attribute.BasicFileAttributes; import java.util.Objects; class CopyHelper { - static void copyRecursive(Path src, Path dst) throws IOException { + static void copyRecursive(Path src, Path dst, PathMatcher pathMatcher) throws IOException { + copyRecursive(src, dst, pathMatcher, d -> true); + } + + static void copyRecursive(Path src, Path dst, PathMatcher pathMatcher, PathMatcher dirMatcher) throws IOException { Files.createDirectories(dst.getParent()); - Files.walkFileTree(src, new CopyRecursiveVisitor(src, dst)); + Files.walkFileTree(src, new CopyRecursiveVisitor(src, dst, pathMatcher, dirMatcher)); } private static class CopyRecursiveVisitor extends SimpleFileVisitor { private final Path src; private final Path dst; + private final PathMatcher pathMatcher; + private final PathMatcher dirMatcher; - public CopyRecursiveVisitor(Path src, Path dst) { + public CopyRecursiveVisitor(Path src, Path dst, PathMatcher pathMatcher, PathMatcher dirMatcher) { this.src = Objects.requireNonNull(src); this.dst = Objects.requireNonNull(dst); + this.pathMatcher = Objects.requireNonNull(pathMatcher); + this.dirMatcher = Objects.requireNonNull(dirMatcher); } @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - // Note: toString() necessary for src/dst that don't share the same root FS - Files.copy(dir, dst.resolve(src.relativize(dir).toString()), StandardCopyOption.COPY_ATTRIBUTES); - return FileVisitResult.CONTINUE; + final Path relativeDir = src.relativize(dir); + if (dirMatcher.matches(relativeDir) || pathMatcher.matches(relativeDir)) { + // Note: toString() necessary for src/dst that don't share the same root FS + Files.copy(dir, dst.resolve(relativeDir.toString()), StandardCopyOption.COPY_ATTRIBUTES); + return FileVisitResult.CONTINUE; + } + return FileVisitResult.SKIP_SUBTREE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - // Note: toString() necessary for src/dst that don't share the same root FS - Files.copy(file, dst.resolve(src.relativize(file).toString()), StandardCopyOption.COPY_ATTRIBUTES); + final Path relativeFile = src.relativize(file); + if (pathMatcher.matches(relativeFile)) { + // Note: toString() necessary for src/dst that don't share the same root FS + Files.copy(file, dst.resolve(relativeFile.toString()), StandardCopyOption.COPY_ATTRIBUTES); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + if (exc != null) { + throw exc; + } + final Path relativeDir = src.relativize(dir); + if (!pathMatcher.matches(relativeDir)) { + // If the specific dir does not match as a path (even if it _did_ match as a directory), we + // "optimistically" try and delete it; if the directory is not empty (b/c some subpath matched and was + // copied), the delete will fail. (We could have an alternative impl that keeps track w/ a stack if any + // subpaths matched.) + try { + Files.delete(dir); + } catch (DirectoryNotEmptyException e) { + // ignore + } + } return FileVisitResult.CONTINUE; } } diff --git a/server/jetty/src/main/java/io/deephaven/server/jetty/JsPluginManifest.java b/server/jetty/src/main/java/io/deephaven/server/jetty/JsPluginManifest.java deleted file mode 100644 index 6b7f162834e..00000000000 --- a/server/jetty/src/main/java/io/deephaven/server/jetty/JsPluginManifest.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.server.jetty; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.deephaven.annotations.SimpleStyle; -import org.immutables.value.Value.Immutable; -import org.immutables.value.Value.Parameter; - -import java.util.List; - -@Immutable -@SimpleStyle -abstract class JsPluginManifest { - public static final String PLUGINS = "plugins"; - - @JsonCreator - public static JsPluginManifest of( - @JsonProperty(value = PLUGINS, required = true) List plugins) { - return ImmutableJsPluginManifest.of(plugins); - } - - @Parameter - @JsonProperty(PLUGINS) - public abstract List plugins(); -} diff --git a/server/jetty/src/main/java/io/deephaven/server/jetty/JsPlugins.java b/server/jetty/src/main/java/io/deephaven/server/jetty/JsPlugins.java index ffebf7b1700..41f1527c52a 100644 --- a/server/jetty/src/main/java/io/deephaven/server/jetty/JsPlugins.java +++ b/server/jetty/src/main/java/io/deephaven/server/jetty/JsPlugins.java @@ -4,19 +4,12 @@ package io.deephaven.server.jetty; import io.deephaven.plugin.js.JsPlugin; -import io.deephaven.plugin.js.JsPluginManifestPath; -import io.deephaven.plugin.js.JsPluginPackagePath; import io.deephaven.plugin.js.JsPluginRegistration; import java.io.IOException; -import java.io.InputStream; import java.io.UncheckedIOException; import java.net.URI; -import java.nio.file.Files; import java.util.Objects; -import java.util.function.Consumer; - -import static io.deephaven.server.jetty.Json.OBJECT_MAPPER; /** * Jetty-specific implementation of {@link JsPluginRegistration} to collect plugins and advertise their contents to @@ -42,56 +35,9 @@ public URI filesystem() { @Override public void register(JsPlugin jsPlugin) { try { - if (jsPlugin instanceof JsPluginPackagePath) { - copy((JsPluginPackagePath) jsPlugin, zipFs); - return; - } - if (jsPlugin instanceof JsPluginManifestPath) { - copyAll((JsPluginManifestPath) jsPlugin, zipFs); - return; - } + zipFs.add(jsPlugin); } catch (IOException e) { throw new UncheckedIOException(e); } - throw new IllegalStateException("Unexpected JsPlugin class: " + jsPlugin.getClass()); - } - - private static void copy(JsPluginPackagePath srcPackagePath, JsPluginsZipFilesystem dest) - throws IOException { - copy(srcPackagePath, dest, null); - } - - private static void copy(JsPluginPackagePath srcPackagePath, JsPluginsZipFilesystem dest, - JsPluginManifestEntry expected) - throws IOException { - final JsPluginManifestEntry srcEntry = entry(srcPackagePath); - if (expected != null && !expected.equals(srcEntry)) { - throw new IllegalStateException(String.format( - "Inconsistency between manifest.json and package.json, expected=%s, actual=%s", expected, - srcEntry)); - } - dest.copyFrom(srcPackagePath, srcEntry); - } - - private static void copyAll(JsPluginManifestPath srcManifestPath, JsPluginsZipFilesystem dest) throws IOException { - final JsPluginManifest manifestInfo = manifest(srcManifestPath); - for (JsPluginManifestEntry manifestEntry : manifestInfo.plugins()) { - final JsPluginPackagePath packagePath = srcManifestPath.packagePath(manifestEntry.name()); - copy(packagePath, dest, manifestEntry); - } - } - - private static JsPluginManifest manifest(JsPluginManifestPath manifest) throws IOException { - // jackson impl does buffering internally - try (final InputStream in = Files.newInputStream(manifest.manifestJson())) { - return OBJECT_MAPPER.readValue(in, JsPluginManifest.class); - } - } - - private static JsPluginManifestEntry entry(JsPluginPackagePath packagePath) throws IOException { - // jackson impl does buffering internally - try (final InputStream in = Files.newInputStream(packagePath.packageJson())) { - return OBJECT_MAPPER.readValue(in, JsPluginManifestEntry.class); - } } } diff --git a/server/jetty/src/main/java/io/deephaven/server/jetty/JsPluginsZipFilesystem.java b/server/jetty/src/main/java/io/deephaven/server/jetty/JsPluginsZipFilesystem.java index 554ac81098b..7250a8e71b0 100644 --- a/server/jetty/src/main/java/io/deephaven/server/jetty/JsPluginsZipFilesystem.java +++ b/server/jetty/src/main/java/io/deephaven/server/jetty/JsPluginsZipFilesystem.java @@ -4,8 +4,8 @@ package io.deephaven.server.jetty; import io.deephaven.configuration.CacheDir; -import io.deephaven.plugin.js.JsPluginManifestPath; -import io.deephaven.plugin.js.JsPluginPackagePath; +import io.deephaven.plugin.js.JsPlugin; +import io.deephaven.server.plugin.js.JsPluginManifest; import java.io.IOException; import java.io.OutputStream; @@ -14,6 +14,7 @@ import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.PathMatcher; import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.List; @@ -21,6 +22,7 @@ import java.util.Objects; import static io.deephaven.server.jetty.Json.OBJECT_MAPPER; +import static io.deephaven.server.plugin.js.JsPluginManifest.MANIFEST_JSON; class JsPluginsZipFilesystem { private static final String ZIP_ROOT = "/"; @@ -44,37 +46,38 @@ public static JsPluginsZipFilesystem create() throws IOException { } private final URI filesystem; - private final List entries; + private final List plugins; private JsPluginsZipFilesystem(URI filesystem) { this.filesystem = Objects.requireNonNull(filesystem); - this.entries = new ArrayList<>(); + this.plugins = new ArrayList<>(); } public URI filesystem() { return filesystem; } - public synchronized void copyFrom(JsPluginPackagePath srcPackagePath, JsPluginManifestEntry srcEntry) - throws IOException { - checkExisting(srcEntry); + public synchronized void add(JsPlugin plugin) throws IOException { + checkExisting(plugin.name()); // TODO(deephaven-core#3005): js-plugins checksum-based caching // Note: FileSystem#close is necessary to write out contents for ZipFileSystem try (final FileSystem fs = FileSystems.newFileSystem(filesystem, Map.of())) { - final JsPluginManifestPath manifest = manifest(fs); - copyRecursive(srcPackagePath, manifest.packagePath(srcEntry.name())); - entries.add(srcEntry); + final Path manifestRoot = manifestRoot(fs); + final Path dstPath = manifestRoot.resolve(plugin.name()); + // This is using internal knowledge that paths() must be PathsInternal and extends PathsMatcher. + final PathMatcher pathMatcher = (PathMatcher) plugin.paths(); + // If listing and traversing the contents of development directories (and skipping the copy) becomes + // too expensive, we can add logic here wrt PathsInternal/PathsPrefix to specify a dirMatcher. Or, + // properly route directly from the filesystem via Jetty. + CopyHelper.copyRecursive(plugin.path(), dstPath, pathMatcher); + plugins.add(plugin); writeManifest(fs); } } - private static void copyRecursive(JsPluginPackagePath src, JsPluginPackagePath dst) throws IOException { - CopyHelper.copyRecursive(src.path(), dst.path()); - } - - private void checkExisting(JsPluginManifestEntry info) { - for (JsPluginManifestEntry existing : entries) { - if (info.name().equals(existing.name())) { + private void checkExisting(String name) { + for (JsPlugin existing : plugins) { + if (name.equals(existing.name())) { // TODO(deephaven-core#3048): Improve JS plugin support around plugins with conflicting names throw new IllegalArgumentException(String.format( "js plugin with name '%s' already exists. See https://github.com/deephaven/deephaven-core/issues/3048", @@ -91,11 +94,11 @@ private synchronized void init() throws IOException { } private void writeManifest(FileSystem fs) throws IOException { - final Path manifestJson = manifest(fs).manifestJson(); + final Path manifestJson = manifestRoot(fs).resolve(MANIFEST_JSON); final Path manifestJsonTmp = manifestJson.resolveSibling(manifestJson.getFileName().toString() + ".tmp"); // jackson impl does buffering internally try (final OutputStream out = Files.newOutputStream(manifestJsonTmp)) { - OBJECT_MAPPER.writeValue(out, JsPluginManifest.of(entries)); + OBJECT_MAPPER.writeValue(out, manifest()); out.flush(); } Files.move(manifestJsonTmp, manifestJson, @@ -104,7 +107,11 @@ private void writeManifest(FileSystem fs) throws IOException { StandardCopyOption.ATOMIC_MOVE); } - private static JsPluginManifestPath manifest(FileSystem fs) { - return JsPluginManifestPath.of(fs.getPath(ZIP_ROOT)); + private JsPluginManifest manifest() { + return JsPluginManifest.from(plugins); + } + + private static Path manifestRoot(FileSystem fs) { + return fs.getPath(ZIP_ROOT); } } diff --git a/server/jetty/src/test/java/io/deephaven/server/jetty/JettyFlightRoundTripTest.java b/server/jetty/src/test/java/io/deephaven/server/jetty/JettyFlightRoundTripTest.java index 9d00dfd06b9..7278b18ee3f 100644 --- a/server/jetty/src/test/java/io/deephaven/server/jetty/JettyFlightRoundTripTest.java +++ b/server/jetty/src/test/java/io/deephaven/server/jetty/JettyFlightRoundTripTest.java @@ -6,20 +6,27 @@ import dagger.Component; import dagger.Module; import dagger.Provides; -import io.deephaven.server.arrow.ArrowModule; -import io.deephaven.server.config.ConfigServiceModule; -import io.deephaven.server.console.ConsoleModule; -import io.deephaven.server.log.LogModule; +import io.deephaven.server.jetty.js.Example123Registration; +import io.deephaven.server.jetty.js.Sentinel; +import io.deephaven.server.plugin.js.JsPluginsManifestRegistration; +import io.deephaven.server.plugin.js.JsPluginsNpmPackageRegistration; import io.deephaven.server.runner.ExecutionContextUnitTestModule; -import io.deephaven.server.session.ObfuscatingErrorTransformerModule; -import io.deephaven.server.session.SessionModule; -import io.deephaven.server.table.TableModule; -import io.deephaven.server.test.TestAuthModule; import io.deephaven.server.test.FlightMessageRoundTripTest; +import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.http.HttpFields; +import org.eclipse.jetty.http.HttpMethod; +import org.eclipse.jetty.http.HttpStatus; +import org.junit.Test; import javax.inject.Singleton; +import java.nio.file.Path; import java.time.Duration; import java.time.temporal.ChronoUnit; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; + +import static org.assertj.core.api.Assertions.assertThat; public class JettyFlightRoundTripTest extends FlightMessageRoundTripTest { @@ -36,18 +43,10 @@ static JettyConfig providesJettyConfig() { @Singleton @Component(modules = { - ArrowModule.class, - ConfigServiceModule.class, - ConsoleModule.class, ExecutionContextUnitTestModule.class, FlightTestModule.class, JettyServerModule.class, JettyTestConfig.class, - LogModule.class, - SessionModule.class, - TableModule.class, - TestAuthModule.class, - ObfuscatingErrorTransformerModule.class, }) public interface JettyTestComponent extends TestComponent { } @@ -56,4 +55,142 @@ public interface JettyTestComponent extends TestComponent { protected TestComponent component() { return DaggerJettyFlightRoundTripTest_JettyTestComponent.create(); } + + @Test + public void jsPlugins() throws Exception { + // Note: JettyFlightRoundTripTest is not the most minimal / appropriate bootstrapping for this test, but it is + // the most convenient since it has all of the necessary prerequisites + new Example123Registration().registerInto(component.registration()); + testJsPluginExamples(false, true, true); + } + + @Test + public void jsPluginsFromManifest() throws Exception { + // Note: JettyFlightRoundTripTest is not the most minimal / appropriate bootstrapping for this test, but it is + // the most convenient since it has all of the necessary prerequisites + final Path manifestRoot = Path.of(Sentinel.class.getResource("examples").toURI()); + new JsPluginsManifestRegistration(manifestRoot) + .registerInto(component.registration()); + testJsPluginExamples(false, false, true); + } + + @Test + public void jsPluginsFromNpmPackages() throws Exception { + // Note: JettyFlightRoundTripTest is not the most minimal / appropriate bootstrapping for this test, but it is + // the most convenient since it has all of the necessary prerequisites + final Path example1Root = Path.of(Sentinel.class.getResource("examples/@deephaven_test/example1").toURI()); + final Path example2Root = Path.of(Sentinel.class.getResource("examples/@deephaven_test/example2").toURI()); + // example3 is *not* a npm package, no package.json. + new JsPluginsNpmPackageRegistration(example1Root) + .registerInto(component.registration()); + new JsPluginsNpmPackageRegistration(example2Root) + .registerInto(component.registration()); + testJsPluginExamples(true, true, false); + } + + private void testJsPluginExamples(boolean example1IsLimited, boolean example2IsLimited, boolean hasExample3) + throws Exception { + final HttpClient client = new HttpClient(); + client.start(); + try { + if (hasExample3) { + manifestTest123(client); + } else { + manifestTest12(client); + } + example1Tests(client, example1IsLimited); + example2Tests(client, example2IsLimited); + if (hasExample3) { + example3Tests(client); + } + } finally { + client.stop(); + } + } + + private void manifestTest12(HttpClient client) throws InterruptedException, TimeoutException, ExecutionException { + final ContentResponse manifestResponse = get(client, "js-plugins/manifest.json"); + assertOk(manifestResponse, "application/json", + "{\"plugins\":[{\"name\":\"@deephaven_test/example1\",\"version\":\"0.1.0\",\"main\":\"dist/index.js\"},{\"name\":\"@deephaven_test/example2\",\"version\":\"0.2.0\",\"main\":\"dist/index.js\"}]}"); + } + + private void manifestTest123(HttpClient client) throws InterruptedException, TimeoutException, ExecutionException { + final ContentResponse manifestResponse = get(client, "js-plugins/manifest.json"); + assertOk(manifestResponse, "application/json", + "{\"plugins\":[{\"name\":\"@deephaven_test/example1\",\"version\":\"0.1.0\",\"main\":\"dist/index.js\"},{\"name\":\"@deephaven_test/example2\",\"version\":\"0.2.0\",\"main\":\"dist/index.js\"},{\"name\":\"@deephaven_test/example3\",\"version\":\"0.3.0\",\"main\":\"index.js\"}]}"); + } + + private void example1Tests(HttpClient client, boolean isLimited) + throws InterruptedException, TimeoutException, ExecutionException { + if (isLimited) { + assertThat(get(client, "js-plugins/@deephaven_test/example1/package.json").getStatus()) + .isEqualTo(HttpStatus.NOT_FOUND_404); + } else { + assertOk(get(client, "js-plugins/@deephaven_test/example1/package.json"), + "application/json", + "{\"name\":\"@deephaven_test/example1\",\"version\":\"0.1.0\",\"main\":\"dist/index.js\",\"files\":[\"dist\"]}"); + } + + assertOk( + get(client, "js-plugins/@deephaven_test/example1/dist/index.js"), + "text/javascript", + "// example1/dist/index.js"); + + assertOk( + get(client, "js-plugins/@deephaven_test/example1/dist/index2.js"), + "text/javascript", + "// example1/dist/index2.js"); + } + + private void example2Tests(HttpClient client, boolean isLimited) + throws InterruptedException, TimeoutException, ExecutionException { + if (isLimited) { + assertThat(get(client, "js-plugins/@deephaven_test/example2/package.json").getStatus()) + .isEqualTo(HttpStatus.NOT_FOUND_404); + } else { + assertOk(get(client, "js-plugins/@deephaven_test/example2/package.json"), + "application/json", + "{\"name\":\"@deephaven_test/example2\",\"version\":\"0.2.0\",\"main\":\"dist/index.js\",\"files\":[\"dist\"]}"); + } + + assertOk( + get(client, "js-plugins/@deephaven_test/example2/dist/index.js"), + "text/javascript", + "// example2/dist/index.js"); + + assertOk( + get(client, "js-plugins/@deephaven_test/example2/dist/index2.js"), + "text/javascript", + "// example2/dist/index2.js"); + } + + private void example3Tests(HttpClient client) throws InterruptedException, TimeoutException, ExecutionException { + assertOk( + get(client, "js-plugins/@deephaven_test/example3/index.js"), + "text/javascript", + "// example3/index.js"); + } + + private ContentResponse get(HttpClient client, String path) + throws InterruptedException, TimeoutException, ExecutionException { + return client + .newRequest("localhost", localPort) + .path(path) + .method(HttpMethod.GET) + .send(); + } + + private static void assertOk(ContentResponse response, String contentType, String expected) { + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK_200); + assertThat(response.getMediaType()).isEqualTo(contentType); + assertThat(response.getContentAsString()).isEqualTo(expected); + assertNoCache(response); + } + + private static void assertNoCache(ContentResponse response) { + final HttpFields headers = response.getHeaders(); + assertThat(headers.getDateField("Expires")).isEqualTo(0); + assertThat(headers.get("Pragma")).isEqualTo("no-cache"); + assertThat(headers.get("Cache-control")).isEqualTo("no-cache, must-revalidate, pre-check=0, post-check=0"); + } } diff --git a/server/jetty/src/test/java/io/deephaven/server/jetty/js/Example123Registration.java b/server/jetty/src/test/java/io/deephaven/server/jetty/js/Example123Registration.java new file mode 100644 index 00000000000..9fd0e7300fb --- /dev/null +++ b/server/jetty/src/test/java/io/deephaven/server/jetty/js/Example123Registration.java @@ -0,0 +1,68 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.server.jetty.js; + +import io.deephaven.plugin.Registration; +import io.deephaven.plugin.js.JsPlugin; +import io.deephaven.plugin.js.Paths; + +import java.net.URISyntaxException; +import java.nio.file.Path; + +public final class Example123Registration implements Registration { + + public Example123Registration() {} + + @Override + public void registerInto(Callback callback) { + final JsPlugin example1; + final JsPlugin example2; + final JsPlugin example3; + try { + example1 = example1(); + example2 = example2(); + example3 = example3(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + callback.register(example1); + callback.register(example2); + callback.register(example3); + } + + private static JsPlugin example1() throws URISyntaxException { + final Path resourcePath = Path.of(Sentinel.class.getResource("examples/@deephaven_test/example1").toURI()); + final Path main = resourcePath.relativize(resourcePath.resolve("dist/index.js")); + return JsPlugin.builder() + .name("@deephaven_test/example1") + .version("0.1.0") + .main(main) + .path(resourcePath) + .build(); + } + + private static JsPlugin example2() throws URISyntaxException { + final Path resourcePath = Path.of(Sentinel.class.getResource("examples/@deephaven_test/example2").toURI()); + final Path dist = resourcePath.relativize(resourcePath.resolve("dist")); + final Path main = dist.resolve("index.js"); + return JsPlugin.builder() + .name("@deephaven_test/example2") + .version("0.2.0") + .main(main) + .path(resourcePath) + .paths(Paths.ofPrefixes(dist)) + .build(); + } + + private static JsPlugin example3() throws URISyntaxException { + final Path resourcePath = Path.of(Sentinel.class.getResource("examples/@deephaven_test/example3").toURI()); + final Path main = resourcePath.relativize(resourcePath.resolve("index.js")); + return JsPlugin.builder() + .name("@deephaven_test/example3") + .version("0.3.0") + .main(main) + .path(resourcePath) + .build(); + } +} diff --git a/server/jetty/src/test/java/io/deephaven/server/jetty/js/Sentinel.java b/server/jetty/src/test/java/io/deephaven/server/jetty/js/Sentinel.java new file mode 100644 index 00000000000..ff61539b94e --- /dev/null +++ b/server/jetty/src/test/java/io/deephaven/server/jetty/js/Sentinel.java @@ -0,0 +1,8 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.server.jetty.js; + +public class Sentinel { + // just for the class +} diff --git a/server/jetty/src/test/java/io/deephaven/server/plugin/js/JsPluginsManifestRegistration.java b/server/jetty/src/test/java/io/deephaven/server/plugin/js/JsPluginsManifestRegistration.java new file mode 100644 index 00000000000..508eae4521e --- /dev/null +++ b/server/jetty/src/test/java/io/deephaven/server/plugin/js/JsPluginsManifestRegistration.java @@ -0,0 +1,35 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.server.plugin.js; + +import io.deephaven.plugin.Registration; +import io.deephaven.plugin.js.JsPlugin; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Objects; + +public class JsPluginsManifestRegistration implements Registration { + + private final Path path; + + public JsPluginsManifestRegistration(Path path) { + this.path = Objects.requireNonNull(path); + } + + @Override + public void registerInto(Callback callback) { + final List plugins; + try { + plugins = JsPluginsFromManifest.of(path); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + for (JsPlugin plugin : plugins) { + callback.register(plugin); + } + } +} diff --git a/server/jetty/src/test/java/io/deephaven/server/plugin/js/JsPluginsNpmPackageRegistration.java b/server/jetty/src/test/java/io/deephaven/server/plugin/js/JsPluginsNpmPackageRegistration.java new file mode 100644 index 00000000000..ded46d30b7d --- /dev/null +++ b/server/jetty/src/test/java/io/deephaven/server/plugin/js/JsPluginsNpmPackageRegistration.java @@ -0,0 +1,32 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.server.plugin.js; + +import io.deephaven.plugin.Registration; +import io.deephaven.plugin.js.JsPlugin; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.Objects; + +public class JsPluginsNpmPackageRegistration implements Registration { + + private final Path path; + + public JsPluginsNpmPackageRegistration(Path path) { + this.path = Objects.requireNonNull(path); + } + + @Override + public void registerInto(Callback callback) { + final JsPlugin plugin; + try { + plugin = JsPluginFromNpmPackage.of(path); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + callback.register(plugin); + } +} diff --git a/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example1/dist/index.js b/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example1/dist/index.js new file mode 100644 index 00000000000..de46952cc24 --- /dev/null +++ b/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example1/dist/index.js @@ -0,0 +1 @@ +// example1/dist/index.js \ No newline at end of file diff --git a/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example1/dist/index2.js b/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example1/dist/index2.js new file mode 100644 index 00000000000..ef31df67846 --- /dev/null +++ b/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example1/dist/index2.js @@ -0,0 +1 @@ +// example1/dist/index2.js \ No newline at end of file diff --git a/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example1/package.json b/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example1/package.json new file mode 100644 index 00000000000..b3733e4fe6d --- /dev/null +++ b/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example1/package.json @@ -0,0 +1 @@ +{"name":"@deephaven_test/example1","version":"0.1.0","main":"dist/index.js","files":["dist"]} \ No newline at end of file diff --git a/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example2/dist/index.js b/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example2/dist/index.js new file mode 100644 index 00000000000..f84594080ee --- /dev/null +++ b/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example2/dist/index.js @@ -0,0 +1 @@ +// example2/dist/index.js \ No newline at end of file diff --git a/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example2/dist/index2.js b/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example2/dist/index2.js new file mode 100644 index 00000000000..536a8edceee --- /dev/null +++ b/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example2/dist/index2.js @@ -0,0 +1 @@ +// example2/dist/index2.js \ No newline at end of file diff --git a/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example2/package.json b/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example2/package.json new file mode 100644 index 00000000000..64ca82a446b --- /dev/null +++ b/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example2/package.json @@ -0,0 +1 @@ +{"name":"@deephaven_test/example2","version":"0.2.0","main":"dist/index.js","files":["dist"]} \ No newline at end of file diff --git a/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example3/index.js b/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example3/index.js new file mode 100644 index 00000000000..62c773b6ad4 --- /dev/null +++ b/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/@deephaven_test/example3/index.js @@ -0,0 +1 @@ +// example3/index.js \ No newline at end of file diff --git a/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/manifest.json b/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/manifest.json new file mode 100644 index 00000000000..eaa745d1b6b --- /dev/null +++ b/server/jetty/src/test/resources/io/deephaven/server/jetty/js/examples/manifest.json @@ -0,0 +1,19 @@ +{ + "plugins": [ + { + "name": "@deephaven_test/example1", + "main": "dist/index.js", + "version": "0.1.0" + }, + { + "name": "@deephaven_test/example2", + "main": "dist/index.js", + "version": "0.2.0" + }, + { + "name": "@deephaven_test/example3", + "main": "index.js", + "version": "0.3.0" + } + ] +} diff --git a/server/netty/src/test/java/io/deephaven/server/netty/NettyFlightRoundTripTest.java b/server/netty/src/test/java/io/deephaven/server/netty/NettyFlightRoundTripTest.java index 1876b0924c2..89cc9833d26 100644 --- a/server/netty/src/test/java/io/deephaven/server/netty/NettyFlightRoundTripTest.java +++ b/server/netty/src/test/java/io/deephaven/server/netty/NettyFlightRoundTripTest.java @@ -6,15 +6,7 @@ import dagger.Component; import dagger.Module; import dagger.Provides; -import io.deephaven.server.arrow.ArrowModule; -import io.deephaven.server.config.ConfigServiceModule; -import io.deephaven.server.console.ConsoleModule; -import io.deephaven.server.log.LogModule; import io.deephaven.server.runner.ExecutionContextUnitTestModule; -import io.deephaven.server.session.ObfuscatingErrorTransformerModule; -import io.deephaven.server.session.SessionModule; -import io.deephaven.server.table.TableModule; -import io.deephaven.server.test.TestAuthModule; import io.deephaven.server.test.FlightMessageRoundTripTest; import javax.inject.Singleton; @@ -36,18 +28,10 @@ static NettyConfig providesNettyConfig() { @Singleton @Component(modules = { - ArrowModule.class, - ConfigServiceModule.class, - ConsoleModule.class, ExecutionContextUnitTestModule.class, FlightTestModule.class, - LogModule.class, NettyServerModule.class, NettyTestConfig.class, - SessionModule.class, - TableModule.class, - TestAuthModule.class, - ObfuscatingErrorTransformerModule.class, }) public interface NettyTestComponent extends TestComponent { } diff --git a/server/src/main/java/io/deephaven/server/plugin/PluginRegistrationVisitor.java b/server/src/main/java/io/deephaven/server/plugin/PluginRegistrationVisitor.java index 79d766f66f8..97ec345a555 100644 --- a/server/src/main/java/io/deephaven/server/plugin/PluginRegistrationVisitor.java +++ b/server/src/main/java/io/deephaven/server/plugin/PluginRegistrationVisitor.java @@ -11,7 +11,6 @@ import javax.inject.Inject; import java.util.Objects; -import java.util.function.Consumer; /** * Plugin {@link io.deephaven.plugin.Registration.Callback} implementation that forwards registered plugins to a diff --git a/server/src/main/java/io/deephaven/server/plugin/js/Jackson.java b/server/src/main/java/io/deephaven/server/plugin/js/Jackson.java new file mode 100644 index 00000000000..1454b3e5735 --- /dev/null +++ b/server/src/main/java/io/deephaven/server/plugin/js/Jackson.java @@ -0,0 +1,12 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.server.plugin.js; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +class Jackson { + static final ObjectMapper OBJECT_MAPPER = + new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); +} diff --git a/server/src/main/java/io/deephaven/server/plugin/js/JsPluginConfigDirRegistration.java b/server/src/main/java/io/deephaven/server/plugin/js/JsPluginConfigDirRegistration.java new file mode 100644 index 00000000000..96b36d7d157 --- /dev/null +++ b/server/src/main/java/io/deephaven/server/plugin/js/JsPluginConfigDirRegistration.java @@ -0,0 +1,65 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.server.plugin.js; + +import dagger.Binds; +import dagger.multibindings.IntoSet; +import io.deephaven.configuration.ConfigDir; +import io.deephaven.plugin.Registration; +import io.deephaven.plugin.js.JsPlugin; + +import javax.inject.Inject; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static io.deephaven.server.plugin.js.JsPluginManifest.MANIFEST_JSON; + + +/** + * Registers the {@link JsPlugin JS plugins} sourced from the {@link JsPluginManifest manifest} root located at + * {@link ConfigDir} / {@value JS_PLUGINS} (if {@value io.deephaven.server.plugin.js.JsPluginManifest#MANIFEST_JSON} + * exists). + */ +public final class JsPluginConfigDirRegistration implements Registration { + + public static final String JS_PLUGINS = "js-plugins"; + + /** + * Binds {@link JsPluginConfigDirRegistration} into the set of {@link Registration}. + */ + @dagger.Module + public interface Module { + @Binds + @IntoSet + Registration bindsRegistration(JsPluginConfigDirRegistration registration); + } + + @Inject + JsPluginConfigDirRegistration() {} + + @Override + public void registerInto(Callback callback) { + // /js-plugins/ (manifest root) + final Path manifestRoot = ConfigDir.get() + .map(p -> p.resolve(JS_PLUGINS).resolve(MANIFEST_JSON)) + .filter(Files::exists) + .map(Path::getParent) + .orElse(null); + if (manifestRoot == null) { + return; + } + final List plugins; + try { + plugins = JsPluginsFromManifest.of(manifestRoot); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + for (JsPlugin plugin : plugins) { + callback.register(plugin); + } + } +} diff --git a/server/src/main/java/io/deephaven/server/plugin/js/JsPluginFromNpmPackage.java b/server/src/main/java/io/deephaven/server/plugin/js/JsPluginFromNpmPackage.java new file mode 100644 index 00000000000..a966904ba62 --- /dev/null +++ b/server/src/main/java/io/deephaven/server/plugin/js/JsPluginFromNpmPackage.java @@ -0,0 +1,40 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.server.plugin.js; + +import io.deephaven.plugin.js.JsPlugin; +import io.deephaven.plugin.js.JsPlugin.Builder; +import io.deephaven.plugin.js.Paths; + +import java.io.IOException; +import java.nio.file.Path; + +class JsPluginFromNpmPackage { + + static JsPlugin of(Path packageRoot) throws IOException { + final Path packageJsonPath = packageRoot.resolve(JsPluginNpmPackageRegistration.PACKAGE_JSON); + final NpmPackage packageJson = NpmPackage.read(packageJsonPath); + final Path main = packageRoot.relativize(packageRoot.resolve(packageJson.main())); + final Paths paths; + if (main.getNameCount() > 1) { + // We're requiring that all of the necessary files to serve be under the top-level directory as sourced from + // package.json/main. For example, "build/index.js" -> "build", "dist/bundle/index.js" -> "dist". This + // supports development use cases where the top-level directory may be interspersed with unrelated + // development files (node_modules, .git, etc). + // + // Note: this logic only comes into play for development use cases where plugins are configured via + // deephaven.jsPlugins.myPlugin=/path/to/my/js + paths = Paths.ofPrefixes(main.subpath(0, 1)); + } else { + paths = Paths.all(); + } + final Builder builder = JsPlugin.builder() + .name(packageJson.name()) + .version(packageJson.version()) + .main(main) + .path(packageRoot) + .paths(paths); + return builder.build(); + } +} diff --git a/server/src/main/java/io/deephaven/server/plugin/js/JsPluginManifest.java b/server/src/main/java/io/deephaven/server/plugin/js/JsPluginManifest.java new file mode 100644 index 00000000000..a74bb7e1c77 --- /dev/null +++ b/server/src/main/java/io/deephaven/server/plugin/js/JsPluginManifest.java @@ -0,0 +1,48 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.server.plugin.js; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.deephaven.annotations.SimpleStyle; +import io.deephaven.plugin.js.JsPlugin; +import org.immutables.value.Value.Immutable; +import org.immutables.value.Value.Parameter; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; + +import static io.deephaven.server.plugin.js.Jackson.OBJECT_MAPPER; + +@Immutable +@SimpleStyle +public abstract class JsPluginManifest { + public static final String PLUGINS = "plugins"; + public static final String MANIFEST_JSON = "manifest.json"; + + @JsonCreator + public static JsPluginManifest of( + @JsonProperty(value = PLUGINS, required = true) List plugins) { + return ImmutableJsPluginManifest.of(plugins); + } + + public static JsPluginManifest from(List plugins) { + return of(plugins.stream().map(JsPluginManifestEntry::from).collect(Collectors.toList())); + } + + static JsPluginManifest read(Path manifestJson) throws IOException { + // jackson impl does buffering internally + try (final InputStream in = Files.newInputStream(manifestJson)) { + return OBJECT_MAPPER.readValue(in, JsPluginManifest.class); + } + } + + @Parameter + @JsonProperty(PLUGINS) + public abstract List plugins(); +} diff --git a/server/jetty/src/main/java/io/deephaven/server/jetty/JsPluginManifestEntry.java b/server/src/main/java/io/deephaven/server/plugin/js/JsPluginManifestEntry.java similarity index 82% rename from server/jetty/src/main/java/io/deephaven/server/jetty/JsPluginManifestEntry.java rename to server/src/main/java/io/deephaven/server/plugin/js/JsPluginManifestEntry.java index 385397809d9..4aeca8b05ab 100644 --- a/server/jetty/src/main/java/io/deephaven/server/jetty/JsPluginManifestEntry.java +++ b/server/src/main/java/io/deephaven/server/plugin/js/JsPluginManifestEntry.java @@ -1,11 +1,12 @@ /** * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending */ -package io.deephaven.server.jetty; +package io.deephaven.server.plugin.js; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import io.deephaven.annotations.SimpleStyle; +import io.deephaven.plugin.js.JsPlugin; import org.immutables.value.Value.Immutable; import org.immutables.value.Value.Parameter; @@ -14,7 +15,7 @@ */ @Immutable @SimpleStyle -abstract class JsPluginManifestEntry { +public abstract class JsPluginManifestEntry { public static final String NAME = "name"; public static final String VERSION = "version"; @@ -28,6 +29,10 @@ public static JsPluginManifestEntry of( return ImmutableJsPluginManifestEntry.of(name, version, main); } + public static JsPluginManifestEntry from(JsPlugin plugin) { + return of(plugin.name(), plugin.version(), plugin.main().toString()); + } + /** * The name of the plugin. */ diff --git a/server/src/main/java/io/deephaven/server/plugin/js/JsPluginManifestRegistration.java b/server/src/main/java/io/deephaven/server/plugin/js/JsPluginManifestRegistration.java new file mode 100644 index 00000000000..93f1aa66f39 --- /dev/null +++ b/server/src/main/java/io/deephaven/server/plugin/js/JsPluginManifestRegistration.java @@ -0,0 +1,56 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.server.plugin.js; + +import dagger.Binds; +import dagger.multibindings.IntoSet; +import io.deephaven.configuration.Configuration; +import io.deephaven.plugin.Registration; +import io.deephaven.plugin.js.JsPlugin; + +import javax.inject.Inject; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.List; + +/** + * Registers the {@link JsPlugin JS plugins} sourced from the {@link JsPluginManifest manifest} root configuration + * property {@value JsPluginManifestRegistration#JS_PLUGIN_RESOURCE_BASE}. + */ +public final class JsPluginManifestRegistration implements Registration { + + public static final String JS_PLUGIN_RESOURCE_BASE = JsPluginModule.DEEPHAVEN_JS_PLUGINS_PREFIX + "resourceBase"; + + /** + * Binds {@link JsPluginManifestRegistration} into the set of {@link Registration}. + */ + @dagger.Module + public interface Module { + @Binds + @IntoSet + Registration bindsRegistration(JsPluginManifestRegistration registration); + } + + @Inject + JsPluginManifestRegistration() {} + + @Override + public void registerInto(Callback callback) { + // deephaven.jsPlugins.resourceBase (manifest root) + final String resourceBase = Configuration.getInstance().getStringWithDefault(JS_PLUGIN_RESOURCE_BASE, null); + if (resourceBase == null) { + return; + } + final List plugins; + try { + plugins = JsPluginsFromManifest.of(Path.of(resourceBase)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + for (JsPlugin plugin : plugins) { + callback.register(plugin); + } + } +} diff --git a/server/src/main/java/io/deephaven/server/plugin/js/JsPluginModule.java b/server/src/main/java/io/deephaven/server/plugin/js/JsPluginModule.java index 06f0dcf4ea9..1b966150836 100644 --- a/server/src/main/java/io/deephaven/server/plugin/js/JsPluginModule.java +++ b/server/src/main/java/io/deephaven/server/plugin/js/JsPluginModule.java @@ -4,116 +4,19 @@ package io.deephaven.server.plugin.js; import dagger.Module; -import dagger.Provides; -import dagger.multibindings.ElementsIntoSet; import io.deephaven.configuration.ConfigDir; -import io.deephaven.configuration.Configuration; -import io.deephaven.plugin.Registration; -import io.deephaven.plugin.js.JsPluginManifestPath; -import io.deephaven.plugin.js.JsPluginPackagePath; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Optional; -import java.util.Set; /** - * Provides the {@link JsPluginManifestPath manifest path} of {@value JS_PLUGIN_RESOURCE_BASE} if the configuration - * property is set. Provides the {@link JsPluginManifestPath manifest path} of {@link ConfigDir} / {@value JS_PLUGINS} - * if {@value JsPluginManifestPath#MANIFEST_JSON} exists. Provides the {@link JsPluginPackagePath package path} for all - * configuration properties that start with {@value DEEPHAVEN_JS_PLUGINS_PREFIX} and have a single part after. + * Includes the modules {@link JsPluginManifestRegistration.Module}, {@link JsPluginConfigDirRegistration.Module}, and + * {@link JsPluginNpmPackageRegistration.Module}; these modules add various means of configuration support for producing + * and registering {@link io.deephaven.plugin.js.JsPlugin}. */ -@Module +@Module(includes = { + JsPluginManifestRegistration.Module.class, + JsPluginConfigDirRegistration.Module.class, + JsPluginNpmPackageRegistration.Module.class, +}) public interface JsPluginModule { String DEEPHAVEN_JS_PLUGINS_PREFIX = "deephaven.jsPlugins."; - String JS_PLUGIN_RESOURCE_BASE = DEEPHAVEN_JS_PLUGINS_PREFIX + "resourceBase"; - String JS_PLUGINS = "js-plugins"; - - @Provides - @ElementsIntoSet - static Set providesResourceBaseRegistration() { - return jsPluginsResourceBase() - .map(Registration.class::cast) - .map(Set::of) - .orElseGet(Set::of); - } - - @Provides - @ElementsIntoSet - static Set providesConfigDirRegistration() { - return jsPluginsConfigDir() - .map(Registration.class::cast) - .map(Set::of) - .orElseGet(Set::of); - } - - @Provides - @ElementsIntoSet - static Set providesPackageRoots() { - return Set.copyOf(jsPluginsPackageRoots()); - } - - // deephaven.jsPlugins.resourceBase (manifest root) - private static Optional jsPluginsResourceBase() { - final String resourceBase = Configuration.getInstance().getStringWithDefault(JS_PLUGIN_RESOURCE_BASE, null); - return Optional.ofNullable(resourceBase) - .map(Path::of) - .map(JsPluginManifestPath::of); - } - - // /js-plugins/ (manifest root) - private static Optional jsPluginsConfigDir() { - return ConfigDir.get() - .map(JsPluginModule::resolveJsPlugins) - .map(JsPluginManifestPath::of) - .filter(JsPluginModule::manifestJsonExists); - } - - private static Path resolveJsPlugins(Path p) { - return p.resolve(JS_PLUGINS); - } - - private static boolean manifestJsonExists(JsPluginManifestPath path) { - return Files.exists(path.manifestJson()); - } - - // deephaven.jsPlugins. (package root) - private static Set jsPluginsPackageRoots() { - final Configuration config = Configuration.getInstance(); - final Set parts = partsThatStartWith(DEEPHAVEN_JS_PLUGINS_PREFIX, config); - final Set packageRoots = new HashSet<>(parts.size()); - for (String part : parts) { - final String propertyName = DEEPHAVEN_JS_PLUGINS_PREFIX + part; - if (JS_PLUGIN_RESOURCE_BASE.equals(propertyName)) { - // handled by jsPluginsResourceBase - continue; - } - final String packageRoot = config.getStringWithDefault(propertyName, null); - if (packageRoot == null) { - continue; - } - packageRoots.add(JsPluginPackagePath.of(Path.of(packageRoot))); - } - return packageRoots; - } - - private static Set partsThatStartWith(String prefix, Configuration configuration) { - final Set parts = new HashSet<>(); - final Iterator it = configuration.getProperties(prefix).keys().asIterator(); - while (it.hasNext()) { - final Object next = it.next(); - if (next instanceof String) { - parts.add(firstPart((String) next)); - } - } - return parts; - } - - private static String firstPart(String x) { - final int index = x.indexOf('.'); - return index == -1 ? x : x.substring(0, index); - } } diff --git a/server/src/main/java/io/deephaven/server/plugin/js/JsPluginNpmPackageRegistration.java b/server/src/main/java/io/deephaven/server/plugin/js/JsPluginNpmPackageRegistration.java new file mode 100644 index 00000000000..0acf62337ce --- /dev/null +++ b/server/src/main/java/io/deephaven/server/plugin/js/JsPluginNpmPackageRegistration.java @@ -0,0 +1,130 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.server.plugin.js; + +import dagger.Binds; +import dagger.multibindings.IntoSet; +import io.deephaven.configuration.Configuration; +import io.deephaven.plugin.Registration; +import io.deephaven.plugin.js.JsPlugin; + +import javax.inject.Inject; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystemNotFoundException; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +import static io.deephaven.server.plugin.js.JsPluginManifestRegistration.JS_PLUGIN_RESOURCE_BASE; +import static io.deephaven.server.plugin.js.JsPluginModule.DEEPHAVEN_JS_PLUGINS_PREFIX; + +/** + * Registers the {@link JsPlugin JS plugins} sourced from the NPM package roots as specified via the configuration + * properties that start with {@value io.deephaven.server.plugin.js.JsPluginModule#DEEPHAVEN_JS_PLUGINS_PREFIX}. This + * configuration is meant for development-oriented use-cases and is done on a "best-effort" basis. + * + *

+ * The configuration value of the above property corresponds to the {@link JsPlugin#path()} directory. A + * {@value PACKAGE_JSON} must exist in this directory (as specified via + * package-json). The {@value NAME} json value + * corresponds to {@link JsPlugin#name()}, the {@value VERSION} json value corresponds to {@link JsPlugin#version()}, + * and the {@value MAIN} json value corresponds to {@link JsPlugin#main()}. Furthermore, the top-level directory of the + * {@value MAIN} json value will be used to set {@link JsPlugin#paths()} using + * {@link io.deephaven.plugin.js.Paths#ofPrefixes(Path)}; for example, a {@value MAIN} of "build/index.js" will limit + * the resources to the "build/" directory; a {@value MAIN} of "dist/bundle/index.js" will limit the resources to the + * "dist/" directory. + */ +public final class JsPluginNpmPackageRegistration implements Registration { + public static final String PACKAGE_JSON = "package.json"; + public static final String NAME = "name"; + public static final String VERSION = "version"; + public static final String MAIN = "main"; + + // TODO(deephaven-core#4817): JS Plugins development is slow + // We may wish to make parsing NPM package.json easier with a bespoke "deephaven" field, or try to exactly match + // the semantics of the existing "files" field. This may not be necessary if the top-level directory of "main" works + // well enough, or if we have a non-copying jetty route-based impl. + + /** + * Binds {@link JsPluginNpmPackageRegistration} into the set of {@link Registration}. + */ + @dagger.Module + public interface Module { + @Binds + @IntoSet + Registration bindsRegistration(JsPluginNpmPackageRegistration registration); + } + + @Inject + JsPluginNpmPackageRegistration() {} + + @Override + public void registerInto(Callback callback) { + // deephaven.jsPlugins. (package root) + final Configuration config = Configuration.getInstance(); + final Set parts = partsThatStartWith(DEEPHAVEN_JS_PLUGINS_PREFIX, config); + for (String part : parts) { + final String propertyName = DEEPHAVEN_JS_PLUGINS_PREFIX + part; + if (JS_PLUGIN_RESOURCE_BASE.equals(propertyName)) { + // handled by jsPluginsResourceBase + continue; + } + final String packageRoot = config.getStringWithDefault(propertyName, null); + if (packageRoot == null) { + continue; + } + URI uri = URI.create(packageRoot); + if (uri.getScheme() == null) { + uri = URI.create("file:" + packageRoot); + } + final FileSystem fileSystem = getOrCreateFileSystem(uri); + final JsPlugin plugin; + try { + plugin = JsPluginFromNpmPackage.of(fileSystem.provider().getPath(uri)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + callback.register(plugin); + } + } + + private static FileSystem getOrCreateFileSystem(URI uri) { + if ("file".equalsIgnoreCase(uri.getScheme())) { + return FileSystems.getDefault(); + } + try { + return FileSystems.getFileSystem(uri); + } catch (FileSystemNotFoundException e) { + // ignore + } + try { + return FileSystems.newFileSystem(uri, Map.of()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static Set partsThatStartWith(String prefix, Configuration configuration) { + final Set parts = new HashSet<>(); + final Iterator it = configuration.getProperties(prefix).keys().asIterator(); + while (it.hasNext()) { + final Object next = it.next(); + if (next instanceof String) { + parts.add(firstPart((String) next)); + } + } + return parts; + } + + private static String firstPart(String x) { + final int index = x.indexOf('.'); + return index == -1 ? x : x.substring(0, index); + } +} diff --git a/server/src/main/java/io/deephaven/server/plugin/js/JsPluginsFromManifest.java b/server/src/main/java/io/deephaven/server/plugin/js/JsPluginsFromManifest.java new file mode 100644 index 00000000000..c07540d27db --- /dev/null +++ b/server/src/main/java/io/deephaven/server/plugin/js/JsPluginsFromManifest.java @@ -0,0 +1,33 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.server.plugin.js; + +import io.deephaven.plugin.js.JsPlugin; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +class JsPluginsFromManifest { + + static List of(Path manifestRoot) throws IOException { + final JsPluginManifest manifest = JsPluginManifest.read(manifestRoot.resolve(JsPluginManifest.MANIFEST_JSON)); + final List plugins = new ArrayList<>(manifest.plugins().size()); + for (JsPluginManifestEntry entry : manifest.plugins()) { + final Path pluginPath = manifestRoot.resolve(entry.name()); + final Path pluginMain = pluginPath.relativize(pluginPath.resolve(entry.main())); + final JsPlugin plugin = JsPlugin.builder() + .name(entry.name()) + .version(entry.version()) + .main(pluginMain) + .path(pluginPath) + .build(); + // We expect manifests to be "production" use cases - they should already be packed as appropriate. + // Additionally, there is no strict requirement that they have package.json anyways. + plugins.add(plugin); + } + return plugins; + } +} diff --git a/server/src/main/java/io/deephaven/server/plugin/js/NpmPackage.java b/server/src/main/java/io/deephaven/server/plugin/js/NpmPackage.java new file mode 100644 index 00000000000..711dfa34831 --- /dev/null +++ b/server/src/main/java/io/deephaven/server/plugin/js/NpmPackage.java @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.server.plugin.js; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.deephaven.annotations.SimpleStyle; +import org.immutables.value.Value.Immutable; +import org.immutables.value.Value.Parameter; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import static io.deephaven.server.plugin.js.Jackson.OBJECT_MAPPER; + +@Immutable +@SimpleStyle +abstract class NpmPackage { + + @JsonCreator + public static NpmPackage of( + @JsonProperty(value = JsPluginNpmPackageRegistration.NAME, required = true) String name, + @JsonProperty(value = JsPluginNpmPackageRegistration.VERSION, required = true) String version, + @JsonProperty(value = JsPluginNpmPackageRegistration.MAIN, required = true) String main) { + return ImmutableNpmPackage.of(name, version, main); + } + + public static NpmPackage read(Path packageJson) throws IOException { + // jackson impl does buffering internally + try (final InputStream in = Files.newInputStream(packageJson)) { + return OBJECT_MAPPER.readValue(in, NpmPackage.class); + } + } + + @Parameter + @JsonProperty(JsPluginNpmPackageRegistration.NAME) + public abstract String name(); + + @Parameter + @JsonProperty(JsPluginNpmPackageRegistration.VERSION) + public abstract String version(); + + @Parameter + @JsonProperty(JsPluginNpmPackageRegistration.MAIN) + public abstract String main(); +} diff --git a/server/src/main/java/io/deephaven/server/plugin/python/CallbackAdapter.java b/server/src/main/java/io/deephaven/server/plugin/python/CallbackAdapter.java index 27e2fcc09bd..7270a895b61 100644 --- a/server/src/main/java/io/deephaven/server/plugin/python/CallbackAdapter.java +++ b/server/src/main/java/io/deephaven/server/plugin/python/CallbackAdapter.java @@ -4,6 +4,7 @@ package io.deephaven.server.plugin.python; import io.deephaven.plugin.Registration.Callback; +import io.deephaven.plugin.js.JsPlugin; import org.jpy.PyObject; class CallbackAdapter { @@ -18,4 +19,9 @@ public CallbackAdapter(Callback callback) { public void registerObjectType(String name, PyObject objectTypeAdapter) { callback.register(new ObjectTypeAdapter(name, objectTypeAdapter)); } + + @SuppressWarnings("unused") + public void registerJsPlugin(JsPlugin jsPlugin) { + callback.register(jsPlugin); + } } diff --git a/server/test/src/main/java/io/deephaven/server/test/FlightMessageRoundTripTest.java b/server/test/src/main/java/io/deephaven/server/test/FlightMessageRoundTripTest.java index 67845489b7d..860eed84cb5 100644 --- a/server/test/src/main/java/io/deephaven/server/test/FlightMessageRoundTripTest.java +++ b/server/test/src/main/java/io/deephaven/server/test/FlightMessageRoundTripTest.java @@ -40,17 +40,24 @@ import io.deephaven.extensions.barrage.util.BarrageUtil; import io.deephaven.io.logger.LogBuffer; import io.deephaven.io.logger.LogBufferGlobal; +import io.deephaven.plugin.Registration; import io.deephaven.proto.backplane.grpc.SortTableRequest; import io.deephaven.proto.backplane.grpc.WrappedAuthenticationRequest; import io.deephaven.proto.backplane.script.grpc.BindTableToVariableRequest; import io.deephaven.proto.flight.util.FlightExportTicketHelper; import io.deephaven.proto.util.ScopeTicketHelper; import io.deephaven.qst.table.TicketTable; +import io.deephaven.server.arrow.ArrowModule; import io.deephaven.server.auth.AuthorizationProvider; +import io.deephaven.server.config.ConfigServiceModule; +import io.deephaven.server.console.ConsoleModule; import io.deephaven.server.console.ScopeTicketResolver; +import io.deephaven.server.log.LogModule; +import io.deephaven.server.plugin.PluginsModule; import io.deephaven.server.runner.GrpcServer; import io.deephaven.server.runner.MainHelper; import io.deephaven.server.session.*; +import io.deephaven.server.table.TableModule; import io.deephaven.server.test.TestAuthModule.FakeBearer; import io.deephaven.server.util.Scheduler; import io.deephaven.util.SafeCloseable; @@ -100,7 +107,17 @@ public abstract class FlightMessageRoundTripTest { private static final String ANONYMOUS = "Anonymous"; private static final String DISABLED_FOR_TEST = "Disabled For Test"; - @Module + @Module(includes = { + ArrowModule.class, + ConfigServiceModule.class, + ConsoleModule.class, + LogModule.class, + SessionModule.class, + TableModule.class, + TestAuthModule.class, + ObfuscatingErrorTransformerModule.class, + PluginsModule.class, + }) public static class FlightTestModule { @IntoSet @Provides @@ -187,11 +204,13 @@ public interface TestComponent { ExecutionContext executionContext(); TestAuthorizationProvider authorizationProvider(); + + Registration.Callback registration(); } private LogBuffer logBuffer; private GrpcServer server; - + protected int localPort; private FlightClient flightClient; protected SessionService sessionService; @@ -200,7 +219,7 @@ public interface TestComponent { private AbstractScriptSession scriptSession; private SafeCloseable executionContext; private Location serverLocation; - private TestComponent component; + protected TestComponent component; private ManagedChannel clientChannel; private ScheduledExecutorService clientScheduler; @@ -222,12 +241,12 @@ public void setup() throws IOException { server = component.server(); server.start(); - int actualPort = server.getPort(); + localPort = server.getPort(); scriptSession = component.scriptSession(); sessionService = component.sessionService(); - serverLocation = Location.forGrpcInsecure("localhost", actualPort); + serverLocation = Location.forGrpcInsecure("localhost", localPort); currentSession = sessionService.newSession(new AuthContext.SuperUser()); flightClient = FlightClient.builder().location(serverLocation) .allocator(new RootAllocator()).intercept(info -> new FlightClientMiddleware() { @@ -244,7 +263,7 @@ public void onHeadersReceived(CallHeaders incomingHeaders) {} public void onCallCompleted(CallStatus status) {} }).build(); - clientChannel = ManagedChannelBuilder.forTarget("localhost:" + actualPort) + clientChannel = ManagedChannelBuilder.forTarget("localhost:" + localPort) .usePlaintext() .intercept(new TestAuthClientInterceptor(currentSession.getExpiration().token.toString())) .build();