From bf579e0bd14374657cc5dd0fa94cb2e04444ec72 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 20 Sep 2023 10:01:47 -0500 Subject: [PATCH] Simplify ScriptSession dependencies (#4190) This removes the only python init script, and removes an unused groovy init script. The config variables are left in place for now, though they will default to blank. One slightly unexpected change that this results in is that the jpy, sys, and os modules are no longer imported by default into a python script session - this probably should be considered a feature, but does need to be documented. Partial #4040 --- engine/table/build.gradle | 17 ---- .../groovy/core/deephaven_core_utils.groovy | 85 ------------------- .../table/python/core/deephaven_jpy_init.py | 18 ---- .../io/deephaven/engine/util/jpy/JpyInit.java | 3 +- .../main/resources}/groovy/0-base.groovy | 0 .../main/resources}/groovy/2-calendars.groovy | 0 .../resources}/groovy/4-count-metrics.groovy | 0 .../src/main/resources/dh-defaults.prop | 4 +- .../src/main/resources/dh-tests.prop | 4 +- py/server/deephaven_internal/jvm/__init__.py | 22 +++++ .../engine/util/WorkerPythonEnvironment.java | 4 +- 11 files changed, 30 insertions(+), 127 deletions(-) delete mode 100644 engine/table/groovy/core/deephaven_core_utils.groovy delete mode 100644 engine/table/python/core/deephaven_jpy_init.py rename engine/table/{groovy => src/main/resources}/groovy/0-base.groovy (100%) rename engine/table/{groovy => src/main/resources}/groovy/2-calendars.groovy (100%) rename engine/table/{groovy => src/main/resources}/groovy/4-count-metrics.groovy (100%) diff --git a/engine/table/build.gradle b/engine/table/build.gradle index eb5a39abccf..3269614fe89 100644 --- a/engine/table/build.gradle +++ b/engine/table/build.gradle @@ -6,23 +6,6 @@ plugins { description 'Engine Table: Implementation and closely-coupled utilities' -sourceSets { - main { - resources { - srcDir 'groovy' - include 'core/*.groovy' - } - resources { - srcDir 'python' - include 'core/*.py' - } - resources { - srcDir 'src/main/resources' - include '**' - } - } -} - configurations { // Ensure jmh picks up the same dependencies as tests jmhImplementation.extendsFrom testImplementation diff --git a/engine/table/groovy/core/deephaven_core_utils.groovy b/engine/table/groovy/core/deephaven_core_utils.groovy deleted file mode 100644 index 71cbbcc0384..00000000000 --- a/engine/table/groovy/core/deephaven_core_utils.groovy +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2016-2021 Deephaven Data Labs and Patent Pending - */ - -__QUERY_NAME__="core" - -source = {fileName -> - __groovySession.runScript(fileName) -} - -sourceOnce = {fileName -> - __groovySession.runScriptOnce(fileName) -} - -scriptImportClass = {c -> __groovySession.addScriptImportClass(c)} -scriptImportStatic = {c -> __groovySession.addScriptImportStatic(c)} - -isValidVariableName = {name -> name.matches("^[a-zA-Z_][a-zA-Z_0-9]*")} - -publishVariable = { String name, value -> - if(!isValidVariableName(name)){ - throw new RuntimeException("publishVariable: Attempting to publish an invalid variable name: " + name) - } - - binding.setVariable(name, value) -} - -removeVariable = {name -> - if(!isValidVariableName(name)){ - throw new RuntimeException("removeVariable: Attempting to remove an invalid variable name: " + name) - } - - binding.variables.remove(name) -} - -///////////////////// Performance ///////////////////// - -///////////////////// Calendars ///////////////////// -import static io.deephaven.time.calendar.Calendars.calendar -import static io.deephaven.time.calendar.Calendars.calendarNames - -//todo prefix name with CALENDAR? - -publishVariable( "CALENDAR_DEFAULT", calendar()) - -for( String n : calendarNames() ) { - publishVariable("CALENDAR_" + n, calendar(n)) -} - - -///////////////////// Plotting ///////////////////// - -//todo what about these commented out imports? -// import static io.deephaven.plot.LineStyle.LineEndStyle -// import static io.deephaven.plot.LineStyle.LineJoinStyle -// import static io.deephaven.plot.Font.FontStyle -//////////////////// Colors //////////////////////// - -for( String c : io.deephaven.gui.color.Color.colorNames() ) { - publishVariable( "COLOR_" + c, io.deephaven.gui.color.Color.valueOf(c) ) -} - -colorTable = { - t = emptyTable(1) - .updateView("Colors = colorNames()") - .ungroup() - //todo simplify the following with the improved color branch - .updateView("Paint = io.deephaven.gui.color.Color.color(Colors).javaColor()") - .formatColumns("Colors = io.deephaven.engine.util.ColorUtil.bgfga(Paint.getRed(), Paint.getGreen(), Paint.getBlue())") - .dropColumns("Paint") -} - -//////////////////// Count Metrics ////////////////////////// - -resetMetricsCounts = { - io.deephaven.util.metrics.MetricsManager.resetCounters() -} - -getMetricsCounts = { - io.deephaven.util.metrics.MetricsManager.getCounters() -} - -printMetricsCounts = { - println(io.deephaven.util.metrics.MetricsManager.getCounters()) -} diff --git a/engine/table/python/core/deephaven_jpy_init.py b/engine/table/python/core/deephaven_jpy_init.py deleted file mode 100644 index 36158d942d0..00000000000 --- a/engine/table/python/core/deephaven_jpy_init.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/python - -import jpy -import os -import sys -from deephaven_internal.stream import TeeStream - -# Set stdin to /dev/null to prevent functions (like help()) that attempt to read from stdin from hanging python -# execution from within Java. -os.dup2(os.open("/dev/null", os.O_RDONLY), 0) - -jpy.VerboseExceptions.enabled = True -# If you want jpy to tell you about all that it is doing, change this -# jpy.diag.flags = jpy.diag.F_ALL - -j_sys = jpy.get_type('java.lang.System') -sys.stdout = TeeStream.redirect(sys.stdout, j_sys.out) -sys.stderr = TeeStream.redirect(sys.stderr, j_sys.err) diff --git a/engine/table/src/main/java/io/deephaven/engine/util/jpy/JpyInit.java b/engine/table/src/main/java/io/deephaven/engine/util/jpy/JpyInit.java index f02014805d3..c437c0f222b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/jpy/JpyInit.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/jpy/JpyInit.java @@ -61,7 +61,8 @@ private static void markReadyAndCheckEnv() { try ( final PyModule deephavenJpyModule = PyModule.importModule("deephaven_internal.jvm"); final PyObject readyObj = deephavenJpyModule.callMethod("ready"); - final PyObject checkObj = deephavenJpyModule.callMethod("check_py_env")) { + final PyObject checkObj = deephavenJpyModule.callMethod("check_py_env"); + final PyObject initObj = deephavenJpyModule.callMethod("init_py")) { // empty } } diff --git a/engine/table/groovy/groovy/0-base.groovy b/engine/table/src/main/resources/groovy/0-base.groovy similarity index 100% rename from engine/table/groovy/groovy/0-base.groovy rename to engine/table/src/main/resources/groovy/0-base.groovy diff --git a/engine/table/groovy/groovy/2-calendars.groovy b/engine/table/src/main/resources/groovy/2-calendars.groovy similarity index 100% rename from engine/table/groovy/groovy/2-calendars.groovy rename to engine/table/src/main/resources/groovy/2-calendars.groovy diff --git a/engine/table/groovy/groovy/4-count-metrics.groovy b/engine/table/src/main/resources/groovy/4-count-metrics.groovy similarity index 100% rename from engine/table/groovy/groovy/4-count-metrics.groovy rename to engine/table/src/main/resources/groovy/4-count-metrics.groovy diff --git a/props/configs/src/main/resources/dh-defaults.prop b/props/configs/src/main/resources/dh-defaults.prop index 502ad119c46..99ff56f26cb 100644 --- a/props/configs/src/main/resources/dh-defaults.prop +++ b/props/configs/src/main/resources/dh-defaults.prop @@ -35,9 +35,9 @@ QueryPerformanceRecorder.packageFilter.internal=defaultPackageFilters.qpr ########## Deephaven Groovy and Python Session defaults ########## -GroovyDeephavenSession.initScripts=core/deephaven_core_utils.groovy +GroovyDeephavenSession.initScripts= -PythonDeephavenSession.initScripts=core/deephaven_jpy_init.py +PythonDeephavenSession.initScripts= default.processEnvironmentFactory=io.deephaven.util.process.DefaultProcessEnvironment$Factory diff --git a/props/test-configs/src/main/resources/dh-tests.prop b/props/test-configs/src/main/resources/dh-tests.prop index 686b4e79801..64dfddae10a 100644 --- a/props/test-configs/src/main/resources/dh-tests.prop +++ b/props/test-configs/src/main/resources/dh-tests.prop @@ -57,9 +57,9 @@ QueryPerformanceRecorder.packageFilter.internal=defaultPackageFilters.qpr ########## Deephaven Groovy and Python Session defaults ########## -GroovyDeephavenSession.initScripts=core/deephaven_core_utils.groovy +GroovyDeephavenSession.initScripts= -PythonDeephavenSession.initScripts=core/deephaven_jpy_init.py +PythonDeephavenSession.initScripts= ###### Measurement Options ###### statsdriver.enabled=false diff --git a/py/server/deephaven_internal/jvm/__init__.py b/py/server/deephaven_internal/jvm/__init__.py index c5b4c0c7578..93073e380ad 100644 --- a/py/server/deephaven_internal/jvm/__init__.py +++ b/py/server/deephaven_internal/jvm/__init__.py @@ -66,3 +66,25 @@ def init_jvm(*args, **kwargs): "Unable to initialize JVM, try setting the environment variable JAVA_HOME (JDK 11+ required)") from e ready() return result + +def init_py(): + """Finishes starting Python to be usable from inside of a Java process. Not intended to be called in cases + where the process was started as Python, and Java was started from inside Python. + """ + + import jpy + import os + import sys + from deephaven_internal.stream import TeeStream + + # Set stdin to /dev/null to prevent functions (like help()) that attempt to read from stdin from hanging python + # execution from within Java. + os.dup2(os.open("/dev/null", os.O_RDONLY), 0) + + jpy.VerboseExceptions.enabled = True + # If you want jpy to tell you about all that it is doing, change this + # jpy.diag.flags = jpy.diag.F_ALL + + j_sys = jpy.get_type('java.lang.System') + sys.stdout = TeeStream.redirect(sys.stdout, j_sys.out) + sys.stderr = TeeStream.redirect(sys.stderr, j_sys.err) diff --git a/python-engine-test/src/test/java/io/deephaven/engine/util/WorkerPythonEnvironment.java b/python-engine-test/src/test/java/io/deephaven/engine/util/WorkerPythonEnvironment.java index bc005845e0b..cda1eb05d59 100644 --- a/python-engine-test/src/test/java/io/deephaven/engine/util/WorkerPythonEnvironment.java +++ b/python-engine-test/src/test/java/io/deephaven/engine/util/WorkerPythonEnvironment.java @@ -68,8 +68,8 @@ public enum WorkerPythonEnvironment { .getStringWithDefault("WorkerPythonEnvironment.defaultScriptPath", "."); final ScriptFinder scriptFinder = new ScriptFinder(defaultScriptPath); - final String initScript = Configuration.getInstance().getStringWithDefault("WorkerPythonEnvironment.initScript", - "core/deephaven_jpy_init.py"); + final String initScript = + Configuration.getInstance().getStringWithDefault("WorkerPythonEnvironment.initScript", ""); final ScriptFinder.FileOrStream file; try {