Merge branch 'deephaven:main' into update_by

deephaven · Sep 20, 2023 · 2eead7f · 2eead7f
2 parents 0e60509 + bcb240f
commit 2eead7f
Show file tree

Hide file tree

Showing 12 changed files with 47 additions and 131 deletions.
diff --git a/engine/table/build.gradle b/engine/table/build.gradle
@@ -6,23 +6,6 @@ plugins {
 
 description 'Engine Table: Implementation and closely-coupled utilities'
 
-sourceSets {
-    main {
-        resources {
-            srcDir 'groovy'
-            include 'core/*.groovy'
-        }
-        resources {
-            srcDir 'python'
-            include 'core/*.py'
-        }
-        resources {
-            srcDir 'src/main/resources'
-            include '**'
-        }
-    }
-}
-
 configurations {
     // Ensure jmh picks up the same dependencies as tests
     jmhImplementation.extendsFrom testImplementation

diff --git a/engine/table/groovy/core/deephaven_core_utils.groovy b/engine/table/groovy/core/deephaven_core_utils.groovy
diff --git a/engine/table/python/core/deephaven_jpy_init.py b/engine/table/python/core/deephaven_jpy_init.py
diff --git a/engine/table/src/main/java/io/deephaven/engine/util/jpy/JpyInit.java b/engine/table/src/main/java/io/deephaven/engine/util/jpy/JpyInit.java
@@ -61,7 +61,8 @@ private static void markReadyAndCheckEnv() {
         try (
                 final PyModule deephavenJpyModule = PyModule.importModule("deephaven_internal.jvm");
                 final PyObject readyObj = deephavenJpyModule.callMethod("ready");
-                final PyObject checkObj = deephavenJpyModule.callMethod("check_py_env")) {
+                final PyObject checkObj = deephavenJpyModule.callMethod("check_py_env");
+                final PyObject initObj = deephavenJpyModule.callMethod("init_py")) {
             // empty
         }
     }

diff --git a/engine/table/groovy/groovy/0-base.groovy → ...e/src/main/resources/groovy/0-base.groovy b/engine/table/groovy/groovy/0-base.groovy → ...e/src/main/resources/groovy/0-base.groovy
diff --git a/...ne/table/groovy/groovy/2-calendars.groovy → .../main/resources/groovy/2-calendars.groovy b/...ne/table/groovy/groovy/2-calendars.groovy → .../main/resources/groovy/2-calendars.groovy
diff --git a/...able/groovy/groovy/4-count-metrics.groovy → ...n/resources/groovy/4-count-metrics.groovy b/...able/groovy/groovy/4-count-metrics.groovy → ...n/resources/groovy/4-count-metrics.groovy
diff --git a/props/configs/src/main/resources/dh-defaults.prop b/props/configs/src/main/resources/dh-defaults.prop
@@ -35,9 +35,9 @@ QueryPerformanceRecorder.packageFilter.internal=defaultPackageFilters.qpr
 
 ########## Deephaven Groovy and Python Session defaults ##########
 
-GroovyDeephavenSession.initScripts=core/deephaven_core_utils.groovy
+GroovyDeephavenSession.initScripts=
 
-PythonDeephavenSession.initScripts=core/deephaven_jpy_init.py
+PythonDeephavenSession.initScripts=
 
 default.processEnvironmentFactory=io.deephaven.util.process.DefaultProcessEnvironment$Factory
 

diff --git a/props/test-configs/src/main/resources/dh-tests.prop b/props/test-configs/src/main/resources/dh-tests.prop
@@ -57,9 +57,9 @@ QueryPerformanceRecorder.packageFilter.internal=defaultPackageFilters.qpr
 
 ########## Deephaven Groovy and Python Session defaults ##########
 
-GroovyDeephavenSession.initScripts=core/deephaven_core_utils.groovy
+GroovyDeephavenSession.initScripts=
 
-PythonDeephavenSession.initScripts=core/deephaven_jpy_init.py
+PythonDeephavenSession.initScripts=
 
 ###### Measurement Options ######
 statsdriver.enabled=false

diff --git a/py/server/deephaven_internal/jvm/__init__.py b/py/server/deephaven_internal/jvm/__init__.py
@@ -66,3 +66,25 @@ def init_jvm(*args, **kwargs):
             "Unable to initialize JVM, try setting the environment variable JAVA_HOME (JDK 11+ required)") from e
     ready()
     return result
+
+def init_py():
+    """Finishes starting Python to be usable from inside of a Java process. Not intended to be called in cases
+    where the process was started as Python, and Java was started from inside Python.
+    """
+
+    import jpy
+    import os
+    import sys
+    from deephaven_internal.stream import TeeStream
+
+    # Set stdin to /dev/null to prevent functions (like help()) that attempt to read from stdin from hanging python
+    # execution from within Java.
+    os.dup2(os.open("/dev/null", os.O_RDONLY), 0)
+
+    jpy.VerboseExceptions.enabled = True
+    # If you want jpy to tell you about all that it is doing, change this
+    # jpy.diag.flags = jpy.diag.F_ALL
+
+    j_sys = jpy.get_type('java.lang.System')
+    sys.stdout = TeeStream.redirect(sys.stdout, j_sys.out)
+    sys.stderr = TeeStream.redirect(sys.stderr, j_sys.err)
diff --git a/python-engine-test/src/test/java/io/deephaven/engine/util/WorkerPythonEnvironment.java b/python-engine-test/src/test/java/io/deephaven/engine/util/WorkerPythonEnvironment.java
@@ -68,8 +68,8 @@ public enum WorkerPythonEnvironment {
                 .getStringWithDefault("WorkerPythonEnvironment.defaultScriptPath", ".");
 
         final ScriptFinder scriptFinder = new ScriptFinder(defaultScriptPath);
-        final String initScript = Configuration.getInstance().getStringWithDefault("WorkerPythonEnvironment.initScript",
-                "core/deephaven_jpy_init.py");
+        final String initScript =
+                Configuration.getInstance().getStringWithDefault("WorkerPythonEnvironment.initScript", "");
 
         final ScriptFinder.FileOrStream file;
         try {

diff --git a/server/src/main/java/io/deephaven/server/object/ObjectServiceGrpcImpl.java b/server/src/main/java/io/deephaven/server/object/ObjectServiceGrpcImpl.java
@@ -6,6 +6,8 @@
 import com.google.protobuf.ByteString;
 import com.google.rpc.Code;
 import io.deephaven.base.verify.Assert;
+import io.deephaven.engine.liveness.LivenessScope;
+import io.deephaven.engine.liveness.LivenessScopeStack;
 import io.deephaven.extensions.barrage.util.GrpcUtil;
 import io.deephaven.plugin.type.ObjectCommunicationException;
 import io.deephaven.plugin.type.ObjectType;
@@ -17,6 +19,7 @@
 import io.deephaven.server.session.SessionState;
 import io.deephaven.server.session.SessionState.ExportObject;
 import io.deephaven.server.session.TicketRouter;
+import io.deephaven.util.SafeCloseable;
 import io.deephaven.util.function.ThrowingRunnable;
 import io.grpc.stub.StreamObserver;
 import org.jetbrains.annotations.NotNull;
@@ -147,11 +150,21 @@ public void onNext(final StreamRequest request) {
                             "Data message sent before Connect message");
                 }
                 Data data = request.getData();
-                List<SessionState.ExportObject<Object>> referenceObjects = data.getExportedReferencesList().stream()
-                        .map(typedTicket -> ticketRouter.resolve(session, typedTicket.getTicket(), "ticket"))
-                        .collect(Collectors.toList());
+                LivenessScope exportScope = new LivenessScope();
+
+                List<SessionState.ExportObject<Object>> referenceObjects;
+                try (SafeCloseable ignored = LivenessScopeStack.open(exportScope, false)) {
+                    referenceObjects = data.getExportedReferencesList().stream()
+                            .map(typedTicket -> ticketRouter.resolve(session, typedTicket.getTicket(), "ticket"))
+                            .collect(Collectors.toList());
+                }
                 runOrEnqueue(referenceObjects, () -> {
-                    Object[] objs = referenceObjects.stream().map(ExportObject::get).toArray();
+                    Object[] objs;
+                    try {
+                        objs = referenceObjects.stream().map(ExportObject::get).toArray();
+                    } finally {
+                        exportScope.release();
+                    }
                     messageStream.onData(data.getPayload().asReadOnlyByteBuffer(), objs);
                 });
             }