PythonScopeJpyImpl: Disable Conversion Cache

Integrations/src/main/java/io/deephaven/integrations/python/PythonDeephavenSession.java

@@ -41,9 +41,7 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.function.Consumer;
@@ -301,10 +299,23 @@ protected synchronized Object setVariable(String name, @Nullable Object newValue
  }
 
  @Override
- protected Map<String, Object> getAllValues() {
- return PyLib
- .ensureGil(() -> scope.getEntries()
- .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)));
+ protected Map<String, Object> getAllValues(@NotNull final Predicate<Map.Entry<String, Object>> predicate) {
+ final HashMap<String, Object> result = PyLib.ensureGil(
+ () -> scope.getEntriesRaw().<Map.Entry<String, PyObject>>map(
+ e -> new AbstractMap.SimpleImmutableEntry<>(scope.convertStringKey(e.getKey()), e.getValue()))
+ .collect(HashMap::new, (map, entry) -> map.put(entry.getKey(), entry.getValue()),
+ HashMap::putAll));
+
+ final Iterator<Map.Entry<String, Object>> iter = result.entrySet().iterator();
+ while (iter.hasNext()) {
+ final Map.Entry<String, Object> entry = iter.next();
+ entry.setValue(scope.convertValue((PyObject) entry.getValue()));
+ if (!predicate.test(entry)) {
+ iter.remove();
+ }
+ }
+
+ return result;
  }
 
  @Override
@@ -315,7 +326,7 @@ public String scriptType() {
  // TODO core#41 move this logic into the python console instance or scope like this - can go further and move
  // isWidget too
  @Override
- public Object unwrapObject(Object object) {
+ public Object unwrapObject(@Nullable Object object) {
  if (object instanceof PyObject) {
  final PyObject pyObject = (PyObject) object;
  final Object unwrapped = module.javaify(pyObject);

Integrations/src/main/java/io/deephaven/integrations/python/PythonObjectWrapper.java

@@ -13,6 +13,11 @@
 public class PythonObjectWrapper {
  private static final PyModule PY_WRAPPER_MODULE = PyModule.importModule("deephaven._wrapper");
 
+ /**
+ * Ensure that the class initializer runs.
+ */
+ public static void init() {}
+
  /**
  * Unwrap a Python object to return the wrapped Java object.
  * 

engine/context/src/main/java/io/deephaven/engine/context/EmptyQueryScope.java

@@ -10,6 +10,7 @@
 import java.util.Collections;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Predicate;
 import java.util.stream.Stream;
 
 public class EmptyQueryScope implements QueryScope {
@@ -48,7 +49,7 @@ public <T> void putParam(String name, T value) {
  }
 
  @Override
- public Map<String, Object> toMap() {
+ public Map<String, Object> toMap(@NotNull Predicate<Map.Entry<String, Object>> predicate) {
  return Collections.emptyMap();
  }
 

engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryScope.java

@@ -10,6 +10,7 @@
 import java.lang.ref.WeakReference;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Predicate;
 import java.util.stream.Stream;
 
 public class PoisonedQueryScope implements QueryScope {
@@ -53,7 +54,7 @@ public <T> void putParam(String name, T value) {
  }
 
  @Override
- public Map<String, Object> toMap() {
+ public Map<String, Object> toMap(@NotNull Predicate<Map.Entry<String, Object>> predicate) {
  return fail();
  }
 

engine/context/src/main/java/io/deephaven/engine/context/QueryScope.java

@@ -6,12 +6,15 @@
 import io.deephaven.engine.liveness.LivenessNode;
 import io.deephaven.base.log.LogOutput;
 import io.deephaven.base.log.LogOutputAppendable;
+import io.deephaven.util.annotations.FinalDefault;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.Predicate;
 
 /**
  * Variable scope used to resolve parameter values during query execution and to expose named objects to users. Objects
@@ -69,8 +72,8 @@ private MissingVariableException(final Throwable cause) {
  * @return A newly-constructed array of newly-constructed Params.
  * @throws QueryScope.MissingVariableException If any of the named scope variables does not exist.
  */
- default QueryScopeParam[] getParams(final Collection<String> names) throws MissingVariableException {
- final QueryScopeParam[] result = new QueryScopeParam[names.size()];
+ default QueryScopeParam<?>[] getParams(final Collection<String> names) throws MissingVariableException {
+ final QueryScopeParam<?>[] result = new QueryScopeParam[names.size()];
  int pi = 0;
  for (final String name : names) {
  result[pi++] = createParam(name);
@@ -131,12 +134,24 @@ default QueryScopeParam[] getParams(final Collection<String> names) throws Missi
  <T> void putParam(final String name, final T value);
 
  /**
- * Returns an immutable map with all objects in the scope. Callers may want to unwrap language-specific values using
- * {@link #unwrapObject(Object)} before using them.
+ * Returns a mutable map with all objects in the scope. Callers may want to unwrap language-specific values using
+ * {@link #unwrapObject(Object)} before using them. This map is owned by the caller and may be mutated.
+ *
+ * @param predicate a predicate to filter the map entries
+ * @return a caller-owned mutable map with all known variables and their values.
+ */
+ Map<String, Object> toMap(@NotNull Predicate<Map.Entry<String, Object>> predicate);
+
+ /**
+ * Returns a mutable map with all objects in the scope. Callers may want to unwrap language-specific values using
+ * {@link #unwrapObject(Object)} before using them. This map is owned by the caller and may be mutated.
  *
- * @return an immutable map with all known variables and their values.
+ * @return a caller-owned mutable map with all known variables and their values.
  */
- Map<String, Object> toMap();
+ @FinalDefault
+ default Map<String, Object> toMap() {
+ return toMap(entry -> true);
+ }
 
  /**
  * Removes any wrapping that exists on a scope param object so that clients can fetch them. Defaults to returning
@@ -145,16 +160,16 @@ default QueryScopeParam[] getParams(final Collection<String> names) throws Missi
  * @param object the scoped object
  * @return an obj which can be consumed by a client
  */
- default Object unwrapObject(Object object) {
+ default Object unwrapObject(@Nullable Object object) {
  return object;
  }
 
  @Override
  default LogOutput append(@NotNull final LogOutput logOutput) {
  logOutput.append('{');
- for (final String paramName : getParamNames()) {
- final Object paramValue = readParamValue(paramName);
- logOutput.nl().append(paramName).append("=");
+ for (final Map.Entry<String, Object> param : toMap().entrySet()) {
+ logOutput.nl().append(param.getKey()).append("=");
+ Object paramValue = param.getValue();
  if (paramValue == this) {
  logOutput.append("this QueryScope (" + paramValue.getClass().getName() + ':'
  + System.identityHashCode(paramValue) + ')');

engine/context/src/main/java/io/deephaven/engine/context/QueryScopeParam.java

@@ -5,8 +5,6 @@
 
 public class QueryScopeParam<T> {
 
- public static final QueryScopeParam<?>[] ZERO_LENGTH_PARAM_ARRAY = new QueryScopeParam[0];
-
  private final String name;
  private final T value;
 

engine/context/src/main/java/io/deephaven/engine/context/StandaloneQueryScope.java

@@ -6,10 +6,13 @@
 import io.deephaven.engine.updategraph.DynamicNode;
 import io.deephaven.hash.KeyedObjectHashMap;
 import io.deephaven.hash.KeyedObjectKey;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.jetbrains.annotations.NotNull;
 
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
+import java.util.function.Predicate;
 
 /**
  * Map-based implementation, extending LivenessArtifact to manage the objects passed into it.
@@ -77,9 +80,13 @@ public <T> void putParam(final String name, final T value) {
  }
 
  @Override
- public Map<String, Object> toMap() {
- return valueRetrievers.entrySet().stream()
- .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> e.getValue().value));
+ public Map<String, Object> toMap(@NotNull final Predicate<Map.Entry<String, Object>> predicate) {
+ final HashMap<String, Object> result = new HashMap<>();
+ valueRetrievers.entrySet().stream()
+ .map(e -> ImmutablePair.of(e.getKey(), (Object) e.getValue().value))
+ .filter(predicate)
+ .forEach(e -> result.put(e.getKey(), e.getValue()));
+ return result;
  }
 
  private static class ValueRetriever<T> {