Merge remote-tracking branch 'upstream/main' into cfs-update-cpp-agai…

…n-again

.github/CODEOWNERS

@@ -24,3 +24,4 @@
 /py @chipkent @jmao-denver @rcaudy
 /R @chipkent @alexpeters1208 @rcaudy
 *.proto @devinrsmith @nbauernfeind @niloc132 @rcaudy
+*.gwt.xml @niloc132 @rcaudy @nbauernfeind

.github/workflows/cla.yml

@@ -21,7 +21,7 @@ jobs:
  steps:
  - name: "CLA Assistant"
  if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
- uses: cla-assistant/github-action@v2.4.0
+ uses: cla-assistant/github-action@v2.5.1
  env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_PERSONAL_ACCESS_TOKEN }}

.github/workflows/create-docs-issues.yml

@@ -26,7 +26,7 @@ jobs:
  await script({github, context});
 
  - name: Slack Failure Message
- uses: slackapi/slack-github-action@v1.26.0
+ uses: slackapi/slack-github-action@v1.27.0
  id: slack-failure-message
  if: failure() && github.ref == 'refs/heads/main' && github.repository_owner == 'deephaven'
  env:

.github/workflows/nightly-check-ci.yml

@@ -100,7 +100,7 @@ jobs:
  report_paths: '**/build/test-results/*/TEST-*.xml'
 
  - name: Slack Nightly Failure
- uses: slackapi/slack-github-action@v1.26.0
+ uses: slackapi/slack-github-action@v1.27.0
  id: slack-nightly-failure
  if: ${{ failure() && github.repository_owner == 'deephaven' }}
  with:

.github/workflows/nightly-image-check.yml

@@ -35,7 +35,7 @@ jobs:
  run: ./gradlew --continue pullImage compareImage
 
  - name: Notify Slack
- uses: slackapi/slack-github-action@v1.26.0
+ uses: slackapi/slack-github-action@v1.27.0
  id: notify-slack
  if: ${{ failure() }}
  env:

.github/workflows/nightly-publish-ci.yml

@@ -55,7 +55,7 @@ jobs:
  ORG_GRADLE_PROJECT_signingRequired: true
 
  - name: Slack Nightly Failure
- uses: slackapi/slack-github-action@v1.26.0
+ uses: slackapi/slack-github-action@v1.27.0
  id: slack-nightly-failure
  if: failure()
  with:

.github/workflows/update-web.yml

@@ -33,7 +33,7 @@ jobs:
  sed -i "s/^ARG CHART_VERSION=.*/ARG CHART_VERSION=$CHART_VERSION/" ./web/client-ui/Dockerfile
  sed -i "s/^ARG WIDGET_VERSION=.*/ARG WIDGET_VERSION=$WIDGET_VERSION/" ./web/client-ui/Dockerfile
  - name: Create Pull Request
- uses: peter-evans/create-pull-request@v6
+ uses: peter-evans/create-pull-request@v7
  env:
  WEB_VERSION: ${{steps.web_versions.outputs.WEB_VERSION}}
  with:

Base/src/main/resources/io/deephaven/base/Base.gwt.xml

@@ -0,0 +1,4 @@
+<module>
+ <source path="" includes="MathUtil.java" />
+ <source path="verify" />
+</module>

IO/src/main/resources/io/deephaven/io/IO.gwt.xml

@@ -0,0 +1,3 @@
+<module>
+ <source path="streams" includes="ByteBufferInputStream.java" />
+</module>

Integrations/build.gradle

@@ -15,6 +15,7 @@ dependencies {
  implementation project(':plugin')
  implementation project(':Configuration')
  implementation project(':log-factory')
+ implementation libs.commons.lang3
 
  testImplementation project(':engine-test-utils')
  testImplementation project(path: ':Base', configuration: 'tests')

Integrations/src/main/java/io/deephaven/integrations/python/PythonDeephavenSession.java

@@ -57,8 +57,25 @@ public class PythonDeephavenSession extends AbstractScriptSession<PythonSnapshot
  private static final String DEFAULT_SCRIPT_PATH = Configuration.getInstance()
  .getStringWithDefault("PythonDeephavenSession.defaultScriptPath", ".");
 
+ /**
+ * This is following the convention set by io.deephaven.server.config.ConfigServiceGrpcImpl / dh-defaults.prop of
+ * relaying version information to the client via Configuration properties.
+ */
+ private static final String PYTHON_VERSION_PROPERTY = "python.version";
+
  public static String SCRIPT_TYPE = "Python";
 
+ private static void setPythonVersion(Configuration configuration) {
+ final String pythonVersion;
+ try (final PyModule platformModule = PyModule.importModule("platform")) {
+ pythonVersion = platformModule.call(String.class, "python_version", new Class[0], new Object[0]);
+ } catch (RuntimeException e) {
+ log.warn(e).append("Unable to retrieve python version").endl();
+ return;
+ }
+ configuration.setProperty(PYTHON_VERSION_PROPERTY, pythonVersion);
+ }
+
  private final PythonEvaluator evaluator;
  private final PythonScope<PyObject> scope;
  private final PythonScriptSessionModule module;
@@ -67,6 +84,10 @@ public class PythonDeephavenSession extends AbstractScriptSession<PythonSnapshot
  /**
  * Create a Python ScriptSession.
  *
+ * <p>
+ * Sets the configuration property {@value PYTHON_VERSION_PROPERTY} to the value returned from the python code
+ * {@code platform.python_version()}.
+ *
  * @param updateGraph the default update graph to install for the repl
  * @param operationInitializer the default operation initializer to install for the repl
  * @param objectTypeLookup the object type lookup
@@ -96,16 +117,19 @@ public PythonDeephavenSession(
 
  registerJavaExecutor(threadInitializationFactory);
  publishInitial();
+
+ final Configuration configuration = Configuration.getInstance();
  /*
  * And now the user-defined initialization scripts, if any.
  */
  if (runInitScripts) {
- String[] scripts = Configuration.getInstance().getProperty("PythonDeephavenSession.initScripts").split(",");
+ String[] scripts = configuration.getProperty("PythonDeephavenSession.initScripts").split(",");
 
  for (String script : scripts) {
  runScript(script);
  }
  }
+ setPythonVersion(configuration);
  }
 
  /**

Integrations/src/main/java/io/deephaven/integrations/python/PythonMergedListenerAdapter.java

@@ -7,14 +7,18 @@
 import io.deephaven.engine.rowset.RowSetFactory;
 import io.deephaven.engine.rowset.RowSetShiftData;
 import io.deephaven.engine.table.ModifiedColumnSet;
+import io.deephaven.engine.table.TableListener;
 import io.deephaven.engine.table.TableUpdate;
 import io.deephaven.engine.table.impl.ListenerRecorder;
 import io.deephaven.engine.table.impl.MergedListener;
 import io.deephaven.engine.table.impl.TableUpdateImpl;
 import io.deephaven.engine.updategraph.NotificationQueue;
 import io.deephaven.engine.updategraph.UpdateGraph;
+import io.deephaven.internal.log.LoggerFactory;
+import io.deephaven.io.logger.Logger;
 import io.deephaven.util.SafeCloseable;
 import io.deephaven.util.annotations.ScriptApi;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jpy.PyObject;
@@ -33,7 +37,10 @@
  */
 @ScriptApi
 public class PythonMergedListenerAdapter extends MergedListener {
- private final PyObject pyCallable;
+ private static final Logger log = LoggerFactory.getLogger(PythonMergedListenerAdapter.class);
+
+ private final PyObject pyListenerCallable;
+ private final PyObject pyOnFailureCallback;
 
  /**
  * Create a Python merged listener.
@@ -42,23 +49,26 @@ public class PythonMergedListenerAdapter extends MergedListener {
  * @param dependencies The tables that must be satisfied before this listener is executed.
  * @param listenerDescription A description for the UpdatePerformanceTracker to append to its entry description, may
  * be null.
- * @param pyObjectIn Python listener object.
+ * @param pyListener Python listener object.
  */
  private PythonMergedListenerAdapter(
  @NotNull ListenerRecorder[] recorders,
  @Nullable NotificationQueue.Dependency[] dependencies,
  @Nullable String listenerDescription,
- @NotNull PyObject pyObjectIn) {
+ @NotNull PyObject pyListener,
+ @NotNull PyObject pyOnFailureCallback) {
  super(Arrays.asList(recorders), Arrays.asList(dependencies), listenerDescription, null);
  Arrays.stream(recorders).forEach(rec -> rec.setMergedListener(this));
- this.pyCallable = PythonUtils.pyMergeListenerFunc(pyObjectIn);
+ this.pyListenerCallable = PythonUtils.pyMergeListenerFunc(Objects.requireNonNull(pyListener));
+ this.pyOnFailureCallback = Objects.requireNonNull(pyOnFailureCallback);
  }
 
  public static PythonMergedListenerAdapter create(
  @NotNull ListenerRecorder[] recorders,
  @Nullable NotificationQueue.Dependency[] dependencies,
  @Nullable String listenerDescription,
- @NotNull PyObject pyObjectIn) {
+ @NotNull PyObject pyListener,
+ @NotNull PyObject pyOnFailureCallback) {
  if (recorders.length < 2) {
  throw new IllegalArgumentException("At least two listener recorders must be provided");
  }
@@ -71,7 +81,8 @@ public static PythonMergedListenerAdapter create(
  final UpdateGraph updateGraph = allItems[0].getUpdateGraph(allItems);
 
  try (final SafeCloseable ignored = ExecutionContext.getContext().withUpdateGraph(updateGraph).open()) {
- return new PythonMergedListenerAdapter(recorders, dependencies, listenerDescription, pyObjectIn);
+ return new PythonMergedListenerAdapter(recorders, dependencies, listenerDescription, pyListener,
+ pyOnFailureCallback);
  }
  }
 
@@ -91,6 +102,22 @@ public ArrayList<TableUpdate> currentRowsAsUpdates() {
 
  @Override
  protected void process() {
- pyCallable.call("__call__");
+ pyListenerCallable.call("__call__");
+ }
+
+ @Override
+ protected void propagateErrorDownstream(boolean fromProcess, @NotNull Throwable error,
+ TableListener.@Nullable Entry entry) {
+ if (!pyOnFailureCallback.isNone()) {
+ try {
+ pyOnFailureCallback.call("__call__", ExceptionUtils.getStackTrace(error));
+ } catch (Exception e2) {
+ // If the Python onFailure callback fails, log the new exception
+ // and continue with the original exception.
+ log.error().append("Python on_error callback failed: ").append(e2).endl();
+ }
+ } else {
+ log.error().append("Python on_error callback is None: ").append(ExceptionUtils.getStackTrace(error)).endl();
+ }
  }
 }

Integrations/src/main/java/io/deephaven/integrations/python/PythonReplayListenerAdapter.java

@@ -3,6 +3,7 @@
 //
 package io.deephaven.integrations.python;
 
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import io.deephaven.engine.context.ExecutionContext;
 import io.deephaven.engine.table.Table;
 import io.deephaven.engine.table.TableUpdate;
@@ -14,12 +15,16 @@
 import io.deephaven.engine.rowset.RowSetShiftData;
 import io.deephaven.engine.updategraph.NotificationQueue;
 import io.deephaven.engine.updategraph.UpdateGraph;
+import io.deephaven.internal.log.LoggerFactory;
+import io.deephaven.io.logger.Logger;
 import io.deephaven.util.SafeCloseable;
 import io.deephaven.util.annotations.ScriptApi;
+import org.jetbrains.annotations.NotNull;
 import org.jpy.PyObject;
 
 import javax.annotation.Nullable;
 import java.util.Arrays;
+import java.util.Objects;
 
 
 /**
@@ -33,7 +38,10 @@
 public class PythonReplayListenerAdapter extends InstrumentedTableUpdateListenerAdapter
  implements TableSnapshotReplayer {
  private static final long serialVersionUID = -8882402061960621245L;
- private final PyObject pyCallable;
+ private static final Logger log = LoggerFactory.getLogger(PythonReplayListenerAdapter.class);
+
+ private final PyObject pyListenerCallable;
+ private final PyObject pyOnFailureCallback;
  private final NotificationQueue.Dependency[] dependencies;
 
  /**
@@ -43,22 +51,34 @@ public class PythonReplayListenerAdapter extends InstrumentedTableUpdateListener
  * null.
  * @param source The source table to which this listener will subscribe.
  * @param retain Whether a hard reference to this listener should be maintained to prevent it from being collected.
- * @param pyObjectIn Python listener object.
+ * @param pyListener Python listener object.
  * @param dependencies The tables that must be satisfied before this listener is executed.
  */
- public static PythonReplayListenerAdapter create(@Nullable String description, Table source, boolean retain,
- PyObject pyObjectIn, NotificationQueue.Dependency... dependencies) {
+ public static PythonReplayListenerAdapter create(
+ @Nullable String description,
+ @NotNull Table source,
+ boolean retain,
+ @NotNull PyObject pyListener,
+ @NotNull PyObject pyOnFailureCallback,
+ @Nullable NotificationQueue.Dependency... dependencies) {
  final UpdateGraph updateGraph = source.getUpdateGraph(dependencies);
  try (final SafeCloseable ignored = ExecutionContext.getContext().withUpdateGraph(updateGraph).open()) {
- return new PythonReplayListenerAdapter(description, source, retain, pyObjectIn, dependencies);
+ return new PythonReplayListenerAdapter(description, source, retain, pyListener, pyOnFailureCallback,
+ dependencies);
  }
  }
 
- private PythonReplayListenerAdapter(@Nullable String description, Table source, boolean retain, PyObject pyObjectIn,
- NotificationQueue.Dependency... dependencies) {
+ private PythonReplayListenerAdapter(
+ @Nullable String description,
+ @NotNull Table source,
+ boolean retain,
+ @NotNull PyObject pyListener,
+ @NotNull PyObject pyOnFailureCallback,
+ @Nullable NotificationQueue.Dependency... dependencies) {
  super(description, source, retain);
  this.dependencies = dependencies;
- this.pyCallable = PythonUtils.pyListenerFunc(pyObjectIn);
+ this.pyListenerCallable = PythonUtils.pyListenerFunc(Objects.requireNonNull(pyListener));
+ this.pyOnFailureCallback = Objects.requireNonNull(pyOnFailureCallback);
  }
 
  @Override
@@ -69,18 +89,39 @@ public void replay() {
  final TableUpdate update =
  new TableUpdateImpl(source.getRowSet(), emptyRowSet, emptyRowSet, emptyShift, emptyColumnSet);
  final boolean isReplay = true;
- pyCallable.call("__call__", update, isReplay);
+ pyListenerCallable.call("__call__", update, isReplay);
  }
 
  @Override
  public void onUpdate(final TableUpdate update) {
  final boolean isReplay = false;
- pyCallable.call("__call__", update, isReplay);
+ pyListenerCallable.call("__call__", update, isReplay);
+ }
+
+ @Override
+ public void onFailureInternal(Throwable originalException, Entry sourceEntry) {
+ if (!pyOnFailureCallback.isNone()) {
+ try {
+ pyOnFailureCallback.call("__call__", ExceptionUtils.getStackTrace(originalException));
+ } catch (Throwable e) {
+ // If the Python onFailure callback fails, log the new exception
+ // and continue with the original exception.
+ log.error().append("Python on_error callback failed: ").append(e).endl();
+ }
+ } else {
+ log.error().append("Python on_error callback is None: ")
+ .append(ExceptionUtils.getStackTrace(originalException)).endl();
+ }
+ super.onFailureInternal(originalException, sourceEntry);
  }
 
  @Override
  public boolean canExecute(final long step) {
  return super.canExecute(step)
  && (dependencies.length == 0 || Arrays.stream(dependencies).allMatch(t -> t.satisfied(step)));
  }
+
+ public boolean isFailed() {
+ return failed;
+ }
 }

R/build.gradle

@@ -138,7 +138,7 @@ def rClientDoc = Docker.registerDockerTask(project, 'rClientDoc') {
  ''')
  runCommand('''echo "status = tryCatch(" \
  " {" \
- " install.packages('roxygen2', repos='https://cran.r-project.org'); " \
+ " install.packages('roxygen2'); " \
  " 0" \
  " }," \
  " error=function(e) 1," \
@@ -179,7 +179,7 @@ def rClientSite = Docker.registerDockerTask(project, 'rClientSite') {
  runCommand("mkdir -p ${prefix}/src/rdeephaven/docs")
  runCommand('''echo "status = tryCatch(" \
  " {" \
- " install.packages('pkgdown', repos='https://cran.r-project.org'); " \
+ " install.packages('pkgdown'); " \
  " 0" \
  " }," \
  " error=function(e) 1," \

R/rdeephaven/DESCRIPTION

@@ -1,7 +1,7 @@
 Package: rdeephaven
 Type: Package
 Title: R Client for Deephaven Core
-Version: 0.36.0
+Version: 0.37.0
 Date: 2023-05-12
 Author: Deephaven Data Labs
 Maintainer: Alex Peters <alexpeters@deephaven.io>