diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ed440a0c5cc..f9b7f12a289 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -24,3 +24,4 @@ /py @chipkent @jmao-denver @rcaudy /R @chipkent @alexpeters1208 @rcaudy *.proto @devinrsmith @nbauernfeind @niloc132 @rcaudy +*.gwt.xml @niloc132 @rcaudy @nbauernfeind diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index d337a5782cf..02acd5109e9 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -21,7 +21,7 @@ jobs: steps: - name: "CLA Assistant" if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' - uses: cla-assistant/github-action@v2.4.0 + uses: cla-assistant/github-action@v2.5.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_PERSONAL_ACCESS_TOKEN }} diff --git a/.github/workflows/create-docs-issues.yml b/.github/workflows/create-docs-issues.yml index 77f52569deb..6e6fc938ea5 100644 --- a/.github/workflows/create-docs-issues.yml +++ b/.github/workflows/create-docs-issues.yml @@ -26,7 +26,7 @@ jobs: await script({github, context}); - name: Slack Failure Message - uses: slackapi/slack-github-action@v1.26.0 + uses: slackapi/slack-github-action@v1.27.0 id: slack-failure-message if: failure() && github.ref == 'refs/heads/main' && github.repository_owner == 'deephaven' env: diff --git a/.github/workflows/nightly-check-ci.yml b/.github/workflows/nightly-check-ci.yml index d8b5c93fec7..2e5c37cec73 100644 --- a/.github/workflows/nightly-check-ci.yml +++ b/.github/workflows/nightly-check-ci.yml @@ -100,7 +100,7 @@ jobs: report_paths: '**/build/test-results/*/TEST-*.xml' - name: Slack Nightly Failure - uses: slackapi/slack-github-action@v1.26.0 + uses: slackapi/slack-github-action@v1.27.0 id: slack-nightly-failure if: ${{ failure() && github.repository_owner == 'deephaven' }} with: diff --git a/.github/workflows/nightly-image-check.yml b/.github/workflows/nightly-image-check.yml index 1e868280782..e2f75b3a15f 100644 --- a/.github/workflows/nightly-image-check.yml +++ b/.github/workflows/nightly-image-check.yml @@ -35,7 +35,7 @@ jobs: run: ./gradlew --continue pullImage compareImage - name: Notify Slack - uses: slackapi/slack-github-action@v1.26.0 + uses: slackapi/slack-github-action@v1.27.0 id: notify-slack if: ${{ failure() }} env: diff --git a/.github/workflows/nightly-publish-ci.yml b/.github/workflows/nightly-publish-ci.yml index a23f9e04a2d..927c0b2009a 100644 --- a/.github/workflows/nightly-publish-ci.yml +++ b/.github/workflows/nightly-publish-ci.yml @@ -55,7 +55,7 @@ jobs: ORG_GRADLE_PROJECT_signingRequired: true - name: Slack Nightly Failure - uses: slackapi/slack-github-action@v1.26.0 + uses: slackapi/slack-github-action@v1.27.0 id: slack-nightly-failure if: failure() with: diff --git a/.github/workflows/update-web.yml b/.github/workflows/update-web.yml index be6a7357ab3..a467a16d65e 100644 --- a/.github/workflows/update-web.yml +++ b/.github/workflows/update-web.yml @@ -33,7 +33,7 @@ jobs: sed -i "s/^ARG CHART_VERSION=.*/ARG CHART_VERSION=$CHART_VERSION/" ./web/client-ui/Dockerfile sed -i "s/^ARG WIDGET_VERSION=.*/ARG WIDGET_VERSION=$WIDGET_VERSION/" ./web/client-ui/Dockerfile - name: Create Pull Request - uses: peter-evans/create-pull-request@v6 + uses: peter-evans/create-pull-request@v7 env: WEB_VERSION: ${{steps.web_versions.outputs.WEB_VERSION}} with: diff --git a/Base/src/main/resources/io/deephaven/base/Base.gwt.xml b/Base/src/main/resources/io/deephaven/base/Base.gwt.xml new file mode 100644 index 00000000000..6cf87067eae --- /dev/null +++ b/Base/src/main/resources/io/deephaven/base/Base.gwt.xml @@ -0,0 +1,4 @@ + + + + diff --git a/IO/src/main/resources/io/deephaven/io/IO.gwt.xml b/IO/src/main/resources/io/deephaven/io/IO.gwt.xml new file mode 100644 index 00000000000..51000b6cea3 --- /dev/null +++ b/IO/src/main/resources/io/deephaven/io/IO.gwt.xml @@ -0,0 +1,3 @@ + + + diff --git a/Integrations/build.gradle b/Integrations/build.gradle index 99abec47535..ac16784d585 100644 --- a/Integrations/build.gradle +++ b/Integrations/build.gradle @@ -15,6 +15,7 @@ dependencies { implementation project(':plugin') implementation project(':Configuration') implementation project(':log-factory') + implementation libs.commons.lang3 testImplementation project(':engine-test-utils') testImplementation project(path: ':Base', configuration: 'tests') diff --git a/Integrations/src/main/java/io/deephaven/integrations/python/PythonDeephavenSession.java b/Integrations/src/main/java/io/deephaven/integrations/python/PythonDeephavenSession.java index f14f3459504..f8b69e9d0ae 100644 --- a/Integrations/src/main/java/io/deephaven/integrations/python/PythonDeephavenSession.java +++ b/Integrations/src/main/java/io/deephaven/integrations/python/PythonDeephavenSession.java @@ -57,8 +57,25 @@ public class PythonDeephavenSession extends AbstractScriptSession scope; private final PythonScriptSessionModule module; @@ -67,6 +84,10 @@ public class PythonDeephavenSession extends AbstractScriptSession + * Sets the configuration property {@value PYTHON_VERSION_PROPERTY} to the value returned from the python code + * {@code platform.python_version()}. + * * @param updateGraph the default update graph to install for the repl * @param operationInitializer the default operation initializer to install for the repl * @param objectTypeLookup the object type lookup @@ -96,16 +117,19 @@ public PythonDeephavenSession( registerJavaExecutor(threadInitializationFactory); publishInitial(); + + final Configuration configuration = Configuration.getInstance(); /* * And now the user-defined initialization scripts, if any. */ if (runInitScripts) { - String[] scripts = Configuration.getInstance().getProperty("PythonDeephavenSession.initScripts").split(","); + String[] scripts = configuration.getProperty("PythonDeephavenSession.initScripts").split(","); for (String script : scripts) { runScript(script); } } + setPythonVersion(configuration); } /** diff --git a/Integrations/src/main/java/io/deephaven/integrations/python/PythonMergedListenerAdapter.java b/Integrations/src/main/java/io/deephaven/integrations/python/PythonMergedListenerAdapter.java index 215aec0b81a..0a43aae2ac7 100644 --- a/Integrations/src/main/java/io/deephaven/integrations/python/PythonMergedListenerAdapter.java +++ b/Integrations/src/main/java/io/deephaven/integrations/python/PythonMergedListenerAdapter.java @@ -7,14 +7,18 @@ import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.rowset.RowSetShiftData; import io.deephaven.engine.table.ModifiedColumnSet; +import io.deephaven.engine.table.TableListener; import io.deephaven.engine.table.TableUpdate; import io.deephaven.engine.table.impl.ListenerRecorder; import io.deephaven.engine.table.impl.MergedListener; import io.deephaven.engine.table.impl.TableUpdateImpl; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.updategraph.UpdateGraph; +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.logger.Logger; import io.deephaven.util.SafeCloseable; import io.deephaven.util.annotations.ScriptApi; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jpy.PyObject; @@ -33,7 +37,10 @@ */ @ScriptApi public class PythonMergedListenerAdapter extends MergedListener { - private final PyObject pyCallable; + private static final Logger log = LoggerFactory.getLogger(PythonMergedListenerAdapter.class); + + private final PyObject pyListenerCallable; + private final PyObject pyOnFailureCallback; /** * Create a Python merged listener. @@ -42,23 +49,26 @@ public class PythonMergedListenerAdapter extends MergedListener { * @param dependencies The tables that must be satisfied before this listener is executed. * @param listenerDescription A description for the UpdatePerformanceTracker to append to its entry description, may * be null. - * @param pyObjectIn Python listener object. + * @param pyListener Python listener object. */ private PythonMergedListenerAdapter( @NotNull ListenerRecorder[] recorders, @Nullable NotificationQueue.Dependency[] dependencies, @Nullable String listenerDescription, - @NotNull PyObject pyObjectIn) { + @NotNull PyObject pyListener, + @NotNull PyObject pyOnFailureCallback) { super(Arrays.asList(recorders), Arrays.asList(dependencies), listenerDescription, null); Arrays.stream(recorders).forEach(rec -> rec.setMergedListener(this)); - this.pyCallable = PythonUtils.pyMergeListenerFunc(pyObjectIn); + this.pyListenerCallable = PythonUtils.pyMergeListenerFunc(Objects.requireNonNull(pyListener)); + this.pyOnFailureCallback = Objects.requireNonNull(pyOnFailureCallback); } public static PythonMergedListenerAdapter create( @NotNull ListenerRecorder[] recorders, @Nullable NotificationQueue.Dependency[] dependencies, @Nullable String listenerDescription, - @NotNull PyObject pyObjectIn) { + @NotNull PyObject pyListener, + @NotNull PyObject pyOnFailureCallback) { if (recorders.length < 2) { throw new IllegalArgumentException("At least two listener recorders must be provided"); } @@ -71,7 +81,8 @@ public static PythonMergedListenerAdapter create( final UpdateGraph updateGraph = allItems[0].getUpdateGraph(allItems); try (final SafeCloseable ignored = ExecutionContext.getContext().withUpdateGraph(updateGraph).open()) { - return new PythonMergedListenerAdapter(recorders, dependencies, listenerDescription, pyObjectIn); + return new PythonMergedListenerAdapter(recorders, dependencies, listenerDescription, pyListener, + pyOnFailureCallback); } } @@ -91,6 +102,22 @@ public ArrayList currentRowsAsUpdates() { @Override protected void process() { - pyCallable.call("__call__"); + pyListenerCallable.call("__call__"); + } + + @Override + protected void propagateErrorDownstream(boolean fromProcess, @NotNull Throwable error, + TableListener.@Nullable Entry entry) { + if (!pyOnFailureCallback.isNone()) { + try { + pyOnFailureCallback.call("__call__", ExceptionUtils.getStackTrace(error)); + } catch (Exception e2) { + // If the Python onFailure callback fails, log the new exception + // and continue with the original exception. + log.error().append("Python on_error callback failed: ").append(e2).endl(); + } + } else { + log.error().append("Python on_error callback is None: ").append(ExceptionUtils.getStackTrace(error)).endl(); + } } } diff --git a/Integrations/src/main/java/io/deephaven/integrations/python/PythonReplayListenerAdapter.java b/Integrations/src/main/java/io/deephaven/integrations/python/PythonReplayListenerAdapter.java index b6eae8bcda5..8df5e544224 100644 --- a/Integrations/src/main/java/io/deephaven/integrations/python/PythonReplayListenerAdapter.java +++ b/Integrations/src/main/java/io/deephaven/integrations/python/PythonReplayListenerAdapter.java @@ -3,6 +3,7 @@ // package io.deephaven.integrations.python; +import org.apache.commons.lang3.exception.ExceptionUtils; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableUpdate; @@ -14,12 +15,16 @@ import io.deephaven.engine.rowset.RowSetShiftData; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.updategraph.UpdateGraph; +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.logger.Logger; import io.deephaven.util.SafeCloseable; import io.deephaven.util.annotations.ScriptApi; +import org.jetbrains.annotations.NotNull; import org.jpy.PyObject; import javax.annotation.Nullable; import java.util.Arrays; +import java.util.Objects; /** @@ -33,7 +38,10 @@ public class PythonReplayListenerAdapter extends InstrumentedTableUpdateListenerAdapter implements TableSnapshotReplayer { private static final long serialVersionUID = -8882402061960621245L; - private final PyObject pyCallable; + private static final Logger log = LoggerFactory.getLogger(PythonReplayListenerAdapter.class); + + private final PyObject pyListenerCallable; + private final PyObject pyOnFailureCallback; private final NotificationQueue.Dependency[] dependencies; /** @@ -43,22 +51,34 @@ public class PythonReplayListenerAdapter extends InstrumentedTableUpdateListener * null. * @param source The source table to which this listener will subscribe. * @param retain Whether a hard reference to this listener should be maintained to prevent it from being collected. - * @param pyObjectIn Python listener object. + * @param pyListener Python listener object. * @param dependencies The tables that must be satisfied before this listener is executed. */ - public static PythonReplayListenerAdapter create(@Nullable String description, Table source, boolean retain, - PyObject pyObjectIn, NotificationQueue.Dependency... dependencies) { + public static PythonReplayListenerAdapter create( + @Nullable String description, + @NotNull Table source, + boolean retain, + @NotNull PyObject pyListener, + @NotNull PyObject pyOnFailureCallback, + @Nullable NotificationQueue.Dependency... dependencies) { final UpdateGraph updateGraph = source.getUpdateGraph(dependencies); try (final SafeCloseable ignored = ExecutionContext.getContext().withUpdateGraph(updateGraph).open()) { - return new PythonReplayListenerAdapter(description, source, retain, pyObjectIn, dependencies); + return new PythonReplayListenerAdapter(description, source, retain, pyListener, pyOnFailureCallback, + dependencies); } } - private PythonReplayListenerAdapter(@Nullable String description, Table source, boolean retain, PyObject pyObjectIn, - NotificationQueue.Dependency... dependencies) { + private PythonReplayListenerAdapter( + @Nullable String description, + @NotNull Table source, + boolean retain, + @NotNull PyObject pyListener, + @NotNull PyObject pyOnFailureCallback, + @Nullable NotificationQueue.Dependency... dependencies) { super(description, source, retain); this.dependencies = dependencies; - this.pyCallable = PythonUtils.pyListenerFunc(pyObjectIn); + this.pyListenerCallable = PythonUtils.pyListenerFunc(Objects.requireNonNull(pyListener)); + this.pyOnFailureCallback = Objects.requireNonNull(pyOnFailureCallback); } @Override @@ -69,13 +89,30 @@ public void replay() { final TableUpdate update = new TableUpdateImpl(source.getRowSet(), emptyRowSet, emptyRowSet, emptyShift, emptyColumnSet); final boolean isReplay = true; - pyCallable.call("__call__", update, isReplay); + pyListenerCallable.call("__call__", update, isReplay); } @Override public void onUpdate(final TableUpdate update) { final boolean isReplay = false; - pyCallable.call("__call__", update, isReplay); + pyListenerCallable.call("__call__", update, isReplay); + } + + @Override + public void onFailureInternal(Throwable originalException, Entry sourceEntry) { + if (!pyOnFailureCallback.isNone()) { + try { + pyOnFailureCallback.call("__call__", ExceptionUtils.getStackTrace(originalException)); + } catch (Throwable e) { + // If the Python onFailure callback fails, log the new exception + // and continue with the original exception. + log.error().append("Python on_error callback failed: ").append(e).endl(); + } + } else { + log.error().append("Python on_error callback is None: ") + .append(ExceptionUtils.getStackTrace(originalException)).endl(); + } + super.onFailureInternal(originalException, sourceEntry); } @Override @@ -83,4 +120,8 @@ public boolean canExecute(final long step) { return super.canExecute(step) && (dependencies.length == 0 || Arrays.stream(dependencies).allMatch(t -> t.satisfied(step))); } + + public boolean isFailed() { + return failed; + } } diff --git a/R/build.gradle b/R/build.gradle index 85583260473..7e542cd1597 100644 --- a/R/build.gradle +++ b/R/build.gradle @@ -138,7 +138,7 @@ def rClientDoc = Docker.registerDockerTask(project, 'rClientDoc') { ''') runCommand('''echo "status = tryCatch(" \ " {" \ - " install.packages('roxygen2', repos='https://cran.r-project.org'); " \ + " install.packages('roxygen2'); " \ " 0" \ " }," \ " error=function(e) 1," \ @@ -179,7 +179,7 @@ def rClientSite = Docker.registerDockerTask(project, 'rClientSite') { runCommand("mkdir -p ${prefix}/src/rdeephaven/docs") runCommand('''echo "status = tryCatch(" \ " {" \ - " install.packages('pkgdown', repos='https://cran.r-project.org'); " \ + " install.packages('pkgdown'); " \ " 0" \ " }," \ " error=function(e) 1," \ diff --git a/R/rdeephaven/DESCRIPTION b/R/rdeephaven/DESCRIPTION index 40d96753586..99188c3bdaa 100644 --- a/R/rdeephaven/DESCRIPTION +++ b/R/rdeephaven/DESCRIPTION @@ -1,7 +1,7 @@ Package: rdeephaven Type: Package Title: R Client for Deephaven Core -Version: 0.36.0 +Version: 0.37.0 Date: 2023-05-12 Author: Deephaven Data Labs Maintainer: Alex Peters diff --git a/R/rdeephaven/README.md b/R/rdeephaven/README.md index 5f81c09ad6c..11c64197b07 100644 --- a/R/rdeephaven/README.md +++ b/R/rdeephaven/README.md @@ -64,90 +64,57 @@ Currently, the R client is only supported on Ubuntu 20.04 or 22.04 and must be b $ sudo apt -y install libxml2-dev pandoc ``` -1. Build the cpp-client (and any dependent libraries) according to the instructions in - https://github.com/deephaven/deephaven-core/blob/main/cpp-client/README.md. - Follow the instructions at least to the point for "Build and install Deephaven C++ client". - At that point you would have both the Deephaven C++ client and any C++ libraries it depends on, - all installed in a particular directory of your choosing. In what follows we assume that - directory is `/path/to/dhcpp`. Independently of where that directory is in your - chosen installation, a file called `env.sh` should exist on it, and a `local` subdirectory - as well. - -2. Choose a directory where the Deephaven R client source code will live. - Here, the source code will be downloaded into a new directory called `rdeephaven`. - Navigate into that directory and clone this subdirectory of `deephaven-core` using git's sparse-checkout: - ```bash - mkdir rdeephaven - cd rdeephaven - git init - git remote add -f origin https://github.com/deephaven/deephaven-core.git - git config core.sparseCheckout true - echo "R/rdeephaven" >> .git/info/sparse-checkout - git pull origin main - ``` +1. Build the Deephaven C++ client (and dependencies) according to the instructions in the + [Deephaven C++ client installation guide](https://github.com/deephaven/deephaven-core/blob/main/cpp-client/README.md). + Follow the instructions at least through "Build and Install Deephaven C++ client". After that, you will have both the + Deephaven C++ client and any C++ libraries it depends on installed in a particular directory of your choosing. + In what follows, we assume that directory is `/path/to/dhcpp`. -3. Set environment variables from the C++ client installation required for building the package. - Use: - ``` +3. Set environment variables from the C++ client installation required for building the R client: + ```bash source /path/to/dhcpp/env.sh ``` - where `/path/to/dhcpp` is the directory you created in step (1) above. - You can ensure the environment variables that are necessary for the steps - that follow are set by checking their values by running the commands: - + where `/path/to/dhcpp` is the directory you created in step 1 above. + Ensure the necessary environment variables are set by checking their values as follows: ``` echo $DHCPP echo $LD_LIBRARY_PATH + echo $NCPUS + echo $CMAKE_PREFIX_PATH ``` - Both environment variables need to be defined for installing the package in the - instructions below. Once the package is installed, you will only need - `LD_LIBRARY_PATH` to be set in the R session where you intend to use the `rdeephaven` library. - If you are starting R from the command line, you can set the environment variable as explained - above. If you are using RStudio, see the note in the following point. - - Refer to the instructions on the C++ client installation for more details on the `dhcpp` directory. - For faster compilation of the R client and its dependencies (particularly the Arrow R client), - use the following commands: + use the following command: ```bash - export NCPUS=`getconf _NPROCESSORS_ONLN` export MAKE="make -j$NCPUS" ``` + + Refer to the instructions on the C++ client installation for more details on the `dhcpp` directory. + +4. The C++ client installation will have left you with a local clone of the full [Deephaven Core Git repository](https://github.com/deephaven/deephaven-core). + Navigate to the `deephaven-core/R/rdeephaven` directory within that clone. -4. Start an R console inside the rdeephaven directory. In that console, install the dephaven client dependencies - (since we are building from source, dependencies will not be automatically pulled in): + If you prefer to have an isolated directory where the Deephaven R client source code will live, use git's sparse-checkout: + ```bash + mkdir rdeephaven + cd rdeephaven + git init + git remote add -f origin https://github.com/deephaven/deephaven-core.git + git config core.sparseCheckout true + echo "R/rdeephaven" >> .git/info/sparse-checkout + git pull origin main + ``` + +5. Start an R console inside the `rdeephaven` directory by running `R`. In that console, install the Deephaven R client dependencies: ```r install.packages(c('Rcpp', 'arrow', 'R6', 'dplyr', 'xml2', 'rmarkdown', 'knitr')) ``` - Then, exit the R console with `quit()`. From the rdeephaven directory, build and install the R client: + Then, exit the R console with `quit()`. From the `rdeephaven` directory, build and install the R client: ```r cd .. && R CMD build rdeephaven && R CMD INSTALL --no-multiarch --with-keep.source rdeephaven_*.tar.gz && rm rdeephaven_*.tar.gz ``` This is needed over the typical `install.packages()` to ensure that the vignettes get built and installed. - - **NOTE** - - If using RStudio for this step, the environment variables that were set in step 3 may not persist into the RStudio - R environment if RStudio is not a child process of the shell where the environment variables were set - (ie, if RStudio is not started from that same shell and after the environment variables are set in that shell). - R supports using a `.Renviron` file for settings like this. You can generate the right content to add - to your .Renviron file (or for creating a new one) using the script under `etc/generate-dotRenviron-lines.sh` - - You can create a new `.Renviron` file under the `deephave-core` directory with the lines producing by running - the `etc/generate-dotRenviron-lines.sh` in the same shell where you set the environment variables; - the script will give you the right content for the `.Renviron` file. - Then, create a new R project from the existing `deephaven-core` directory using RStudio, and the corresponding - R session will inherit all the necessary environment variables for successful compilation. - - If RStudio Server is being used, all of the above must be followed for successful compilation. _In addition_, - use the output from the script `etc/generate-rserverdotconf-lines.sh` and add them to the `rserver.conf` file - for the RStudio Server installation (the location of that file may depend on your particular RStudio server - installation, but a common location is `/etc/rstudio/rserver.conf`). - - - 6. Now, run ```r library(rdeephaven) @@ -156,28 +123,73 @@ Currently, the R client is only supported on Ubuntu 20.04 or 22.04 and must be b For an introduction to the package, run `vignette("rdeephaven")`. - -**NOTE** - -If an error like this occurs in step 4: -```bash -client.cpp:7:10: fatal error: deephaven/client/client.h: No such file or directory - 7 | #include "deephaven/client/client.h" - | ^~~~~~~~~~~~~~~~~~~~~~~~~~~ -compilation terminated. -``` -this means that the C++ compiler does not know where to find the relevant header files for the Deephaven C++ client. This can happen for a handul of reasons: -1. Step 1 was skipped, and the Deephaven C++ client was not installed. In this case, please ensure that the client is installed before attempting to build the R client. -2. The Deephaven C++ client is installed, but the `DHCPP` environment variable is not set. To test this, run +7. The `LD_LIBRARY_PATH` environment variable set in step 2 is necessary for loading the R client once it is installed. + Therefore, you must set `LD_LIBRARY_PATH` manually at the start of each session by running the following command: ```bash - echo $DHCPP + source /path/to/dhcpp/env.sh ``` - If this returns an empty string, set `DHCPP` according to the instructions in step 1 with + This will set `LD_LIBRARY_PATH`, as well as the other environment variables from step 2. + + If you would rather not have to run this command every time you open a new session, you can add the following to `$HOME/.profile`: ```bash - export DHCPP=/path/to/dhcpp + # source DHCPP env variables if dhcpp directory exists + current_dhcpp_dir=/path/to/dhcpp" + if [ -d "$current_dhcpp_dir" ] ; then + source "$current_dhcpp_dir"/env.sh + fi ``` -3. The Deephaven C++ client is installed and the `DHCPP` environment variable is set, but the current project is not configured to allow the compiler to access the Deephaven `dhcpp` and `src` directories. This is more difficult to give advice on, as it is an IDE-dependent problem. Consult your IDE's documentation on C/C++ compiler include paths for more information. + This will automatically set `LD_LIBRARY_PATH` at the beginning of every session _for this user_, so that any process (R or RStudio) may access its value. + If you move the dhcpp directory, or need to point R to another version of it, `current_dhcpp_dir` should be modified accordingly. + +## Common Errors + +- **Cannot compile the R client** + If an error like this occurs in step 4: + ```bash + client.cpp:7:10: fatal error: deephaven/client/client.h: No such file or directory + 7 | #include "deephaven/client/client.h" + | ^~~~~~~~~~~~~~~~~~~~~~~~~~~ + compilation terminated. + ``` + this means that the C++ compiler does not know where to find the relevant header files for the Deephaven C++ client. This can happen for a handul of reasons: + 1. Step 1 was skipped, and the Deephaven C++ client was not installed. In this case, please ensure that the client is installed before attempting to build the R client. + 2. The Deephaven C++ client is installed, but the `DHCPP` environment variable is not set. To test this, run + ```bash + echo $DHCPP + ``` + If this returns an empty string, set `DHCPP` according to the instructions in step 2a with + ```bash + source /path/to/dhcpp/env.sh + ``` + 3. The Deephaven C++ client is installed and the `DHCPP` environment variable is set, but the current project is not configured to allow the compiler + to access the Deephaven `dhcpp` and `src` directories. This is more difficult to give advice on, as it is an IDE-dependent problem. Consult your IDE's + documentation on C/C++ compiler include paths for more information. + + - **Cannot load the R client** + + Once the R client is successfully installed, you may try to load it at a later date, only to find this error: + ```bash + > library(rdeephaven) + Error: package or namespace load failed for ‘rdeephaven’ in dyn.load(file, DLLpath = DLLpath, ...): + unable to load shared object '/home/user/R/x86_64-pc-linux-gnu-library/4.4/rdeephaven/libs/rdeephaven.so': + libdhclient.so: cannot open shared object file: No such file or directory + ``` + This very likely means that the `LD_LIBRARY_PATH` environment variable is not set. To rectify this, run + ```bash + source /path/to/dhcpp/env.sh + ``` + from the parent session and try again. Alternatively, see step 6 for a way to solve this problem semi-permanently. + + RStudio presents its own solution to this problem that RStudio users may want to use instead of the semi-permanent solution in step 6. + RStudio supports using a `.Renviron` file for setting environment variables. If the correct environment variables are currently set (see step 2), + you can generate the right content for a `.Renviron` file by running the script at `etc/generate-dotRenviron-lines.sh`. Then, copy the output + of that script into a new file called `.Renviron`, and save it in the `deephaven-core` directory. Then, create a new R project from the existing + `deephaven-core` directory using RStudio, and the corresponding R session will inherit all the necessary environment variables for successful compilation. + + If RStudio Server is being used, the `.Renviron` files _must_ be set for successful compilation. _In addition_, run the `etc/generate-rserverdotconf-lines.sh` script, + and the script's outputs to the `rserver.conf` file for the RStudio Server installation (the location of that file may depend on your particular RStudio server + installation, but a common location is `/etc/rstudio/rserver.conf`). ## Running the unit tests diff --git a/Util/src/main/java/io/deephaven/util/MultiException.java b/Util/src/main/java/io/deephaven/util/MultiException.java index f2dcc13a90c..e5ce94740c8 100644 --- a/Util/src/main/java/io/deephaven/util/MultiException.java +++ b/Util/src/main/java/io/deephaven/util/MultiException.java @@ -105,7 +105,7 @@ public void printStackTrace(PrintStream s) { @Override public String getMessage() { StringBuilder sb = new StringBuilder(); - sb.append(super.getMessage()).append(": \n"); + sb.append(super.getMessage()).append(":\n"); for (int i = 0; i < causes.length; i++) { sb.append("Cause ").append(i).append(": "); sb.append(causes[i].toString()); diff --git a/Util/src/main/java/io/deephaven/util/SimpleTypeMap.java b/Util/src/main/java/io/deephaven/util/SimpleTypeMap.java index 7b15c86f8b5..c1ab8b67abb 100644 --- a/Util/src/main/java/io/deephaven/util/SimpleTypeMap.java +++ b/Util/src/main/java/io/deephaven/util/SimpleTypeMap.java @@ -7,17 +7,56 @@ public final class SimpleTypeMap { - public static SimpleTypeMap create(V forBoolean, V forChar, V forByte, V forShort, V forInt, V forLong, - V forFloat, V forDouble, V forObject) { + /** + * Create a mapping from type {@link Class classes} to a value. + * + * @param forBoolean The mapping for {@code boolean} types (note {@link Boolean} maps to {@code forObject}) + * @param forChar The mapping for {@code char} and {@link Character} types + * @param forByte The mapping for {@code byte} and {@link Byte} types + * @param forShort The mapping for {@code short} and {@link Short} types + * @param forInt The mapping for {@code int} and {@link Integer} types + * @param forLong The mapping for {@code long} and {@link Long} types + * @param forFloat The mapping for {@code float} and {@link Float} types + * @param forDouble The mapping for {@code double} and {@link Double} types + * @param forObject The mapping for all other types + * @return A SimpleTypeMap to the provided values + */ + public static SimpleTypeMap create( + V forBoolean, + V forChar, + V forByte, + V forShort, + V forInt, + V forLong, + V forFloat, + V forDouble, + V forObject) { final HashMap, V> map = new HashMap<>(); + map.put(boolean.class, forBoolean); + // Note: Booleans are treated as Objects, unlike other boxed primitives + map.put(char.class, forChar); + map.put(Character.class, forChar); + map.put(byte.class, forByte); + map.put(Byte.class, forByte); + map.put(short.class, forShort); + map.put(Short.class, forShort); + map.put(int.class, forInt); + map.put(Integer.class, forInt); + map.put(long.class, forLong); + map.put(Long.class, forLong); + map.put(float.class, forFloat); + map.put(Float.class, forFloat); + map.put(double.class, forDouble); + map.put(Double.class, forDouble); + return new SimpleTypeMap<>(map, forObject); } diff --git a/Util/src/main/resources/io/deephaven/Util.gwt.xml b/Util/src/main/resources/io/deephaven/Util.gwt.xml new file mode 100644 index 00000000000..0a6e96c6463 --- /dev/null +++ b/Util/src/main/resources/io/deephaven/Util.gwt.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/buildSrc/src/main/groovy/GwtTools.groovy b/buildSrc/src/main/groovy/GwtTools.groovy index 23b5711d005..148eacfa0d4 100644 --- a/buildSrc/src/main/groovy/GwtTools.groovy +++ b/buildSrc/src/main/groovy/GwtTools.groovy @@ -63,6 +63,7 @@ class GwtTools { generateJsInteropExports = true // TODO move this down a line when we want to give clients js that is not super strict / rigged to blow checkAssertions = true + setExtraArgs('-includeJsInteropExports', 'io.deephaven.*') if (gwtDev) { saveSource = true extra = extras diff --git a/cog.toml b/cog.toml new file mode 100644 index 00000000000..bf4a185fe6c --- /dev/null +++ b/cog.toml @@ -0,0 +1,6 @@ +[changelog] +path = "CHANGELOG.md" +template = "deephaven-changelog" +remote = "github.com" +repository = "deephaven-core" +owner = "deephaven" \ No newline at end of file diff --git a/cpp-client/README.md b/cpp-client/README.md index bff05cf652e..f03e723d972 100644 --- a/cpp-client/README.md +++ b/cpp-client/README.md @@ -281,12 +281,14 @@ Notes 9. Now configure the build for Deephaven Core: ``` - cmake -B build -S . -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX=%DHINSTALL% -DX_VCPKG_APPLOCAL_DEPS_INSTALL=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + cmake -B build -S . -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX=%DHINSTALL% -DX_VCPKG_APPLOCAL_DEPS_INSTALL=ON ``` -10. Finally, build and install Deephaven Core: +10. Finally, build and install Deephaven Core. Note that the build type (RelWithDebInfo) is specified differently for the Windows build + than it is for the Ubuntu build. For Windows, we specify the configuration type directly in the build step using the --config flag. ``` - cmake --build build --target install + # Replace '16' by the number of CPU threads you want to use for building + cmake --build build --config RelWithDebInfo --target install -- /p:CL_MPCount=16 -m:1 ``` 11. Run the tests. diff --git a/cpp-client/deephaven/CMakeLists.txt b/cpp-client/deephaven/CMakeLists.txt index 2c8be6d3a3d..32e7b5fabe5 100644 --- a/cpp-client/deephaven/CMakeLists.txt +++ b/cpp-client/deephaven/CMakeLists.txt @@ -8,7 +8,7 @@ endif() project(deephaven) -set(deephaven_VERSION 0.36.0) +set(deephaven_VERSION 0.37.0) set(CMAKE_CXX_STANDARD 17) # for CMAKE_INSTALL_{dir} diff --git a/cpp-client/deephaven/dhcore/CMakeLists.txt b/cpp-client/deephaven/dhcore/CMakeLists.txt index 219316d3eb0..1a0626757a3 100644 --- a/cpp-client/deephaven/dhcore/CMakeLists.txt +++ b/cpp-client/deephaven/dhcore/CMakeLists.txt @@ -23,6 +23,7 @@ set(ALL_FILES src/immerutil/immer_column_source.cc src/interop/testapi/basic_interop_interactions.cc src/interop/interop_util.cc + src/interop/utility_interop.cc src/ticking/barrage_processor.cc src/ticking/immer_table_state.cc src/ticking/index_decoder.cc @@ -52,6 +53,7 @@ set(ALL_FILES include/public/deephaven/dhcore/container/row_sequence.h include/public/deephaven/dhcore/interop/testapi/basic_interop_interactions.h include/public/deephaven/dhcore/interop/interop_util.h + include/public/deephaven/dhcore/interop/utility_interop.h include/public/deephaven/dhcore/ticking/barrage_processor.h include/public/deephaven/dhcore/ticking/ticking.h include/public/deephaven/dhcore/utility/cython_support.h diff --git a/cpp-client/deephaven/dhcore/include/public/deephaven/dhcore/interop/utility_interop.h b/cpp-client/deephaven/dhcore/include/public/deephaven/dhcore/interop/utility_interop.h new file mode 100644 index 00000000000..4885b8cdfa5 --- /dev/null +++ b/cpp-client/deephaven/dhcore/include/public/deephaven/dhcore/interop/utility_interop.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +#pragma once + +#include "deephaven/dhcore/interop/interop_util.h" + +extern "C" { +void deephaven_dhcore_utility_GetEnv(const char *envname, + deephaven::dhcore::interop::StringHandle *string_handle, + deephaven::dhcore::interop::StringPoolHandle *string_pool_handle, + deephaven::dhcore::interop::ErrorStatus *status); + +void deephaven_dhcore_utility_SetEnv(const char *envname, const char *value, + deephaven::dhcore::interop::ErrorStatus *status); + +void deephaven_dhcore_utility_UnsetEnv(const char *envname, + deephaven::dhcore::interop::ErrorStatus *status); +} // extern "C" diff --git a/cpp-client/deephaven/dhcore/include/public/deephaven/dhcore/utility/utility.h b/cpp-client/deephaven/dhcore/include/public/deephaven/dhcore/utility/utility.h index 40ad15f3a9b..574b6beeea0 100644 --- a/cpp-client/deephaven/dhcore/include/public/deephaven/dhcore/utility/utility.h +++ b/cpp-client/deephaven/dhcore/include/public/deephaven/dhcore/utility/utility.h @@ -217,6 +217,19 @@ std::string Basename(std::string_view path); */ [[nodiscard]] std::optional GetEnv(const std::string& envname); +/** + * Sets a value in the environment. + * @param envname the key + * @param value the value to set in the environment + */ +void SetEnv(const std::string& envname, const std::string& value); + +/** + * Unsets a value in the environment. + * @param envname the key to unset + */ +void UnsetEnv(const std::string& envname); + /** * Enables or disables echo for stdin. * @param enable true to enable, false to disable diff --git a/cpp-client/deephaven/dhcore/src/interop/utility_interop.cc b/cpp-client/deephaven/dhcore/src/interop/utility_interop.cc new file mode 100644 index 00000000000..cb787dc1a5a --- /dev/null +++ b/cpp-client/deephaven/dhcore/src/interop/utility_interop.cc @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +#include "deephaven/dhcore/interop/utility_interop.h" + +#include "deephaven/dhcore/interop/interop_util.h" +#include "deephaven/dhcore/utility/utility.h" + +using deephaven::dhcore::interop::ErrorStatus; +using deephaven::dhcore::interop::StringHandle; +using deephaven::dhcore::interop::StringPoolBuilder; +using deephaven::dhcore::interop::StringPoolHandle; +using deephaven::dhcore::utility::GetEnv; +using deephaven::dhcore::utility::SetEnv; +using deephaven::dhcore::utility::UnsetEnv; + +extern "C" { +void deephaven_dhcore_utility_GetEnv(const char *envname, + StringHandle *string_handle, StringPoolHandle *string_pool_handle, ErrorStatus *status) { + status->Run([=]() { + StringPoolBuilder builder; + auto result = GetEnv(envname); + if (result.has_value()) { + *string_handle = builder.Add(*result); + } + // If envname is not found in the environment, caller will see an empty builder + *string_pool_handle = builder.Build(); + }); +} + +void deephaven_dhcore_utility_SetEnv(const char *envname, const char *value, ErrorStatus *status) { + status->Run([=]() { + SetEnv(envname, value); + }); +} + +void deephaven_dhcore_utility_UnsetEnv(const char *envname, ErrorStatus *status) { + status->Run([=]() { + UnsetEnv(envname); + }); +} +} // extern "C" diff --git a/cpp-client/deephaven/dhcore/src/utility/utility_platform_specific.cc b/cpp-client/deephaven/dhcore/src/utility/utility_platform_specific.cc index 6ccf9407dbe..f0b83a74dd7 100644 --- a/cpp-client/deephaven/dhcore/src/utility/utility_platform_specific.cc +++ b/cpp-client/deephaven/dhcore/src/utility/utility_platform_specific.cc @@ -2,6 +2,7 @@ * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending */ #include +#include #include #include #include "deephaven/dhcore/utility/utility.h" @@ -31,6 +32,36 @@ std::string GetTidAsString() { #endif } +#if defined(_WIN32) +namespace { +// We need to call WSAStartup() before using other Winsock calls. +// We only do this once during the lifetime of the program; and we +// never bother to call WSACleanup(). +void EnsureWsaStartup() { + static std::mutex mutex; + static bool startupSucceeded = false; + + std::unique_lock guard(mutex); + if (startupSucceeded) { + return; + } + + int32_t versionRequested = 0x202; + WSADATA wsaData; + auto err = WSAStartup(versionRequested, &wsaData); + if (err != 0) { + auto message = fmt::format("WSAStartup failed with error: {}", err); + throw std::runtime_error(DEEPHAVEN_LOCATION_STR(message)); + } + if (wsaData.wVersion != versionRequested) { + auto message = fmt::format("Got an unexpected version {:x} of winsock.dll", wsaData.wVersion); + throw std::runtime_error(DEEPHAVEN_LOCATION_STR(message)); + } + startupSucceeded = true; +} +} // namespace +#endif + std::string GetHostname() { #if defined(__unix__) char hostname[HOST_NAME_MAX]; @@ -39,7 +70,8 @@ std::string GetHostname() { addrinfo *info; const int r = getaddrinfo(hostname, nullptr, &hints, &info); if (r != 0 || info == nullptr) { - throw std::runtime_error(DEEPHAVEN_LOCATION_STR("getaddrinfo failed: ") + gai_strerror(r)); + auto message = fmt::format("getaddrinfo failed: {}", gai_strerror(r)); + throw std::runtime_error(DEEPHAVEN_LOCATION_STR(message)); } // Of all the alternatives, pick the longest. std::size_t maxlen = std::strlen(info->ai_canonname); @@ -58,13 +90,13 @@ std::string GetHostname() { freeaddrinfo(info); return result; #elif defined(_WIN32) + EnsureWsaStartup(); char hostname[256]; const int r = gethostname(hostname, sizeof(hostname)); if (r != 0) { int lasterr = WSAGetLastError(); - throw std::runtime_error( - DEEPHAVEN_LOCATION_STR("gethostname failed: error code ") + - std::to_string(lasterr)); + auto message = fmt::format("gethostname failed: error code {}", lasterr); + throw std::runtime_error(DEEPHAVEN_LOCATION_STR(message)); } return std::string(hostname); #else @@ -72,21 +104,99 @@ std::string GetHostname() { #endif } +namespace { +/** + * This method wraps a function-local static mutex. The purpose of this mutex is to + * synchronize the calls GetEnv(), SetEnv(), and UnsetEnv(). + * + * The rationale for synchronizing these calls is that they are not guaranteed + * reentrant on either Linux or Windows. On Linux, "man getenv" says + * + * The implementation of getenv() is not required to be reentrant. The + * string pointed to by the return value of getenv() may be statically al‐ + * located, and can be modified by a subsequent call to getenv(), + * putenv(3), setenv(3), or unsetenv(3). + * + * On Windows, https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-wputenv?view=msvc-170 + * says + * + * The _putenv and _getenv families of functions are not thread-safe. + * _getenv could return a string pointer while _putenv is modifying the string, + * causing random failures. Make sure that calls to these functions are synchronized. + * + * Finally, we use a function-local static rather than a global so we don't have to think about / + * worry about whether global initialization was done correctly on the mutex object. + * This "worry" might be unfounded. + */ +std::mutex &MutexForEnvInvocations() { + static std::mutex the_mutex; + return the_mutex; +} +} // namespace + std::optional GetEnv(const std::string& envname) { #if defined(__unix__) + // Protect against concurrent XXXEnv() calls. See comment in MutexForEnvInvocations() + std::unique_lock guard(MutexForEnvInvocations()); const char* ret = getenv(envname.c_str()); if (ret != nullptr) { return std::string(ret); } return {}; #elif defined(_WIN32) + // Protect against concurrent XXXEnv() calls. See comment in MutexForEnvInvocations() + std::unique_lock guard(MutexForEnvInvocations()); static char ret[1024]; size_t len; const errno_t err = getenv_s(&len, ret, sizeof(ret), envname.c_str()); - if (err == 0) { - return std::string(ret); + // Return an unset optional if there's an error, or if the key is not found. + // len == 0 means "key not found" on Windows. + if (err != 0 || len == 0) { + return {}; } - return {}; + return std::string(ret); +#else +#error "Unsupported configuration" +#endif +} + +void SetEnv(const std::string& envname, const std::string& value) { +#if defined(__unix__) + // Protect against concurrent XXXEnv() calls. See comment in MutexForEnvInvocations() + std::unique_lock guard(MutexForEnvInvocations()); + + auto res = setenv(envname.c_str(), value.c_str(), 1); + if (res != 0) { + auto message = fmt::format("setenv failed, error={}", strerror(errno)); + throw std::runtime_error(DEEPHAVEN_LOCATION_STR(message)); + } +#elif defined(_WIN32) + // Protect against concurrent XXXEnv() calls. See comment in MutexForEnvInvocations() + std::unique_lock guard(MutexForEnvInvocations()); + + auto res = _putenv_s(envname.c_str(), value.c_str()); + if (res != 0) { + int lasterr = WSAGetLastError(); + auto message = fmt::format("_putenv_s failed: error code {}", lasterr); + throw std::runtime_error(DEEPHAVEN_LOCATION_STR(message)); + } +#else +#error "Unsupported configuration" +#endif +} + +void UnsetEnv(const std::string& envname) { +#if defined(__unix__) + // Protect against concurrent XXXEnv() calls. See comment in MutexForEnvInvocations() + std::unique_lock guard(MutexForEnvInvocations()); + + auto res = unsetenv(envname.c_str()); + if (res != 0) { + auto message = fmt::format("unsetenv failed, error={}", strerror(errno)); + throw std::runtime_error(DEEPHAVEN_LOCATION_STR(message)); + } +#elif defined(_WIN32) + SetEnv(envname, ""); #else #error "Unsupported configuration" #endif @@ -117,6 +227,6 @@ void SetStdinEcho(const bool enable) { #else #error "Unsupported configuration" #endif -} +} } // namespace deephaven::dhcore::utility diff --git a/cpp-client/deephaven/tests/src/utility_test.cc b/cpp-client/deephaven/tests/src/utility_test.cc index f42dfdc51d8..aa81b68afc6 100644 --- a/cpp-client/deephaven/tests/src/utility_test.cc +++ b/cpp-client/deephaven/tests/src/utility_test.cc @@ -11,6 +11,8 @@ using deephaven::dhcore::utility::GetEnv; using deephaven::dhcore::utility::GetTidAsString; using deephaven::dhcore::utility::GetHostname; using deephaven::dhcore::utility::ObjectId; +using deephaven::dhcore::utility::SetEnv; +using deephaven::dhcore::utility::UnsetEnv; namespace deephaven::client::tests { TEST_CASE("Base64encode", "[utility]") { @@ -66,12 +68,43 @@ TEST_CASE("GetHostname", "[utility]") { // This isn't much of a test, but if it can compile on all supported // platforms (Linux and Windows) then that is at least a sanity check // (that the entry point exists). For now we just visually spot-check -// that ireturns the right value. +// that it returns the right value. TEST_CASE("GetEnv", "[utility]") { - auto path = GetEnv("PATH"); - // Very suspect if neither Windows nor Linux has a PATH set in their - // environment. - REQUIRE(path.has_value()); - fmt::println("PATH is: {}", *path); +#if defined(__unix__) + const char *expected_key = "PATH"; +#elif defined(_WIN32) + const char *expected_key = "OS"; +#else +#error "Unsupported configuration" +#endif + + auto value = GetEnv(expected_key); + REQUIRE(value.has_value()); + fmt::println("{} is: {}", expected_key, *value); +} + +// Confirm that SetEnv leaves something that GetEnv can find. +TEST_CASE("SetEnv", "[utility]") { + std::string unlikely_key = "Deephaven__serious_realtime_data_tools"; + std::string unlikely_value = "query_engine_APIs_and_user_interfaces"; + { + auto value = GetEnv(unlikely_key); + if (value.has_value()) { + fmt::println(std::cerr, "unexpected value is {}", *value); + } + REQUIRE(!value.has_value()); + } + + SetEnv(unlikely_key, unlikely_value); + { + auto value = GetEnv(unlikely_key); + REQUIRE(unlikely_value == value); + } + + UnsetEnv(unlikely_key); + { + auto value = GetEnv(unlikely_key); + REQUIRE(!value.has_value()); + } } } // namespace deephaven::client::tests diff --git a/cpp-client/deephaven/vcpkg.json b/cpp-client/deephaven/vcpkg.json index c96d5038d11..e52660ae681 100644 --- a/cpp-client/deephaven/vcpkg.json +++ b/cpp-client/deephaven/vcpkg.json @@ -11,7 +11,7 @@ ], "overrides": [ - { "name": "arrow", "version": "16.0.0" }, + { "name": "arrow", "version": "16.1.0#1" }, { "name": "protobuf", "version": "4.25.1" } ] } diff --git a/csharp/ExcelAddIn/DeephavenExcelFunctions.cs b/csharp/ExcelAddIn/DeephavenExcelFunctions.cs new file mode 100644 index 00000000000..9aeab0ad654 --- /dev/null +++ b/csharp/ExcelAddIn/DeephavenExcelFunctions.cs @@ -0,0 +1,65 @@ +using System.Diagnostics.CodeAnalysis; +using Deephaven.ExcelAddIn.ExcelDna; +using Deephaven.ExcelAddIn.Factories; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Operations; +using ExcelDna.Integration; + +namespace Deephaven.ExcelAddIn; + +public static class DeephavenExcelFunctions { + private static readonly StateManager StateManager = new(); + + [ExcelCommand(MenuName = "Deephaven", MenuText = "Connections")] + public static void ShowConnectionsDialog() { + ConnectionManagerDialogFactory.CreateAndShow(StateManager); + } + + [ExcelFunction(Description = "Snapshots a table", IsThreadSafe = true)] + public static object DEEPHAVEN_SNAPSHOT(string tableDescriptor, object filter, object wantHeaders) { + if (!TryInterpretCommonArgs(tableDescriptor, filter, wantHeaders, out var tq, out var wh, out var errorText)) { + return errorText; + } + + // These two are used by ExcelDNA to share results for identical invocations. The functionName is arbitary but unique. + const string functionName = "Deephaven.ExcelAddIn.DeephavenExcelFunctions.DEEPHAVEN_SNAPSHOT"; + var parms = new[] { tableDescriptor, filter, wantHeaders }; + ExcelObservableSource eos = () => new SnapshotOperation(tq, wh, StateManager); + return ExcelAsyncUtil.Observe(functionName, parms, eos); + } + + [ExcelFunction(Description = "Subscribes to a table", IsThreadSafe = true)] + public static object DEEPHAVEN_SUBSCRIBE(string tableDescriptor, object filter, object wantHeaders) { + if (!TryInterpretCommonArgs(tableDescriptor, filter, wantHeaders, out var tq, out var wh, out string errorText)) { + return errorText; + } + // These two are used by ExcelDNA to share results for identical invocations. The functionName is arbitary but unique. + const string functionName = "Deephaven.ExcelAddIn.DeephavenExcelFunctions.DEEPHAVEN_SUBSCRIBE"; + var parms = new[] { tableDescriptor, filter, wantHeaders }; + ExcelObservableSource eos = () => new SubscribeOperation(tq, wh, StateManager); + return ExcelAsyncUtil.Observe(functionName, parms, eos); + } + + private static bool TryInterpretCommonArgs(string tableDescriptor, object filter, object wantHeaders, + [NotNullWhen(true)]out TableQuad? tableQuadResult, out bool wantHeadersResult, out string errorText) { + tableQuadResult = null; + wantHeadersResult = false; + if (!TableTriple.TryParse(tableDescriptor, out var tt, out errorText)) { + return false; + } + + if (!ExcelDnaHelpers.TryInterpretAs(filter, "", out var condition)) { + errorText = "Can't interpret FILTER argument"; + return false; + } + + + if (!ExcelDnaHelpers.TryInterpretAs(wantHeaders, false, out wantHeadersResult)) { + errorText = "Can't interpret WANT_HEADERS argument"; + return false; + } + + tableQuadResult = new TableQuad(tt.EndpointId, tt.PersistentQueryId, tt.TableName, condition); + return true; + } +} diff --git a/csharp/ExcelAddIn/ExcelAddIn.csproj b/csharp/ExcelAddIn/ExcelAddIn.csproj new file mode 100644 index 00000000000..4858c64169e --- /dev/null +++ b/csharp/ExcelAddIn/ExcelAddIn.csproj @@ -0,0 +1,24 @@ + + + + net8.0-windows + enable + enable + true + True + + + + True + + + + True + + + + + + + + diff --git a/csharp/ExcelAddIn/ExcelAddIn.csproj.user b/csharp/ExcelAddIn/ExcelAddIn.csproj.user new file mode 100644 index 00000000000..476df3e2650 --- /dev/null +++ b/csharp/ExcelAddIn/ExcelAddIn.csproj.user @@ -0,0 +1,15 @@ + + + + + + Form + + + Form + + + Form + + + \ No newline at end of file diff --git a/csharp/ExcelAddIn/ExcelAddIn.sln b/csharp/ExcelAddIn/ExcelAddIn.sln new file mode 100644 index 00000000000..1b006efe92c --- /dev/null +++ b/csharp/ExcelAddIn/ExcelAddIn.sln @@ -0,0 +1,31 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.7.34221.43 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ExcelAddIn", "ExcelAddIn.csproj", "{08852A0D-DB96-404E-B3CE-BF30F2AD3F74}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DeephavenClient", "..\client\DeephavenClient\DeephavenClient.csproj", "{6848407D-1CEF-4433-92F4-6047AE3D2C52}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {08852A0D-DB96-404E-B3CE-BF30F2AD3F74}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {08852A0D-DB96-404E-B3CE-BF30F2AD3F74}.Debug|Any CPU.Build.0 = Debug|Any CPU + {08852A0D-DB96-404E-B3CE-BF30F2AD3F74}.Release|Any CPU.ActiveCfg = Release|Any CPU + {08852A0D-DB96-404E-B3CE-BF30F2AD3F74}.Release|Any CPU.Build.0 = Release|Any CPU + {6848407D-1CEF-4433-92F4-6047AE3D2C52}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6848407D-1CEF-4433-92F4-6047AE3D2C52}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6848407D-1CEF-4433-92F4-6047AE3D2C52}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6848407D-1CEF-4433-92F4-6047AE3D2C52}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {A22A4DB3-DD84-46EB-96A6-7935E9E59356} + EndGlobalSection +EndGlobal diff --git a/csharp/ExcelAddIn/Properties/launchSettings.json b/csharp/ExcelAddIn/Properties/launchSettings.json new file mode 100644 index 00000000000..5a0aac58161 --- /dev/null +++ b/csharp/ExcelAddIn/Properties/launchSettings.json @@ -0,0 +1,9 @@ +{ + "profiles": { + "Excel": { + "commandName": "Executable", + "executablePath": "C:\\Program Files\\Microsoft Office\\root\\Office16\\EXCEL.EXE", + "commandLineArgs": "/x \"ExcelAddIn-AddIn64.xll\"" + } + } +} \ No newline at end of file diff --git a/csharp/ExcelAddIn/README.md b/csharp/ExcelAddIn/README.md new file mode 100644 index 00000000000..7717cfd542b --- /dev/null +++ b/csharp/ExcelAddIn/README.md @@ -0,0 +1,219 @@ +# Building the Excel Add-In on Windows 10 / 11. + +These instructions show how to install and run the Deephaven Excel Add-In +on Windows. These instructions also happen to build the Deephaven C# Client as a +side-effect. However if your goal is only to build the Deephaven C# Client, +please see the instructions at %DHSRC%\csharp\client\README.md +(does not exist yet). + +We have tested these instructions on Windows 10 and 11 with Visual Studio +Community Edition. + +# Before using the Excel Add-In + +To actually use the Deephaven Excel Add-In, you will eventually need to have +at least one Community Core or Enterprise Core+ server running. You don't need +the server yet, and you can successfully follow these build instructions +without a server. However, you will eventually need a server when you want to +run it. + +If you don't have a Deephaven Community Core server installation, +you can use these instructions to build one. +https://deephaven.io/core/docs/how-to-guides/launch-build/ + +Note that it is only possible to build a server on Linux. Building a server +on Windows is not currently supported. + +For Deephaven Enterprise Core+, contact your IT administrator. + +# Building the Excel Add-In on Windows 10 / Windows 11 + +## Prerequisites + +## Build machine specifications + +In our experience following this instructions on a fresh Windows 11 VM +required a total of 125G of disk space to install and build everything. +We recommend a machine with at least 200G of free disk space in order to +leave a comfortable margin. + +Also, building the dependencies with vcpkg is very resource-intensive. +A machine that has more cores will be able to finish faster. +We recommend at least 16 cores for the build process. Note that *running* +the Excel Add-In does not require that many cores. + +## Tooling + +### Excel + +You will need a recent version of Excel installed. We recommend Office 21 +or Office 365. Note that the Add-In only works with locally-installed +versions of Excel (i.e. software installed on your computer). It does not +work with the browser-based web version of Excel. + +### .NET + +Install the .NET Core SDK, version 8.0. +Look for the "Windows | x64" link +[here](https://dotnet.microsoft.com/en-us/download/dotnet/8.0) + +### Visual Studio + +Install Visual Studio 2022 Community Edition (or Professional, or Enterprise) +from [here](https://visualstudio.microsoft.com/downloads/) + +When the installer runs, select both workloads +"Desktop development with C++" and ".NET desktop development". + +If Visual Studio is already installed, use Tools -> Get Tools and Features +to add those workloads if necessary. + +### git + +Use your preferred version of git, or install Git from +[here](https://git-scm.com/download/win) + +## C++ client + +The Deephaven Excel Add-In relies on the Deephaven C# Client, which in turn +requires the Deephaven C++ Client (Community Core version). Furthermore, if +you want to use Enterprise Core+ features, you also need the Deephaven C++ +Client for Enterprise Core+. + +The instructions below describe how to build these libraries. + +### Build the Deephaven C++ Client (Community Core version) + +Follow the instructions at [repository root]/cpp-client/README.md under the +section, under "Building the C++ client on Windows 10 / Windows 11". + +When that process is done, you will have C++ client binaries in the +directory you specified in your DHINSTALL environment variable. + +### (Optional) build the Deephaven C++ Client (Enterprise Core+ version) + +To access Enterprise features, build the Enterprise Core+ version as well. +It will also store its binaries in the same DHINSTALL directory. + +(instructions TODO) + +## Build the Excel Add-In and C# Add-In + +You can build the Add-In from inside Visual Studio or from the Visual Studio +Command Prompt. + +### From within Visual Studio + +1. Open the Visual Studio solution file +[repository root]\csharp\ExcelAddIn\ExcelAddIn.sln + +2. Click on BUILD -> Build solution + +### From the Visual Studio Command Prompt + +``` +cd [repository root]\csharp\ExcelAddIn +devenv ExcelAddIn.sln /build Release +``` + +## Run the Add-In + +### Environment variables + +Set these variables (or keep them set from the above steps) to the locations +of your Deephaven source directory and your Deephaven install directory. + +``` +set DHSRC=%HOMEDRIVE%%HOMEPATH%\dhsrc +set DHINSTALL=%HOMEDRIVE%%HOMEPATH%\dhinstall +``` + +### Running from within Visual Studio + +1. In order to actually function, the Add-In requires the C++ Client binaries + built in the above steps. The easiest thing to do is simply copy all the + binaries into your Visual Studio build directory: + +Assuming a Debug build: + +copy /Y %DHINSTALL%\bin %DHSRC%\csharp\ExcelAddIn\bin\Debug\net8.0-windows + +If you are doing a Release build, change "Debug" to "Release" in the above path. + +2. Inside Visual Studio Select Debug -> Start Debugging + +Visual Studio will launch Excel automatically. Excel will launch with a +Security Notice because the add-in is not signed. Click "Enable this add-in +for this session only." + +### From standalone Excel + +To install the add-in into Excel, we need put the relevant files into a +directory and then point Excel to that directory. These steps assume you +have already built the C# project with Visual Studio. + +For simplicity, we will make a folder on the Desktop called "exceladdin". + +``` +set ADDINDIR=%HOMEDRIVE%%HOMEPATH%\Desktop\exceladdin +mkdir %ADDINDIR% +``` + +Now copy the C++ files, the C# files, and the XLL file to this directory: +``` +copy /Y %DHINSTALL%\bin\* %ADDINDIR% +copy /Y %DHSRC%\csharp\ExcelAddIn\bin\Debug\net8.0-windows\* %ADDINDIR% +copy /Y %DHSRC%\csharp\ExcelAddIn\bin\Debug\net8.0-windows\publish\ExcelAddIn-Addin64-packed.xll %ADDINDIR% +``` + +As above, if you happen to have done a Release build in C#, then change Debug to Release in the above paths. + +Then, run Excel and follow the following steps. + +1. Click on "Options" at the lower left of the window. +2. Click on "Add-ins" on the left, second from the bottom. +3. At the bottom of the screen click, near "Manage", select "Excel Add-ins" + from the pulldown, and then click "Go..." +4. In the next screen click "Browse..." +5. Navigate to your %ADDINDIR% directory and click on the ExcelAddIn-Addin64-packed.xll file that you recently copied there +6. Click OK + + +## Test the add-in + +### Without connecting to a Deephaven server + +1. In Excel, click on Add-ins -> Deephaven -> Connections. This should bring + up a Connections form. If so, the C# code is functioning correctly. + +2. In the following steps we deliberately use nonsense connection settings + in order to quickly trigger an error. Even thought he connection settings + are nonsense, getting an error quickly confirms that the functionality + of the C++ code is working. + +3. Inside Connections, click the "New Connection" button. A "Credentials + Editor" window will pop up. Inside this window, enter "con1" for the + Connection ID, select the "Community Core" button, and enter a nonsense + endpoint address like "abc...def" + +4. Press Test Credentials. You should immediately see an error like + "Can't get configuration constants. Error 14: DNS resolution failed for + abc...def" + +5. Enterprise users can do a similar test by selecting the Enterprise Core+ + button and putting the same nonsense address abc...def into the JSON URL + field. + +### By connecting to a Deephaven server + +If the above tests pass, the add-in is probably installed correctly. + +To test the add-in with a Deephaven server, you will need the following +information. + +1. For Community Core, you will need a Connection string in the form of + address:port. For example, 10.0.1.50:10000 + +2. For Enterprise Core+, you will need a JSON URL that references your + Core+ installation. For example + https://10.0.1.50:8123/iris/connection.json diff --git a/csharp/ExcelAddIn/StateManager.cs b/csharp/ExcelAddIn/StateManager.cs new file mode 100644 index 00000000000..9faa64a24ae --- /dev/null +++ b/csharp/ExcelAddIn/StateManager.cs @@ -0,0 +1,178 @@ +using System.Diagnostics; +using System.Net; +using Deephaven.DeephavenClient; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Providers; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn; + +public class StateManager { + public readonly WorkerThread WorkerThread = WorkerThread.Create(); + private readonly Dictionary _credentialsProviders = new(); + private readonly Dictionary _sessionProviders = new(); + private readonly Dictionary _persistentQueryProviders = new(); + private readonly Dictionary _tableProviders = new(); + private readonly ObserverContainer> _credentialsPopulationObservers = new(); + private readonly ObserverContainer _defaultEndpointSelectionObservers = new(); + + private EndpointId? _defaultEndpointId = null; + + public IDisposable SubscribeToCredentialsPopulation(IObserver> observer) { + WorkerThread.EnqueueOrRun(() => { + _credentialsPopulationObservers.Add(observer, out _); + + // Give this observer the current set of endpoint ids. + var keys = _credentialsProviders.Keys.ToArray(); + foreach (var endpointId in keys) { + observer.OnNext(AddOrRemove.OfAdd(endpointId)); + } + }); + + return WorkerThread.EnqueueOrRunWhenDisposed( + () => _credentialsPopulationObservers.Remove(observer, out _)); + } + + public IDisposable SubscribeToDefaultEndpointSelection(IObserver observer) { + WorkerThread.EnqueueOrRun(() => { + _defaultEndpointSelectionObservers.Add(observer, out _); + observer.OnNext(_defaultEndpointId); + }); + + return WorkerThread.EnqueueOrRunWhenDisposed( + () => _defaultEndpointSelectionObservers.Remove(observer, out _)); + } + + /// + /// The major difference between the credentials providers and the other providers + /// is that the credential providers don't remove themselves from the map + /// upon the last dispose of the subscriber. That is, they hang around until we + /// manually remove them. + /// + public IDisposable SubscribeToCredentials(EndpointId endpointId, + IObserver> observer) { + IDisposable? disposer = null; + LookupOrCreateCredentialsProvider(endpointId, + cp => disposer = cp.Subscribe(observer)); + + return WorkerThread.EnqueueOrRunWhenDisposed(() => + Utility.Exchange(ref disposer, null)?.Dispose()); + } + + public void SetCredentials(CredentialsBase credentials) { + LookupOrCreateCredentialsProvider(credentials.Id, + cp => cp.SetCredentials(credentials)); + } + + public void Reconnect(EndpointId id) { + // Quick-and-dirty trick for reconnect is to re-send the credentials to the observers. + LookupOrCreateCredentialsProvider(id, cp => cp.Resend()); + } + + public void TryDeleteCredentials(EndpointId id, Action onSuccess, Action onFailure) { + if (WorkerThread.EnqueueOrNop(() => TryDeleteCredentials(id, onSuccess, onFailure))) { + return; + } + + if (!_credentialsProviders.TryGetValue(id, out var cp)) { + onFailure($"{id} unknown"); + return; + } + + if (cp.ObserverCountUnsafe != 0) { + onFailure($"{id} is still active"); + return; + } + + if (id.Equals(_defaultEndpointId)) { + SetDefaultEndpointId(null); + } + + _credentialsProviders.Remove(id); + _credentialsPopulationObservers.OnNext(AddOrRemove.OfRemove(id)); + onSuccess(); + } + + private void LookupOrCreateCredentialsProvider(EndpointId endpointId, + Action action) { + if (WorkerThread.EnqueueOrNop(() => LookupOrCreateCredentialsProvider(endpointId, action))) { + return; + } + if (!_credentialsProviders.TryGetValue(endpointId, out var cp)) { + cp = new CredentialsProvider(this); + _credentialsProviders.Add(endpointId, cp); + cp.Init(); + _credentialsPopulationObservers.OnNext(AddOrRemove.OfAdd(endpointId)); + } + + action(cp); + } + + public IDisposable SubscribeToSession(EndpointId endpointId, + IObserver> observer) { + IDisposable? disposer = null; + WorkerThread.EnqueueOrRun(() => { + if (!_sessionProviders.TryGetValue(endpointId, out var sp)) { + sp = new SessionProvider(this, endpointId, () => _sessionProviders.Remove(endpointId)); + _sessionProviders.Add(endpointId, sp); + sp.Init(); + } + disposer = sp.Subscribe(observer); + }); + + return WorkerThread.EnqueueOrRunWhenDisposed(() => + Utility.Exchange(ref disposer, null)?.Dispose()); + } + + public IDisposable SubscribeToPersistentQuery(EndpointId endpointId, PersistentQueryId? pqId, + IObserver> observer) { + + IDisposable? disposer = null; + WorkerThread.EnqueueOrRun(() => { + var key = new PersistentQueryKey(endpointId, pqId); + if (!_persistentQueryProviders.TryGetValue(key, out var pqp)) { + pqp = new PersistentQueryProvider(this, endpointId, pqId, + () => _persistentQueryProviders.Remove(key)); + _persistentQueryProviders.Add(key, pqp); + pqp.Init(); + } + disposer = pqp.Subscribe(observer); + }); + + return WorkerThread.EnqueueOrRunWhenDisposed( + () => Utility.Exchange(ref disposer, null)?.Dispose()); + } + + public IDisposable SubscribeToTable(TableQuad key, IObserver> observer) { + IDisposable? disposer = null; + WorkerThread.EnqueueOrRun(() => { + if (!_tableProviders.TryGetValue(key, out var tp)) { + Action onDispose = () => _tableProviders.Remove(key); + if (key.EndpointId == null) { + tp = new DefaultEndpointTableProvider(this, key.PersistentQueryId, key.TableName, key.Condition, + onDispose); + } else if (key.Condition.Length != 0) { + tp = new FilteredTableProvider(this, key.EndpointId, key.PersistentQueryId, key.TableName, + key.Condition, onDispose); + } else { + tp = new TableProvider(this, key.EndpointId, key.PersistentQueryId, key.TableName, onDispose); + } + _tableProviders.Add(key, tp); + tp.Init(); + } + disposer = tp.Subscribe(observer); + }); + + return WorkerThread.EnqueueOrRunWhenDisposed( + () => Utility.Exchange(ref disposer, null)?.Dispose()); + } + + public void SetDefaultEndpointId(EndpointId? defaultEndpointId) { + if (WorkerThread.EnqueueOrNop(() => SetDefaultEndpointId(defaultEndpointId))) { + return; + } + + _defaultEndpointId = defaultEndpointId; + _defaultEndpointSelectionObservers.OnNext(_defaultEndpointId); + } +} diff --git a/csharp/ExcelAddIn/exceldna/ExcelDnaHelpers.cs b/csharp/ExcelAddIn/exceldna/ExcelDnaHelpers.cs new file mode 100644 index 00000000000..4ceb44805a3 --- /dev/null +++ b/csharp/ExcelAddIn/exceldna/ExcelDnaHelpers.cs @@ -0,0 +1,42 @@ +using Deephaven.ExcelAddIn.Util; +using ExcelDna.Integration; + +namespace Deephaven.ExcelAddIn.ExcelDna; + +internal class ExcelDnaHelpers { + public static bool TryInterpretAs(object value, T defaultValue, out T result) { + result = defaultValue; + if (value is ExcelMissing) { + return true; + } + + if (value is T tValue) { + result = tValue; + return true; + } + + return false; + } + + public static IObserver> WrapExcelObserver(IExcelObserver inner) { + return new ExcelObserverWrapper(inner); + } + + private class ExcelObserverWrapper(IExcelObserver inner) : IObserver> { + public void OnNext(StatusOr sov) { + if (!sov.GetValueOrStatus(out var value, out var status)) { + // Reformat the status text as an object[,] 2D array so Excel renders it as 1x1 "table". + value = new object[,] { { status } }; + } + inner.OnNext(value); + } + + public void OnCompleted() { + inner.OnCompleted(); + } + + public void OnError(Exception error) { + inner.OnError(error); + } + } +} diff --git a/csharp/ExcelAddIn/factories/ConnectionManagerDialogFactory.cs b/csharp/ExcelAddIn/factories/ConnectionManagerDialogFactory.cs new file mode 100644 index 00000000000..72441c93323 --- /dev/null +++ b/csharp/ExcelAddIn/factories/ConnectionManagerDialogFactory.cs @@ -0,0 +1,17 @@ +using Deephaven.ExcelAddIn.Managers; +using Deephaven.ExcelAddIn.Util; +using Deephaven.ExcelAddIn.Views; + +namespace Deephaven.ExcelAddIn.Factories; + +internal static class ConnectionManagerDialogFactory { + public static void CreateAndShow(StateManager stateManager) { + Utility.RunInBackground(() => { + var cmDialog = new ConnectionManagerDialog(); + var dm = ConnectionManagerDialogManager.Create(stateManager, cmDialog); + cmDialog.Closed += (_, _) => dm.Dispose(); + // Blocks forever (in this private thread) + cmDialog.ShowDialog(); + }); + } +} diff --git a/csharp/ExcelAddIn/factories/CredentialsDialogFactory.cs b/csharp/ExcelAddIn/factories/CredentialsDialogFactory.cs new file mode 100644 index 00000000000..4c1e2694be7 --- /dev/null +++ b/csharp/ExcelAddIn/factories/CredentialsDialogFactory.cs @@ -0,0 +1,138 @@ +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; +using Deephaven.ExcelAddIn.ViewModels; +using ExcelAddIn.views; +using static System.Windows.Forms.AxHost; + +namespace Deephaven.ExcelAddIn.Factories; + +internal static class CredentialsDialogFactory { + public static void CreateAndShow(StateManager stateManager, CredentialsDialogViewModel cvm, + EndpointId? whitelistId) { + Utility.RunInBackground(() => { + var cd = new CredentialsDialog(cvm); + var state = new CredentialsDialogState(stateManager, cd, cvm, whitelistId); + + + cd.OnSetCredentialsButtonClicked += state.OnSetCredentials; + cd.OnTestCredentialsButtonClicked += state.OnTestCredentials; + + cd.Closed += (_, _) => state.Dispose(); + // Blocks forever (in this private thread) + cd.ShowDialog(); + }); + } +} + +internal class CredentialsDialogState : IObserver>, IDisposable { + private readonly StateManager _stateManager; + private readonly CredentialsDialog _credentialsDialog; + private readonly CredentialsDialogViewModel _cvm; + private readonly EndpointId? _whitelistId; + private IDisposable? _disposer; + private readonly object _sync = new(); + private readonly HashSet _knownIds = new(); + private readonly VersionTracker _versionTracker = new(); + + public CredentialsDialogState( + StateManager stateManager, + CredentialsDialog credentialsDialog, + CredentialsDialogViewModel cvm, + EndpointId? whitelistId) { + _stateManager = stateManager; + _credentialsDialog = credentialsDialog; + _cvm = cvm; + _whitelistId = whitelistId; + _disposer = stateManager.SubscribeToCredentialsPopulation(this); + } + + public void Dispose() { + Utility.Exchange(ref _disposer, null)?.Dispose(); + } + + public void OnCompleted() { + throw new NotImplementedException(); + } + + public void OnError(Exception error) { + throw new NotImplementedException(); + } + + public void OnNext(AddOrRemove value) { + lock (_sync) { + if (value.IsAdd) { + _knownIds.Add(value.Value); + } else { + _knownIds.Remove(value.Value); + } + } + } + + public void OnSetCredentials() { + if (!_cvm.TryMakeCredentials(out var newCreds, out var error)) { + ShowMessageBox(error); + return; + } + + bool isKnown; + lock (_sync) { + isKnown = _knownIds.Contains(newCreds.Id); + } + + if (isKnown && !newCreds.Id.Equals(_whitelistId)) { + const string caption = "Modify existing connection?"; + var text = $"Are you sure you want to modify connection \"{newCreds.Id}\""; + var dhm = new DeephavenMessageBox(caption, text, true); + var dialogResult = dhm.ShowDialog(_credentialsDialog); + if (dialogResult != DialogResult.OK) { + return; + } + } + + _stateManager.SetCredentials(newCreds); + if (_cvm.IsDefault) { + _stateManager.SetDefaultEndpointId(newCreds.Id); + } + + _credentialsDialog!.Close(); + } + + public void OnTestCredentials() { + if (!_cvm.TryMakeCredentials(out var newCreds, out var error)) { + ShowMessageBox(error); + return; + } + + _credentialsDialog!.SetTestResultsBox("Checking credentials"); + // Check credentials on its own thread + Utility.RunInBackground(() => TestCredentialsThreadFunc(newCreds)); + } + + private void TestCredentialsThreadFunc(CredentialsBase creds) { + var latestCookie = _versionTracker.SetNewVersion(); + + var state = "OK"; + try { + // This operation might take some time. + var temp = SessionBaseFactory.Create(creds, _stateManager.WorkerThread); + temp.Dispose(); + } catch (Exception ex) { + state = ex.Message; + } + + if (!latestCookie.IsCurrent) { + // Our results are moot. Dispose of them. + return; + } + + // Our results are valid. Keep them and tell everyone about it. + _credentialsDialog!.SetTestResultsBox(state); + } + + private void ShowMessageBox(string error) { + _credentialsDialog.Invoke(() => { + var dhm = new DeephavenMessageBox("Please provide missing fields", error, false); + dhm.ShowDialog(_credentialsDialog); + }); + } +} diff --git a/csharp/ExcelAddIn/factories/SessionBaseFactory.cs b/csharp/ExcelAddIn/factories/SessionBaseFactory.cs new file mode 100644 index 00000000000..44886d6a16d --- /dev/null +++ b/csharp/ExcelAddIn/factories/SessionBaseFactory.cs @@ -0,0 +1,34 @@ +using Deephaven.DeephavenClient; +using Deephaven.DheClient.Session; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Factories; + +internal static class SessionBaseFactory { + public static SessionBase Create(CredentialsBase credentials, WorkerThread workerThread) { + return credentials.AcceptVisitor( + core => { + var options = new ClientOptions(); + options.SetSessionType(core.SessionTypeIsPython ? "python" : "groovy"); + var client = Client.Connect(core.ConnectionString, options); + return new CoreSession(client); + }, + + corePlus => { + var handler = new HttpClientHandler(); + if (!corePlus.ValidateCertificate) { + handler.ClientCertificateOptions = ClientCertificateOption.Manual; + handler.ServerCertificateCustomValidationCallback = (_, _, _, _) => true; + } + var hc = new HttpClient(handler); + var json = hc.GetStringAsync(corePlus.JsonUrl).Result; + var session = SessionManager.FromJson("Deephaven Excel", json); + if (!session.PasswordAuthentication(corePlus.User, corePlus.Password, corePlus.OperateAs)) { + throw new Exception("Authentication failed"); + } + + return new CorePlusSession(session, workerThread); + }); + } +} diff --git a/csharp/ExcelAddIn/managers/ConnectionManagerDialogManager.cs b/csharp/ExcelAddIn/managers/ConnectionManagerDialogManager.cs new file mode 100644 index 00000000000..5ebcbc958f6 --- /dev/null +++ b/csharp/ExcelAddIn/managers/ConnectionManagerDialogManager.cs @@ -0,0 +1,197 @@ +using System.Collections.Concurrent; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Viewmodels; +using Deephaven.ExcelAddIn.Views; +using Deephaven.ExcelAddIn.Util; +using Deephaven.ExcelAddIn.Factories; +using Deephaven.ExcelAddIn.ViewModels; +using ExcelAddIn.views; + +namespace Deephaven.ExcelAddIn.Managers; + +internal class ConnectionManagerDialogManager : IObserver>, IDisposable { + // + // ConnectionManagerDialog cmDialog, + // ConcurrentDictionary rowToManager, + // StateManager stateManager) + public static ConnectionManagerDialogManager Create(StateManager stateManager, + ConnectionManagerDialog cmDialog) { + var result = new ConnectionManagerDialogManager(stateManager, cmDialog); + cmDialog.OnNewButtonClicked += result.OnNewButtonClicked; + cmDialog.OnDeleteButtonClicked += result.OnDeleteButtonClicked; + cmDialog.OnReconnectButtonClicked += result.OnReconnectButtonClicked; + cmDialog.OnMakeDefaultButtonClicked += result.OnMakeDefaultButtonClicked; + cmDialog.OnEditButtonClicked += result.OnEditButtonClicked; + + var disp = stateManager.SubscribeToCredentialsPopulation(result); + result._disposables.Add(disp); + return result; + } + + private readonly StateManager _stateManager; + private readonly WorkerThread _workerThread; + private readonly ConnectionManagerDialog _cmDialog; + private readonly Dictionary _idToRow = new(); + private readonly Dictionary _rowToManager = new(); + private readonly List _disposables = new(); + + public ConnectionManagerDialogManager(StateManager stateManager, ConnectionManagerDialog cmDialog) { + _stateManager = stateManager; + _workerThread = stateManager.WorkerThread; + _cmDialog = cmDialog; + } + + public void OnNext(AddOrRemove aor) { + if (_workerThread.EnqueueOrNop(() => OnNext(aor))) { + return; + } + + if (aor.IsAdd) { + var endpointId = aor.Value; + var row = new ConnectionManagerDialogRow(endpointId.Id); + var statusRowManager = ConnectionManagerDialogRowManager.Create(row, endpointId, _stateManager); + _rowToManager.Add(row, statusRowManager); + _idToRow.Add(endpointId, row); + _disposables.Add(statusRowManager); + + _cmDialog.AddRow(row); + return; + } + + // Remove! + if (!_idToRow.Remove(aor.Value, out var rowToDelete) || + !_rowToManager.Remove(rowToDelete, out var rowManager)) { + return; + } + + _cmDialog.RemoveRow(rowToDelete); + rowManager.Dispose(); + } + + public void Dispose() { + if (_workerThread.EnqueueOrNop(Dispose)) { + return; + } + + var temp = _disposables.ToArray(); + _disposables.Clear(); + foreach (var disposable in temp) { + disposable.Dispose(); + } + } + + public void OnCompleted() { + // TODO(kosak) + throw new NotImplementedException(); + } + + public void OnError(Exception error) { + // TODO(kosak) + throw new NotImplementedException(); + } + + void OnNewButtonClicked() { + var cvm = CredentialsDialogViewModel.OfEmpty(); + CredentialsDialogFactory.CreateAndShow(_stateManager, cvm, null); + } + + private class FailureCollector { + private readonly ConnectionManagerDialog _cmDialog; + private readonly object _sync = new(); + private int _rowsLeft = 0; + private readonly List _failures = new(); + + public FailureCollector(ConnectionManagerDialog cmDialog, int rowsLeft) { + _cmDialog = cmDialog; + _rowsLeft = rowsLeft; + } + + public void OnFailure(EndpointId id, string reason) { + lock (_sync) { + _failures.Add(reason); + } + + FinalSteps(); + } + + public void OnSuccess(EndpointId id) { + FinalSteps(); + } + + private void FinalSteps() { + string text; + lock (_sync) { + --_rowsLeft; + if (_rowsLeft > 0 || _failures.Count == 0) { + return; + } + + text = string.Join(Environment.NewLine, _failures); + } + + const string caption = "Couldn't delete some selections"; + _cmDialog.Invoke(() => { + var mbox = new DeephavenMessageBox(caption, text, false); + mbox.ShowDialog(_cmDialog); + }); + } + } + + void OnDeleteButtonClicked(ConnectionManagerDialogRow[] rows) { + if (_workerThread.EnqueueOrNop(() => OnDeleteButtonClicked(rows))) { + return; + } + + var fc = new FailureCollector(_cmDialog, rows.Length); + foreach (var row in rows) { + if (!_rowToManager.TryGetValue(row, out var manager)) { + continue; + } + manager.DoDelete(fc.OnSuccess, fc.OnFailure); + } + } + + void OnReconnectButtonClicked(ConnectionManagerDialogRow[] rows) { + if (_workerThread.EnqueueOrNop(() => OnReconnectButtonClicked(rows))) { + return; + } + + foreach (var row in rows) { + if (!_rowToManager.TryGetValue(row, out var manager)) { + continue; + } + manager.DoReconnect(); + } + } + + void OnMakeDefaultButtonClicked(ConnectionManagerDialogRow[] rows) { + if (_workerThread.EnqueueOrNop(() => OnMakeDefaultButtonClicked(rows))) { + return; + } + + // Make the last selected row the default + if (rows.Length == 0) { + return; + } + + var row = rows[^1]; + if (!_rowToManager.TryGetValue(row, out var manager)) { + return; + } + + manager.DoSetAsDefault(); + } + + void OnEditButtonClicked(ConnectionManagerDialogRow[] rows) { + if (_workerThread.EnqueueOrNop(() => OnEditButtonClicked(rows))) { + return; + } + + foreach (var row in rows) { + if (!_rowToManager.TryGetValue(row, out var manager)) { + continue; + } + manager.DoEdit(); + } + } +} diff --git a/csharp/ExcelAddIn/managers/ConnectionManagerDialogRowManager.cs b/csharp/ExcelAddIn/managers/ConnectionManagerDialogRowManager.cs new file mode 100644 index 00000000000..1b98da60e65 --- /dev/null +++ b/csharp/ExcelAddIn/managers/ConnectionManagerDialogRowManager.cs @@ -0,0 +1,132 @@ +using Deephaven.ExcelAddIn.Factories; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; +using Deephaven.ExcelAddIn.Viewmodels; +using Deephaven.ExcelAddIn.ViewModels; + +namespace Deephaven.ExcelAddIn.Managers; + +public sealed class ConnectionManagerDialogRowManager : + IObserver>, + IObserver>, + IObserver, + IDisposable { + + public static ConnectionManagerDialogRowManager Create(ConnectionManagerDialogRow row, + EndpointId endpointId, StateManager stateManager) { + var result = new ConnectionManagerDialogRowManager(row, endpointId, stateManager); + result.Resubscribe(); + return result; + } + + private readonly ConnectionManagerDialogRow _row; + private readonly EndpointId _endpointId; + private readonly StateManager _stateManager; + private readonly WorkerThread _workerThread; + private readonly List _disposables = new(); + + private ConnectionManagerDialogRowManager(ConnectionManagerDialogRow row, EndpointId endpointId, + StateManager stateManager) { + _row = row; + _endpointId = endpointId; + _stateManager = stateManager; + _workerThread = stateManager.WorkerThread; + } + + public void Dispose() { + Unsubscribe(); + } + + private void Resubscribe() { + if (_workerThread.EnqueueOrNop(Resubscribe)) { + return; + } + + if (_disposables.Count != 0) { + throw new Exception("State error: already subscribed"); + } + // We watch for session and credential state changes in our ID + var d1 = _stateManager.SubscribeToSession(_endpointId, this); + var d2 = _stateManager.SubscribeToCredentials(_endpointId, this); + var d3 = _stateManager.SubscribeToDefaultEndpointSelection(this); + _disposables.AddRange(new[] { d1, d2, d3 }); + } + + private void Unsubscribe() { + if (_workerThread.EnqueueOrNop(Unsubscribe)) { + return; + } + var temp = _disposables.ToArray(); + _disposables.Clear(); + + foreach (var disposable in temp) { + disposable.Dispose(); + } + } + + public void OnNext(StatusOr value) { + _row.SetCredentialsSynced(value); + } + + public void OnNext(StatusOr value) { + _row.SetSessionSynced(value); + } + + public void OnNext(EndpointId? value) { + _row.SetDefaultEndpointIdSynced(value); + } + + public void DoEdit() { + var creds = _row.GetCredentialsSynced(); + // If we have valid credentials, then make a populated viewmodel. + // If we don't, then make an empty viewmodel with only Id populated. + var cvm = creds.AcceptVisitor( + crs => CredentialsDialogViewModel.OfIdAndCredentials(_endpointId.Id, crs), + _ => CredentialsDialogViewModel.OfIdButOtherwiseEmpty(_endpointId.Id)); + CredentialsDialogFactory.CreateAndShow(_stateManager, cvm, _endpointId); + } + + public void DoDelete(Action onSuccess, Action onFailure) { + if (_workerThread.EnqueueOrNop(() => DoDelete(onSuccess, onFailure))) { + return; + } + + // Strategy: + // 1. Unsubscribe to everything + // 2. If it turns out that we were the last subscriber to the credentials, then great, the + // delete can proceed. + // 3. If the credentials we are deleting are the default credentials, then unset default credentials + // 4. Otherwise (there is some other subscriber to the credentials), then the delete operation + // should be denied. In that case we restore our state by resubscribing to everything. + Unsubscribe(); + _stateManager.TryDeleteCredentials(_endpointId, + () => onSuccess(_endpointId), + reason => { + Resubscribe(); + onFailure(_endpointId, reason); + }); + } + + public void DoReconnect() { + _stateManager.Reconnect(_endpointId); + } + + public void DoSetAsDefault() { + // If the connection is already the default, do nothing. + if (_row.IsDefault) { + return; + } + + _stateManager.SetDefaultEndpointId(_endpointId); + } + + public void OnCompleted() { + // TODO(kosak) + throw new NotImplementedException(); + } + + public void OnError(Exception error) { + // TODO(kosak) + throw new NotImplementedException(); + } +} diff --git a/csharp/ExcelAddIn/models/Credentials.cs b/csharp/ExcelAddIn/models/Credentials.cs new file mode 100644 index 00000000000..0a2ffdc6b60 --- /dev/null +++ b/csharp/ExcelAddIn/models/Credentials.cs @@ -0,0 +1,42 @@ +namespace Deephaven.ExcelAddIn.Models; + +public abstract class CredentialsBase(EndpointId id) { + public readonly EndpointId Id = id; + + public static CredentialsBase OfCore(EndpointId id, string connectionString, bool sessionTypeIsPython) { + return new CoreCredentials(id, connectionString, sessionTypeIsPython); + } + + public static CredentialsBase OfCorePlus(EndpointId id, string jsonUrl, string userId, + string password, string operateAs, bool validateCertificate) { + return new CorePlusCredentials(id, jsonUrl, userId, password, operateAs, validateCertificate); + } + + public abstract T AcceptVisitor(Func ofCore, + Func ofCorePlus); +} + +public sealed class CoreCredentials( + EndpointId id, + string connectionString, + bool sessionTypeIsPython) : CredentialsBase(id) { + public readonly string ConnectionString = connectionString; + public readonly bool SessionTypeIsPython = sessionTypeIsPython; + + public override T AcceptVisitor(Func ofCore, Func ofCorePlus) { + return ofCore(this); + } +} + +public sealed class CorePlusCredentials(EndpointId id, string jsonUrl, string user, string password, + string operateAs, bool validateCertificate) : CredentialsBase(id) { + public readonly string JsonUrl = jsonUrl; + public readonly string User = user; + public readonly string Password = password; + public readonly string OperateAs = operateAs; + public readonly bool ValidateCertificate = validateCertificate; + + public override T AcceptVisitor(Func ofCore, Func ofCorePlus) { + return ofCorePlus(this); + } +} diff --git a/csharp/ExcelAddIn/models/Session.cs b/csharp/ExcelAddIn/models/Session.cs new file mode 100644 index 00000000000..5f3d34eb02a --- /dev/null +++ b/csharp/ExcelAddIn/models/Session.cs @@ -0,0 +1,79 @@ +using Deephaven.DeephavenClient; +using Deephaven.DheClient.Session; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Models; + +/// +/// A "Session" is an abstraction meant to represent a Core or Core+ "session". +/// For Core, this means having a valid Client. +/// For Core+, this means having a SessionManager, through which you can subscribe to PQs and get Clients. +/// +public abstract class SessionBase : IDisposable { + /// + /// This is meant to act like a Visitor pattern with lambdas. + /// + public abstract T Visit(Func onCore, Func onCorePlus); + + public abstract void Dispose(); +} + +public sealed class CoreSession(Client client) : SessionBase { + private Client? _client = client; + + public override T Visit(Func onCore, Func onCorePlus) { + return onCore(this); + } + + public override void Dispose() { + var temp = Utility.Exchange(ref _client, null); + if (temp == null) { + return; + } + + // Do the actual dispose work on a helper thread. + Utility.RunInBackground(temp.Dispose); + } + + public Client Client { + get { + if (_client == null) { + throw new Exception("Object is disposed"); + } + + return _client; + } + } +} + +public sealed class CorePlusSession(SessionManager sessionManager, WorkerThread workerThread) : SessionBase { + private SessionManager? _sessionManager = sessionManager; + + public override T Visit(Func onCore, Func onCorePlus) { + return onCorePlus(this); + } + + public override void Dispose() { + if (workerThread.EnqueueOrNop(Dispose)) { + return; + } + + var temp = Utility.Exchange(ref _sessionManager, null); + if (temp == null) { + return; + } + + // Do the actual dispose work on a helper thread. + Utility.RunInBackground(temp.Dispose); + } + + public SessionManager SessionManager { + get { + if (_sessionManager == null) { + throw new Exception("Object is disposed"); + } + + return _sessionManager; + } + } +} diff --git a/csharp/ExcelAddIn/models/SimpleModels.cs b/csharp/ExcelAddIn/models/SimpleModels.cs new file mode 100644 index 00000000000..4593e58fef9 --- /dev/null +++ b/csharp/ExcelAddIn/models/SimpleModels.cs @@ -0,0 +1,17 @@ +namespace Deephaven.ExcelAddIn.Models; + +public record AddOrRemove(bool IsAdd, T Value) { + public static AddOrRemove OfAdd(T value) { + return new AddOrRemove(true, value); + } + + public static AddOrRemove OfRemove(T value) { + return new AddOrRemove(false, value); + } +} + +public record EndpointId(string Id) { + public override string ToString() => Id; +} + +public record PersistentQueryId(string Id); diff --git a/csharp/ExcelAddIn/models/TableTriple.cs b/csharp/ExcelAddIn/models/TableTriple.cs new file mode 100644 index 00000000000..89a298dd350 --- /dev/null +++ b/csharp/ExcelAddIn/models/TableTriple.cs @@ -0,0 +1,49 @@ +namespace Deephaven.ExcelAddIn.Models; + +public record PersistentQueryKey( + EndpointId EndpointId, + PersistentQueryId? PersistentQueryId) { +} + +public record TableTriple( + EndpointId? EndpointId, + PersistentQueryId? PersistentQueryId, + string TableName) { + + public static bool TryParse(string text, out TableTriple result, out string errorText) { + // Accepts strings of the following form + // 1. "table" (becomes null, null, "table") + // 2. "endpoint:table" (becomes endpoint, null, table) + // 3. "pq/table" (becomes null, pq, table) + // 4. "endpoint:pq/table" (becomes endpoint, pq, table) + EndpointId? epId = null; + PersistentQueryId? pqid = null; + var tableName = ""; + var colonIndex = text.IndexOf(':'); + if (colonIndex > 0) { + // cases 2 and 4: pull out the endpointId, and then reduce to cases 1 and 3 + epId = new EndpointId(text[..colonIndex]); + text = text[(colonIndex + 1)..]; + } + + var slashIndex = text.IndexOf('/'); + if (slashIndex > 0) { + // case 3: pull out the slash, and reduce to case 1 + pqid = new PersistentQueryId(text[..slashIndex]); + text = text[(slashIndex + 1)..]; + } + + tableName = text; + result = new TableTriple(epId, pqid, tableName); + errorText = ""; + // This version never fails to parse, but we leave open the option in our API to do so. + return true; + } +} + +public record TableQuad( + EndpointId? EndpointId, + PersistentQueryId? PersistentQueryId, + string TableName, + string Condition) { +} diff --git a/csharp/ExcelAddIn/operations/SnapshotOperation.cs b/csharp/ExcelAddIn/operations/SnapshotOperation.cs new file mode 100644 index 00000000000..b9a299b8309 --- /dev/null +++ b/csharp/ExcelAddIn/operations/SnapshotOperation.cs @@ -0,0 +1,76 @@ +using Deephaven.DeephavenClient; +using Deephaven.DeephavenClient.ExcelAddIn.Util; +using Deephaven.ExcelAddIn.ExcelDna; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; +using ExcelDna.Integration; + +namespace Deephaven.ExcelAddIn.Operations; + +internal class SnapshotOperation : IExcelObservable, IObserver> { + private readonly TableQuad _tableQuad; + private readonly bool _wantHeaders; + private readonly StateManager _stateManager; + private readonly ObserverContainer> _observers = new(); + private readonly WorkerThread _workerThread; + private IDisposable? _filteredTableDisposer = null; + + public SnapshotOperation(TableQuad tableQuad, bool wantHeaders, StateManager stateManager) { + _tableQuad = tableQuad; + _wantHeaders = wantHeaders; + _stateManager = stateManager; + // Convenience + _workerThread = _stateManager.WorkerThread; + } + + public IDisposable Subscribe(IExcelObserver observer) { + var wrappedObserver = ExcelDnaHelpers.WrapExcelObserver(observer); + _workerThread.EnqueueOrRun(() => { + _observers.Add(wrappedObserver, out var isFirst); + + if (isFirst) { + _filteredTableDisposer = _stateManager.SubscribeToTable(_tableQuad, this); + } + }); + + return _workerThread.EnqueueOrRunWhenDisposed(() => { + _observers.Remove(wrappedObserver, out var wasLast); + if (!wasLast) { + return; + } + + Utility.Exchange(ref _filteredTableDisposer, null)?.Dispose(); + }); + } + + public void OnNext(StatusOr tableHandle) { + if (_workerThread.EnqueueOrNop(() => OnNext(tableHandle))) { + return; + } + + if (!tableHandle.GetValueOrStatus(out var th, out var status)) { + _observers.SendStatus(status); + return; + } + + _observers.SendStatus($"Snapshotting \"{_tableQuad.TableName}\""); + + try { + using var ct = th.ToClientTable(); + var rendered = Renderer.Render(ct, _wantHeaders); + _observers.SendValue(rendered); + } catch (Exception ex) { + _observers.SendStatus(ex.Message); + } + } + + void IObserver>.OnCompleted() { + // TODO(kosak): TODO + throw new NotImplementedException(); + } + + void IObserver>.OnError(Exception error) { + // TODO(kosak): TODO + throw new NotImplementedException(); + } +} diff --git a/csharp/ExcelAddIn/operations/SubscribeOperation.cs b/csharp/ExcelAddIn/operations/SubscribeOperation.cs new file mode 100644 index 00000000000..f2478d67a53 --- /dev/null +++ b/csharp/ExcelAddIn/operations/SubscribeOperation.cs @@ -0,0 +1,118 @@ +using Deephaven.DeephavenClient; +using Deephaven.DeephavenClient.ExcelAddIn.Util; +using Deephaven.ExcelAddIn.ExcelDna; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; +using ExcelDna.Integration; + +namespace Deephaven.ExcelAddIn.Operations; + +internal class SubscribeOperation : IExcelObservable, IObserver> { + private readonly TableQuad _tableQuad; + private readonly bool _wantHeaders; + private readonly StateManager _stateManager; + private readonly ObserverContainer> _observers = new(); + private readonly WorkerThread _workerThread; + private IDisposable? _tableDisposer = null; + private TableHandle? _currentTableHandle = null; + private SubscriptionHandle? _currentSubHandle = null; + + public SubscribeOperation(TableQuad tableQuad, bool wantHeaders, StateManager stateManager) { + _tableQuad = tableQuad; + _wantHeaders = wantHeaders; + _stateManager = stateManager; + // Convenience + _workerThread = _stateManager.WorkerThread; + } + + public IDisposable Subscribe(IExcelObserver observer) { + var wrappedObserver = ExcelDnaHelpers.WrapExcelObserver(observer); + _workerThread.EnqueueOrRun(() => { + _observers.Add(wrappedObserver, out var isFirst); + + if (isFirst) { + _tableDisposer = _stateManager.SubscribeToTable(_tableQuad, this); + } + }); + + return ActionAsDisposable.Create(() => { + _workerThread.EnqueueOrRun(() => { + _observers.Remove(wrappedObserver, out var wasLast); + if (!wasLast) { + return; + } + + Utility.Exchange(ref _tableDisposer, null)?.Dispose(); + }); + }); + } + + public void OnNext(StatusOr soth) { + if (_workerThread.EnqueueOrNop(() => OnNext(soth))) { + return; + } + + // First tear down old state + if (_currentTableHandle != null) { + if (_currentSubHandle != null) { + _currentTableHandle.Unsubscribe(_currentSubHandle!); + _currentSubHandle!.Dispose(); + _currentSubHandle = null; + } + + _currentTableHandle = null; + } + + if (!soth.GetValueOrStatus(out var tableHandle, out var status)) { + _observers.SendStatus(status); + return; + } + + _observers.SendStatus($"Subscribing to \"{_tableQuad.TableName}\""); + + _currentTableHandle = tableHandle; + _currentSubHandle = _currentTableHandle.Subscribe(new MyTickingCallback(_observers, _wantHeaders)); + + try { + using var ct = tableHandle.ToClientTable(); + var result = Renderer.Render(ct, _wantHeaders); + _observers.SendValue(result); + } catch (Exception ex) { + _observers.SendStatus(ex.Message); + } + } + + void IObserver>.OnCompleted() { + // TODO(kosak): TODO + throw new NotImplementedException(); + } + + void IObserver>.OnError(Exception error) { + // TODO(kosak): TODO + throw new NotImplementedException(); + } + + private class MyTickingCallback : ITickingCallback { + private readonly ObserverContainer> _observers; + private readonly bool _wantHeaders; + + public MyTickingCallback(ObserverContainer> observers, + bool wantHeaders) { + _observers = observers; + _wantHeaders = wantHeaders; + } + + public void OnTick(TickingUpdate update) { + try { + var results = Renderer.Render(update.Current, _wantHeaders); + _observers.SendValue(results); + } catch (Exception e) { + _observers.SendStatus(e.Message); + } + } + + public void OnFailure(string errorText) { + _observers.SendStatus(errorText); + } + } +} diff --git a/csharp/ExcelAddIn/providers/CredentialsProvider.cs b/csharp/ExcelAddIn/providers/CredentialsProvider.cs new file mode 100644 index 00000000000..9a2d8c2deee --- /dev/null +++ b/csharp/ExcelAddIn/providers/CredentialsProvider.cs @@ -0,0 +1,37 @@ +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Providers; + +internal class CredentialsProvider : IObservable> { + private readonly WorkerThread _workerThread; + private readonly ObserverContainer> _observers = new(); + private StatusOr _credentials = StatusOr.OfStatus("[No Credentials]"); + + public CredentialsProvider(StateManager stateManager) { + _workerThread = stateManager.WorkerThread; + } + + public void Init() { + // Do nothing + } + + public IDisposable Subscribe(IObserver> observer) { + _workerThread.EnqueueOrRun(() => { + _observers.Add(observer, out _); + observer.OnNext(_credentials); + }); + + return _workerThread.EnqueueOrRunWhenDisposed(() => _observers.Remove(observer, out _)); + } + + public void SetCredentials(CredentialsBase newCredentials) { + _observers.SetAndSendValue(ref _credentials, newCredentials); + } + + public void Resend() { + _observers.OnNext(_credentials); + } + + public int ObserverCountUnsafe => _observers.Count; +} diff --git a/csharp/ExcelAddIn/providers/DefaultEndpointTableProvider.cs b/csharp/ExcelAddIn/providers/DefaultEndpointTableProvider.cs new file mode 100644 index 00000000000..26d1dab5f35 --- /dev/null +++ b/csharp/ExcelAddIn/providers/DefaultEndpointTableProvider.cs @@ -0,0 +1,83 @@ +using Deephaven.DeephavenClient; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; +using System.Diagnostics; + +namespace Deephaven.ExcelAddIn.Providers; + +internal class DefaultEndpointTableProvider : + IObserver>, + IObserver, + // IObservable>, // redundant, part of ITableProvider + ITableProvider { + private const string UnsetTableHandleText = "[No Default Connection]"; + + private readonly StateManager _stateManager; + private readonly PersistentQueryId? _persistentQueryId; + private readonly string _tableName; + private readonly string _condition; + private readonly WorkerThread _workerThread; + private Action? _onDispose; + private IDisposable? _endpointSubscriptionDisposer = null; + private IDisposable? _upstreamSubscriptionDisposer = null; + private readonly ObserverContainer> _observers = new(); + private StatusOr _tableHandle = StatusOr.OfStatus(UnsetTableHandleText); + + public DefaultEndpointTableProvider(StateManager stateManager, + PersistentQueryId? persistentQueryId, string tableName, string condition, + Action onDispose) { + _stateManager = stateManager; + _workerThread = stateManager.WorkerThread; + _persistentQueryId = persistentQueryId; + _tableName = tableName; + _condition = condition; + _onDispose = onDispose; + } + + public void Init() { + _endpointSubscriptionDisposer = _stateManager.SubscribeToDefaultEndpointSelection(this); + } + + public IDisposable Subscribe(IObserver> observer) { + _workerThread.EnqueueOrRun(() => { + _observers.Add(observer, out _); + observer.OnNext(_tableHandle); + }); + + return _workerThread.EnqueueOrRunWhenDisposed(() => { + _observers.Remove(observer, out var isLast); + if (!isLast) { + return; + } + + Utility.Exchange(ref _endpointSubscriptionDisposer, null)?.Dispose(); + Utility.Exchange(ref _onDispose, null)?.Invoke(); + }); + } + + public void OnNext(EndpointId? endpointId) { + // Unsubscribe from old upstream + Utility.Exchange(ref _upstreamSubscriptionDisposer, null)?.Dispose(); + + // If endpoint is null, then don't subscribe to anything. + if (endpointId == null) { + _observers.SetAndSendStatus(ref _tableHandle, UnsetTableHandleText); + return; + } + + var tq = new TableQuad(endpointId, _persistentQueryId, _tableName, _condition); + _upstreamSubscriptionDisposer = _stateManager.SubscribeToTable(tq, this); + } + + public void OnNext(StatusOr value) { + _observers.SetAndSend(ref _tableHandle, value); + } + + public void OnCompleted() { + throw new NotImplementedException(); + } + + public void OnError(Exception error) { + throw new NotImplementedException(); + } +} diff --git a/csharp/ExcelAddIn/providers/FilteredTableProvider.cs b/csharp/ExcelAddIn/providers/FilteredTableProvider.cs new file mode 100644 index 00000000000..2774ec31c94 --- /dev/null +++ b/csharp/ExcelAddIn/providers/FilteredTableProvider.cs @@ -0,0 +1,106 @@ +using System.Diagnostics; +using Deephaven.DeephavenClient; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Providers; + +internal class FilteredTableProvider : + IObserver>, + // IObservable>, // redundant, part of ITableProvider + ITableProvider { + + private readonly StateManager _stateManager; + private readonly WorkerThread _workerThread; + private readonly EndpointId _endpointId; + private readonly PersistentQueryId? _persistentQueryId; + private readonly string _tableName; + private readonly string _condition; + private Action? _onDispose; + private IDisposable? _tableHandleSubscriptionDisposer = null; + private readonly ObserverContainer> _observers = new(); + private StatusOr _filteredTableHandle = StatusOr.OfStatus("[No Filtered Table]"); + + public FilteredTableProvider(StateManager stateManager, + EndpointId endpointId, PersistentQueryId? persistentQueryId, string tableName, string condition, + Action onDispose) { + _stateManager = stateManager; + _workerThread = stateManager.WorkerThread; + _endpointId = endpointId; + _persistentQueryId = persistentQueryId; + _tableName = tableName; + _condition = condition; + _onDispose = onDispose; + } + + public void Init() { + // Subscribe to a condition-free table + var tq = new TableQuad(_endpointId, _persistentQueryId, _tableName, ""); + Debug.WriteLine($"FTP is subscribing to TableHandle with {tq}"); + _tableHandleSubscriptionDisposer = _stateManager.SubscribeToTable(tq, this); + } + + public IDisposable Subscribe(IObserver> observer) { + _workerThread.EnqueueOrRun(() => { + _observers.Add(observer, out _); + observer.OnNext(_filteredTableHandle); + }); + + return _workerThread.EnqueueOrRunWhenDisposed(() => { + _observers.Remove(observer, out var isLast); + if (!isLast) { + return; + } + + Utility.Exchange(ref _tableHandleSubscriptionDisposer, null)?.Dispose(); + Utility.Exchange(ref _onDispose, null)?.Invoke(); + DisposeTableHandleState(); + }); + } + + public void OnNext(StatusOr tableHandle) { + // Get onto the worker thread if we're not already on it. + if (_workerThread.EnqueueOrNop(() => OnNext(tableHandle))) { + return; + } + + DisposeTableHandleState(); + + // If the new state is just a status message, make that our state and transmit to our observers + if (!tableHandle.GetValueOrStatus(out var th, out var status)) { + _observers.SetAndSendStatus(ref _filteredTableHandle, status); + return; + } + + // It's a real TableHandle so start fetching the table. First notify our observers. + _observers.SetAndSendStatus(ref _filteredTableHandle, "Filtering"); + + try { + var filtered = th.Where(_condition); + _observers.SetAndSendValue(ref _filteredTableHandle, filtered); + } catch (Exception ex) { + _observers.SetAndSendStatus(ref _filteredTableHandle, ex.Message); + } + } + + private void DisposeTableHandleState() { + if (_workerThread.EnqueueOrNop(DisposeTableHandleState)) { + return; + } + + _ = _filteredTableHandle.GetValueOrStatus(out var oldTh, out _); + _observers.SetAndSendStatus(ref _filteredTableHandle, "Disposing TableHandle"); + + if (oldTh != null) { + Utility.RunInBackground(oldTh.Dispose); + } + } + + public void OnCompleted() { + throw new NotImplementedException(); + } + + public void OnError(Exception error) { + throw new NotImplementedException(); + } +} diff --git a/csharp/ExcelAddIn/providers/ITableProvider.cs b/csharp/ExcelAddIn/providers/ITableProvider.cs new file mode 100644 index 00000000000..f0af94f4caa --- /dev/null +++ b/csharp/ExcelAddIn/providers/ITableProvider.cs @@ -0,0 +1,11 @@ +using Deephaven.DeephavenClient; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Providers; + +/// +/// Common interface for TableProvider, FilteredTableProvider, and DefaultEndpointTableProvider +/// +public interface ITableProvider : IObservable> { + void Init(); +} diff --git a/csharp/ExcelAddIn/providers/PersistentQueryProvider.cs b/csharp/ExcelAddIn/providers/PersistentQueryProvider.cs new file mode 100644 index 00000000000..11c240be9ff --- /dev/null +++ b/csharp/ExcelAddIn/providers/PersistentQueryProvider.cs @@ -0,0 +1,110 @@ +using Deephaven.DeephavenClient; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Providers; + +internal class PersistentQueryProvider : + IObserver>, IObservable> { + + private readonly StateManager _stateManager; + private readonly WorkerThread _workerThread; + private readonly EndpointId _endpointId; + private readonly PersistentQueryId? _pqId; + private Action? _onDispose; + private IDisposable? _upstreamSubscriptionDisposer = null; + private readonly ObserverContainer> _observers = new(); + private StatusOr _client = StatusOr.OfStatus("[No Client]"); + private Client? _ownedDndClient = null; + + public PersistentQueryProvider(StateManager stateManager, + EndpointId endpointId, PersistentQueryId? pqId, Action onDispose) { + _stateManager = stateManager; + _workerThread = stateManager.WorkerThread; + _endpointId = endpointId; + _pqId = pqId; + _onDispose = onDispose; + } + + public void Init() { + _upstreamSubscriptionDisposer = _stateManager.SubscribeToSession(_endpointId, this); + } + + public IDisposable Subscribe(IObserver> observer) { + _workerThread.EnqueueOrRun(() => { + _observers.Add(observer, out _); + observer.OnNext(_client); + }); + + return _workerThread.EnqueueOrRunWhenDisposed(() => { + _observers.Remove(observer, out var isLast); + if (!isLast) { + return; + } + + Utility.Exchange(ref _upstreamSubscriptionDisposer, null)?.Dispose(); + Utility.Exchange(ref _onDispose, null)?.Invoke(); + DisposeClientState(); + }); + } + + public void OnNext(StatusOr sessionBase) { + if (_workerThread.EnqueueOrNop(() => OnNext(sessionBase))) { + return; + } + + DisposeClientState(); + + // If the new state is just a status message, make that our state and transmit to our observers + if (!sessionBase.GetValueOrStatus(out var sb, out var status)) { + _observers.SetAndSendStatus(ref _client, status); + return; + } + + // It's a real Session so start fetching it. Also do some validity checking on the PQ id. + _ = sb.Visit( + core => { + var result = _pqId == null + ? StatusOr.OfValue(core.Client) + : StatusOr.OfStatus("PQ specified, but Community Core cannot connect to a PQ"); + _observers.SetAndSend(ref _client, result); + return Unit.Instance; + }, + corePlus => { + if (_pqId == null) { + _observers.SetAndSendStatus(ref _client, "Enterprise Core+ requires a PQ to be specified"); + return Unit.Instance; + } + + _observers.SetAndSendStatus(ref _client, $"Attaching to \"{_pqId}\""); + + try { + _ownedDndClient = corePlus.SessionManager.ConnectToPqByName(_pqId.Id, false); + _observers.SetAndSendValue(ref _client, _ownedDndClient); + } catch (Exception ex) { + _observers.SetAndSendStatus(ref _client, ex.Message); + } + return Unit.Instance; + }); + } + + private void DisposeClientState() { + if (_workerThread.EnqueueOrNop(DisposeClientState)) { + return; + } + + _observers.SetAndSendStatus(ref _client, "Disposing Client"); + var oldClient = Utility.Exchange(ref _ownedDndClient, null); + if (oldClient != null) { + Utility.RunInBackground(oldClient.Dispose); + } + } + + public void OnCompleted() { + throw new NotImplementedException(); + } + + public void OnError(Exception error) { + throw new NotImplementedException(); + } +} diff --git a/csharp/ExcelAddIn/providers/SessionProvider.cs b/csharp/ExcelAddIn/providers/SessionProvider.cs new file mode 100644 index 00000000000..f34fefdb6d8 --- /dev/null +++ b/csharp/ExcelAddIn/providers/SessionProvider.cs @@ -0,0 +1,111 @@ +using Deephaven.ExcelAddIn.Factories; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Providers; + +internal class SessionProvider : IObserver>, IObservable> { + private readonly StateManager _stateManager; + private readonly WorkerThread _workerThread; + private readonly EndpointId _endpointId; + private Action? _onDispose; + private IDisposable? _upstreamSubscriptionDisposer = null; + private StatusOr _session = StatusOr.OfStatus("[Not connected]"); + private readonly ObserverContainer> _observers = new(); + private readonly VersionTracker _versionTracker = new(); + + public SessionProvider(StateManager stateManager, EndpointId endpointId, Action onDispose) { + _stateManager = stateManager; + _workerThread = stateManager.WorkerThread; + _endpointId = endpointId; + _onDispose = onDispose; + } + + public void Init() { + _upstreamSubscriptionDisposer = _stateManager.SubscribeToCredentials(_endpointId, this); + } + + /// + /// Subscribe to session changes + /// + public IDisposable Subscribe(IObserver> observer) { + _workerThread.EnqueueOrRun(() => { + _observers.Add(observer, out _); + observer.OnNext(_session); + }); + + return _workerThread.EnqueueOrRunWhenDisposed(() => { + _observers.Remove(observer, out var isLast); + if (!isLast) { + return; + } + + Utility.Exchange(ref _upstreamSubscriptionDisposer, null)?.Dispose(); + Utility.Exchange(ref _onDispose, null)?.Invoke(); + DisposeSessionState(); + }); + } + + public void OnNext(StatusOr credentials) { + if (_workerThread.EnqueueOrNop(() => OnNext(credentials))) { + return; + } + + DisposeSessionState(); + + if (!credentials.GetValueOrStatus(out var cbase, out var status)) { + _observers.SetAndSendStatus(ref _session, status); + return; + } + + _observers.SetAndSendStatus(ref _session, "Trying to connect"); + + var cookie = _versionTracker.SetNewVersion(); + Utility.RunInBackground(() => CreateSessionBaseInSeparateThread(cbase, cookie)); + } + + private void CreateSessionBaseInSeparateThread(CredentialsBase credentials, VersionTrackerCookie versionCookie) { + SessionBase? sb = null; + StatusOr result; + try { + // This operation might take some time. + sb = SessionBaseFactory.Create(credentials, _workerThread); + result = StatusOr.OfValue(sb); + } catch (Exception ex) { + result = StatusOr.OfStatus(ex.Message); + } + + // Some time has passed. It's possible that the VersionTracker has been reset + // with a newer version. If so, we should throw away our work and leave. + if (!versionCookie.IsCurrent) { + sb?.Dispose(); + return; + } + + // Our results are valid. Keep them and tell everyone about it (on the worker thread). + _workerThread.EnqueueOrRun(() => _observers.SetAndSend(ref _session, result)); + } + + private void DisposeSessionState() { + if (_workerThread.EnqueueOrNop(DisposeSessionState)) { + return; + } + + _ = _session.GetValueOrStatus(out var oldSession, out _); + _observers.SetAndSendStatus(ref _session, "Disposing Session"); + + if (oldSession != null) { + Utility.RunInBackground(oldSession.Dispose); + } + } + + public void OnCompleted() { + // TODO(kosak) + throw new NotImplementedException(); + } + + public void OnError(Exception error) { + // TODO(kosak) + throw new NotImplementedException(); + } +} diff --git a/csharp/ExcelAddIn/providers/TableProvider.cs b/csharp/ExcelAddIn/providers/TableProvider.cs new file mode 100644 index 00000000000..d4da46fef3a --- /dev/null +++ b/csharp/ExcelAddIn/providers/TableProvider.cs @@ -0,0 +1,100 @@ +using Deephaven.DeephavenClient; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Providers; + +internal class TableProvider : + IObserver>, + // IObservable>, // redundant, part of ITableProvider + ITableProvider { + private const string UnsetTableHandleText = "[No Table]"; + + private readonly StateManager _stateManager; + private readonly WorkerThread _workerThread; + private readonly EndpointId _endpointId; + private readonly PersistentQueryId? _persistentQueryId; + private readonly string _tableName; + private Action? _onDispose; + private IDisposable? _pqSubscriptionDisposer = null; + private readonly ObserverContainer> _observers = new(); + private StatusOr _tableHandle = StatusOr.OfStatus(UnsetTableHandleText); + + public TableProvider(StateManager stateManager, EndpointId endpointId, + PersistentQueryId? persistentQueryId, string tableName, Action onDispose) { + _stateManager = stateManager; + _workerThread = stateManager.WorkerThread; + _endpointId = endpointId; + _persistentQueryId = persistentQueryId; + _tableName = tableName; + _onDispose = onDispose; + } + + public void Init() { + _pqSubscriptionDisposer = _stateManager.SubscribeToPersistentQuery( + _endpointId, _persistentQueryId, this); + } + + public IDisposable Subscribe(IObserver> observer) { + _workerThread.EnqueueOrRun(() => { + _observers.Add(observer, out _); + observer.OnNext(_tableHandle); + }); + + return _workerThread.EnqueueOrRunWhenDisposed(() => { + _observers.Remove(observer, out var isLast); + if (!isLast) { + return; + } + + Utility.Exchange(ref _pqSubscriptionDisposer, null)?.Dispose(); + Utility.Exchange(ref _onDispose, null)?.Invoke(); + DisposeTableHandleState(); + }); + } + + public void OnNext(StatusOr client) { + if (_workerThread.EnqueueOrNop(() => OnNext(client))) { + return; + } + + DisposeTableHandleState(); + + // If the new state is just a status message, make that our state and transmit to our observers + if (!client.GetValueOrStatus(out var cli, out var status)) { + _observers.SetAndSendStatus(ref _tableHandle, status); + return; + } + + // It's a real client so start fetching the table. First notify our observers. + _observers.SetAndSendStatus(ref _tableHandle, $"Fetching \"{_tableName}\""); + + try { + var th = cli.Manager.FetchTable(_tableName); + _observers.SetAndSendValue(ref _tableHandle, th); + } catch (Exception ex) { + _observers.SetAndSendStatus(ref _tableHandle, ex.Message); + } + } + + private void DisposeTableHandleState() { + if (_workerThread.EnqueueOrNop(DisposeTableHandleState)) { + return; + } + + _ = _tableHandle.GetValueOrStatus(out var oldTh, out _); + _observers.SetAndSendStatus(ref _tableHandle, UnsetTableHandleText); + + if (oldTh != null) { + Utility.RunInBackground(oldTh.Dispose); + } + } + + public void OnCompleted() { + throw new NotImplementedException(); + } + + public void OnError(Exception error) { + throw new NotImplementedException(); + } +} diff --git a/csharp/ExcelAddIn/util/ActionAsDisposable.cs b/csharp/ExcelAddIn/util/ActionAsDisposable.cs new file mode 100644 index 00000000000..e195d2aa7f6 --- /dev/null +++ b/csharp/ExcelAddIn/util/ActionAsDisposable.cs @@ -0,0 +1,21 @@ +namespace Deephaven.DeephavenClient.ExcelAddIn.Util; + +internal class ActionAsDisposable : IDisposable { + public static IDisposable Create(Action action) { + return new ActionAsDisposable(action); + } + + private Action? _action; + + private ActionAsDisposable(Action action) => _action = action; + + public void Dispose() { + var temp = _action; + if (temp == null) { + return; + } + + _action = null; + temp(); + } +} diff --git a/csharp/ExcelAddIn/util/ObservableConverter.cs b/csharp/ExcelAddIn/util/ObservableConverter.cs new file mode 100644 index 00000000000..b31defc40ee --- /dev/null +++ b/csharp/ExcelAddIn/util/ObservableConverter.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; +using Deephaven.DeephavenClient.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Util; + +internal static class ObservableConverter { + public static ObservableConverter Create( + Func converter, WorkerThread workerThread) { + return new ObservableConverter(converter, workerThread); + } +} + +internal class ObservableConverter(Func converter, WorkerThread workerThread) : + IObserver, IObservable { + private readonly ObserverContainer _observers = new(); + + public void OnNext(TFrom value) { + var converted = converter(value); + _observers.OnNext(converted); + } + + public void OnCompleted() { + _observers.OnCompleted(); + } + + public void OnError(Exception error) { + _observers.OnError(error); + } + + public IDisposable Subscribe(IObserver observer) { + workerThread.EnqueueOrRun(() => _observers.Add(observer, out _)); + + return ActionAsDisposable.Create(() => { + workerThread.EnqueueOrRun(() => { + _observers.Remove(observer, out _); + }); + }); + } +} diff --git a/csharp/ExcelAddIn/util/ObserverContainer.cs b/csharp/ExcelAddIn/util/ObserverContainer.cs new file mode 100644 index 00000000000..e1d0e98cf42 --- /dev/null +++ b/csharp/ExcelAddIn/util/ObserverContainer.cs @@ -0,0 +1,52 @@ +namespace Deephaven.ExcelAddIn.Util; + +public sealed class ObserverContainer : IObserver { + private readonly object _sync = new(); + private readonly HashSet> _observers = new(); + + public int Count { + get { + lock (_sync) { + return _observers.Count; + } + } + } + + public void Add(IObserver observer, out bool isFirst) { + lock (_sync) { + isFirst = _observers.Count == 0; + _observers.Add(observer); + } + } + + public void Remove(IObserver observer, out bool wasLast) { + lock (_sync) { + var removed = _observers.Remove(observer); + wasLast = removed && _observers.Count == 0; + } + } + + public void OnNext(T result) { + foreach (var observer in SafeCopyObservers()) { + observer.OnNext(result); + } + } + + public void OnError(Exception ex) { + foreach (var observer in SafeCopyObservers()) { + observer.OnError(ex); + } + } + + public void OnCompleted() { + foreach (var observer in SafeCopyObservers()) { + observer.OnCompleted(); + } + } + + private IObserver[] SafeCopyObservers() { + lock (_sync) { + return _observers.ToArray(); + } + } +} diff --git a/csharp/ExcelAddIn/util/Renderer.cs b/csharp/ExcelAddIn/util/Renderer.cs new file mode 100644 index 00000000000..b0fba4fecc7 --- /dev/null +++ b/csharp/ExcelAddIn/util/Renderer.cs @@ -0,0 +1,33 @@ +using Deephaven.DeephavenClient; + +namespace Deephaven.ExcelAddIn.Util; + +internal static class Renderer { + public static object?[,] Render(ClientTable table, bool wantHeaders) { + var numRows = table.NumRows; + var numCols = table.NumCols; + var effectiveNumRows = wantHeaders ? numRows + 1 : numRows; + var result = new object?[effectiveNumRows, numCols]; + + var headers = table.Schema.Names; + for (var colIndex = 0; colIndex != numCols; ++colIndex) { + var destIndex = 0; + if (wantHeaders) { + result[destIndex++, colIndex] = headers[colIndex]; + } + + var (col, nulls) = table.GetColumn(colIndex); + for (var i = 0; i != numRows; ++i) { + var temp = nulls[i] ? null : col.GetValue(i); + // sad hack, wrong place, inefficient + if (temp is DhDateTime dh) { + temp = dh.DateTime.ToString("s", System.Globalization.CultureInfo.InvariantCulture); + } + + result[destIndex++, colIndex] = temp; + } + } + + return result; + } +} diff --git a/csharp/ExcelAddIn/util/StatusOr.cs b/csharp/ExcelAddIn/util/StatusOr.cs new file mode 100644 index 00000000000..1de6f583af6 --- /dev/null +++ b/csharp/ExcelAddIn/util/StatusOr.cs @@ -0,0 +1,62 @@ +using System.Diagnostics.CodeAnalysis; +using System.Windows.Forms; + +namespace Deephaven.ExcelAddIn.Util; + +public sealed class StatusOr { + private readonly string? _status; + private readonly T? _value; + + public static StatusOr OfStatus(string status) { + return new StatusOr(status, default); + } + + public static StatusOr OfValue(T value) { + return new StatusOr(null, value); + } + + private StatusOr(string? status, T? value) { + _status = status; + _value = value; + } + + public bool GetValueOrStatus( + [NotNullWhen(true)]out T? value, + [NotNullWhen(false)]out string? status) { + status = _status; + value = _value; + return value != null; + } + + public U AcceptVisitor(Func onValue, Func onStatus) { + return _value != null ? onValue(_value) : onStatus(_status!); + } +} + +public static class ObserverStatusOr_Extensions { + public static void SendStatus(this IObserver> observer, string message) { + var so = StatusOr.OfStatus(message); + observer.OnNext(so); + } + + public static void SendValue(this IObserver> observer, T value) { + var so = StatusOr.OfValue(value); + observer.OnNext(so); + } + + public static void SetAndSendStatus(this IObserver> observer, ref StatusOr sor, + string message) { + SetAndSend(observer, ref sor, StatusOr.OfStatus(message)); + } + + public static void SetAndSendValue(this IObserver> observer, ref StatusOr sor, + T value) { + SetAndSend(observer, ref sor, StatusOr.OfValue(value)); + } + + public static void SetAndSend(this IObserver> observer, ref StatusOr sor, + StatusOr newSor) { + sor = newSor; + observer.OnNext(newSor); + } +} diff --git a/csharp/ExcelAddIn/util/Utility.cs b/csharp/ExcelAddIn/util/Utility.cs new file mode 100644 index 00000000000..4b9de77ca3a --- /dev/null +++ b/csharp/ExcelAddIn/util/Utility.cs @@ -0,0 +1,45 @@ + +using System.Diagnostics; + +namespace Deephaven.ExcelAddIn.Util; + +internal static class Utility { + public static T Exchange(ref T item, T newValue) { + var result = item; + item = newValue; + return result; + } + + public static void RunInBackground(Action a) { + void Doit() { + try { + a(); + } catch (Exception e) { + Debug.WriteLine($"Ignoring exception {e}"); + } + } + new Thread(Doit) { IsBackground = true }.Start(); + } +} + +public class Unit { + public static readonly Unit Instance = new (); + + private Unit() { + } +} + +public class ValueHolder where T : class { + private T? _value = null; + + public T Value { + get { + if (_value == null) { + throw new Exception("Value is unset"); + } + + return _value; + } + set => _value = value; + } +} diff --git a/csharp/ExcelAddIn/util/VersionTracker.cs b/csharp/ExcelAddIn/util/VersionTracker.cs new file mode 100644 index 00000000000..a0fcbe48c71 --- /dev/null +++ b/csharp/ExcelAddIn/util/VersionTracker.cs @@ -0,0 +1,27 @@ +namespace Deephaven.ExcelAddIn.Util; + +internal class VersionTracker { + private readonly object _sync = new(); + private VersionTrackerCookie _cookie; + + public VersionTracker() { + _cookie = new VersionTrackerCookie(this); + } + + public VersionTrackerCookie SetNewVersion() { + lock (_sync) { + _cookie = new VersionTrackerCookie(this); + return _cookie; + } + } + + public bool HasCookie(VersionTrackerCookie cookie) { + lock (_sync) { + return ReferenceEquals(_cookie, cookie); + } + } +} + +internal class VersionTrackerCookie(VersionTracker owner) { + public bool IsCurrent => owner.HasCookie(this); +} diff --git a/csharp/ExcelAddIn/util/WorkerThread.cs b/csharp/ExcelAddIn/util/WorkerThread.cs new file mode 100644 index 00000000000..eceea724114 --- /dev/null +++ b/csharp/ExcelAddIn/util/WorkerThread.cs @@ -0,0 +1,71 @@ +using System.Diagnostics; +using Deephaven.DeephavenClient.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Util; + +public class WorkerThread { + public static WorkerThread Create() { + var result = new WorkerThread(); + var t = new Thread(result.Doit) { IsBackground = true }; + result._thisThread = t; + t.Start(); + return result; + } + + private readonly object _sync = new(); + private readonly Queue _queue = new(); + private Thread? _thisThread; + + private WorkerThread() { + } + + // enquee or run + public void EnqueueOrRun(Action action) { + if (!EnqueueOrNop(action)) { + action(); + } + } + + // conditionalenqueue + public bool EnqueueOrNop(Action action) { + if (ReferenceEquals(Thread.CurrentThread, _thisThread)) { + // Appending to thread queue was not required. Return false. + return false; + } + + lock (_sync) { + _queue.Enqueue(action); + if (_queue.Count == 1) { + // Only need to pulse on transition from 0 to 1, because the + // Doit method only Waits if the queue is empty. + Monitor.PulseAll(_sync); + } + } + + // Appending to thread queue was required. + return true; + } + + public IDisposable EnqueueOrRunWhenDisposed(Action action) { + return ActionAsDisposable.Create(() => EnqueueOrRun(action)); + } + + private void Doit() { + while (true) { + Action action; + lock (_sync) { + while (_queue.Count == 0) { + Monitor.Wait(_sync); + } + + action = _queue.Dequeue(); + } + + try { + action(); + } catch (Exception ex) { + Debug.WriteLine($"Swallowing exception {ex}"); + } + } + } +} diff --git a/csharp/ExcelAddIn/viewmodels/ConnectionManagerDialogRow.cs b/csharp/ExcelAddIn/viewmodels/ConnectionManagerDialogRow.cs new file mode 100644 index 00000000000..a34b9a6fbdb --- /dev/null +++ b/csharp/ExcelAddIn/viewmodels/ConnectionManagerDialogRow.cs @@ -0,0 +1,96 @@ +using System.ComponentModel; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Viewmodels; + +public sealed class ConnectionManagerDialogRow(string id) : INotifyPropertyChanged { + + public event PropertyChangedEventHandler? PropertyChanged; + + private readonly object _sync = new(); + private StatusOr _credentials = StatusOr.OfStatus("[Not set]"); + private StatusOr _session = StatusOr.OfStatus("[Not connected]"); + private EndpointId? _defaultEndpointId = null; + + [DisplayName("Name")] + public string Id { get; init; } = id; + + public string Status { + get { + var session = GetSessionSynced(); + // If we have a valid session, return "[Connected]", otherwise pass through the status text we have. + return session.AcceptVisitor( + _ => "[Connected]", + status => status); + } + } + + [DisplayName("Server Type")] + public string ServerType { + get { + var creds = GetCredentialsSynced(); + // Nested AcceptVisitor!! + // If we have valid credentials, determine whether they are for Core or Core+ and return the appropriate string. + // Otherwise (if we have invalid credentials), ignore their status text and just say "[Unknown]". + return creds.AcceptVisitor( + crs => crs.AcceptVisitor(_ => "Core", _ => "Core+"), + _ => "[Unknown]"); + + } + } + + [DisplayName("Default")] + public bool IsDefault { + get { + var id = Id; // readonly so no synchronization needed. + var defaultEp = GetDefaultEndpointIdSynced(); + return defaultEp != null && defaultEp.Id == id; + } + } + + public StatusOr GetCredentialsSynced() { + lock (_sync) { + return _credentials; + } + } + + public void SetCredentialsSynced(StatusOr value) { + lock (_sync) { + _credentials = value; + } + + OnPropertyChanged(nameof(ServerType)); + OnPropertyChanged(nameof(IsDefault)); + } + + public EndpointId? GetDefaultEndpointIdSynced() { + lock (_sync) { + return _defaultEndpointId; + } + } + + public void SetDefaultEndpointIdSynced(EndpointId? value) { + lock (_sync) { + _defaultEndpointId = value; + } + OnPropertyChanged(nameof(IsDefault)); + } + + public StatusOr GetSessionSynced() { + lock (_sync) { + return _session; + } + } + + public void SetSessionSynced(StatusOr value) { + lock (_sync) { + _session = value; + } + OnPropertyChanged(nameof(Status)); + } + + private void OnPropertyChanged(string name) { + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(name)); + } +} diff --git a/csharp/ExcelAddIn/viewmodels/CredentialsDialogViewModel.cs b/csharp/ExcelAddIn/viewmodels/CredentialsDialogViewModel.cs new file mode 100644 index 00000000000..86cae1c39e0 --- /dev/null +++ b/csharp/ExcelAddIn/viewmodels/CredentialsDialogViewModel.cs @@ -0,0 +1,252 @@ +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.ViewModels; + +public sealed class CredentialsDialogViewModel : INotifyPropertyChanged { + public static CredentialsDialogViewModel OfEmpty() { + return new CredentialsDialogViewModel(); + } + + public static CredentialsDialogViewModel OfIdButOtherwiseEmpty(string id) { + return new CredentialsDialogViewModel { Id = id }; + } + + public static CredentialsDialogViewModel OfIdAndCredentials(string id, CredentialsBase credentials) { + var result = new CredentialsDialogViewModel { + Id = credentials.Id.Id + }; + _ = credentials.AcceptVisitor( + core => { + result._isCorePlus = false; + result.ConnectionString = core.ConnectionString; + result.SessionTypeIsPython = core.SessionTypeIsPython; + return Unit.Instance; + }, + corePlus => { + result._isCorePlus = true; + result.JsonUrl = corePlus.JsonUrl; + result.UserId = corePlus.User; + result.Password = corePlus.Password; + result.OperateAs = corePlus.OperateAs.Equals(corePlus.User) ? "" : corePlus.OperateAs; + result.ValidateCertificate = corePlus.ValidateCertificate; + return Unit.Instance; + }); + + return result; + } + + private string _id = ""; + private bool _isDefault = false; + private bool _isCorePlus = true; + private bool _sessionTypeIsPython = true; + + // Core properties + private string _connectionString = ""; + + // Core+ properties + private string _jsonUrl = ""; + private bool _validateCertificate = true; + private string _userId = ""; + private string _password = ""; + private string _operateAs = ""; + + public event PropertyChangedEventHandler? PropertyChanged; + + public bool TryMakeCredentials([NotNullWhen(true)] out CredentialsBase? result, + [NotNullWhen(false)] out string? errorText) { + result = null; + errorText = null; + + var missingFields = new List(); + void CheckMissing(string field, string name) { + if (field.Length == 0) { + missingFields.Add(name); + } + } + + CheckMissing(Id, "Connection Id"); + + if (!_isCorePlus) { + CheckMissing(ConnectionString, "Connection String"); + } else { + CheckMissing(JsonUrl, "JSON URL"); + CheckMissing(UserId, "User Id"); + CheckMissing(Password, "Password"); + } + + if (missingFields.Count > 0) { + errorText = string.Join(Environment.NewLine, missingFields); + return false; + } + + var epId = new EndpointId(_id); + result = _isCorePlus + ? CredentialsBase.OfCorePlus(epId, JsonUrl, UserId, Password, OperateAsToUse, ValidateCertificate) + : CredentialsBase.OfCore(epId, ConnectionString, SessionTypeIsPython); + return true; + } + + public string Id { + get => _id; + set { + if (value == _id) { + return; + } + + _id = value; + OnPropertyChanged(); + } + } + + public bool IsDefault { + get => _isDefault; + set { + if (value == _isDefault) { + return; + } + + _isDefault = value; + OnPropertyChanged(); + } + } + + /** + * I don't know if I have to do it this way, but I bind IsCore and IsCorePlus to the + * same underlying variable. The property "IsCore" maps to the inverse of the variable + * _isCorePlus, meanwhile the property "IsCorePlus" maps to the normal sense of the + * variable. Setters on either one trigger property change events for both. + */ + public bool IsCore { + get => !_isCorePlus; + set { + if (_isCorePlus == !value) { + return; + } + + _isCorePlus = !value; + OnPropertyChanged(); + OnPropertyChanged(nameof(IsCorePlus)); + } + } + + public bool IsCorePlus { + get => _isCorePlus; + set { + if (_isCorePlus == value) { + return; + } + + _isCorePlus = value; + OnPropertyChanged(); + OnPropertyChanged(nameof(IsCore)); + } + } + + public string ConnectionString { + get => _connectionString; + set { + if (_connectionString == value) { + return; + } + + _connectionString = value; + OnPropertyChanged(); + } + } + + public string JsonUrl { + get => _jsonUrl; + set { + if (_jsonUrl == value) { + return; + } + + _jsonUrl = value; + OnPropertyChanged(); + } + } + + public string UserId { + get => _userId; + set { + if (_userId == value) { + return; + } + + _userId = value; + OnPropertyChanged(); + } + } + + public string Password { + get => _password; + set { + if (_password == value) { + return; + } + + _password = value; + OnPropertyChanged(); + } + } + + public string OperateAs { + get => _operateAs; + set { + if (_operateAs == value) { + return; + } + + _operateAs = value; + OnPropertyChanged(); + } + } + + public string OperateAsToUse => _operateAs.Length != 0 ? _operateAs : UserId; + + public bool ValidateCertificate { + get => _validateCertificate; + set { + if (_validateCertificate == value) { + return; + } + + _validateCertificate = value; + OnPropertyChanged(); + } + } + + public bool SessionTypeIsPython { + get => _sessionTypeIsPython; + set { + if (_sessionTypeIsPython == value) { + return; + } + + _sessionTypeIsPython = value; + OnPropertyChanged(); + OnPropertyChanged(nameof(SessionTypeIsGroovy)); + } + } + + public bool SessionTypeIsGroovy { + get => !_sessionTypeIsPython; + set { + if (!_sessionTypeIsPython == value) { + return; + } + + _sessionTypeIsPython = !value; + OnPropertyChanged(); + OnPropertyChanged(nameof(SessionTypeIsPython)); + } + } + + private void OnPropertyChanged([CallerMemberName] string? name = null) { + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(name)); + } +} diff --git a/csharp/ExcelAddIn/views/ConnectionManagerDialog.Designer.cs b/csharp/ExcelAddIn/views/ConnectionManagerDialog.Designer.cs new file mode 100644 index 00000000000..604c97b2d12 --- /dev/null +++ b/csharp/ExcelAddIn/views/ConnectionManagerDialog.Designer.cs @@ -0,0 +1,147 @@ +namespace Deephaven.ExcelAddIn.Views { + partial class ConnectionManagerDialog { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) { + if (disposing && (components != null)) { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() { + colorDialog1 = new ColorDialog(); + dataGridView1 = new DataGridView(); + newButton = new Button(); + connectionsLabel = new Label(); + editButton = new Button(); + deleteButton = new Button(); + reconnectButton = new Button(); + makeDefaultButton = new Button(); + ((System.ComponentModel.ISupportInitialize)dataGridView1).BeginInit(); + SuspendLayout(); + // + // dataGridView1 + // + dataGridView1.AllowUserToAddRows = false; + dataGridView1.AllowUserToDeleteRows = false; + dataGridView1.Anchor = AnchorStyles.Top | AnchorStyles.Bottom | AnchorStyles.Left | AnchorStyles.Right; + dataGridView1.AutoSizeColumnsMode = DataGridViewAutoSizeColumnsMode.Fill; + dataGridView1.ColumnHeadersHeightSizeMode = DataGridViewColumnHeadersHeightSizeMode.AutoSize; + dataGridView1.Location = new Point(68, 83); + dataGridView1.Name = "dataGridView1"; + dataGridView1.ReadOnly = true; + dataGridView1.RowHeadersWidth = 62; + dataGridView1.SelectionMode = DataGridViewSelectionMode.FullRowSelect; + dataGridView1.Size = new Size(979, 454); + dataGridView1.TabIndex = 0; + // + // newButton + // + newButton.Anchor = AnchorStyles.Bottom | AnchorStyles.Right; + newButton.Location = new Point(919, 560); + newButton.Name = "newButton"; + newButton.Size = new Size(128, 34); + newButton.TabIndex = 5; + newButton.Text = "New..."; + newButton.UseVisualStyleBackColor = true; + newButton.Click += newButton_Click; + // + // connectionsLabel + // + connectionsLabel.AutoSize = true; + connectionsLabel.Font = new Font("Segoe UI", 12F, FontStyle.Regular, GraphicsUnit.Point, 0); + connectionsLabel.Location = new Point(68, 33); + connectionsLabel.Name = "connectionsLabel"; + connectionsLabel.Size = new Size(147, 32); + connectionsLabel.TabIndex = 2; + connectionsLabel.Text = "Connections"; + // + // editButton + // + editButton.Anchor = AnchorStyles.Bottom | AnchorStyles.Right; + editButton.Location = new Point(776, 560); + editButton.Name = "editButton"; + editButton.Size = new Size(112, 34); + editButton.TabIndex = 4; + editButton.Text = "Edit..."; + editButton.UseVisualStyleBackColor = true; + editButton.Click += editButton_Click; + // + // deleteButton + // + deleteButton.Anchor = AnchorStyles.Bottom | AnchorStyles.Right; + deleteButton.Location = new Point(339, 560); + deleteButton.Name = "deleteButton"; + deleteButton.Size = new Size(112, 34); + deleteButton.TabIndex = 1; + deleteButton.Text = "Delete"; + deleteButton.UseVisualStyleBackColor = true; + deleteButton.Click += deleteButton_Click; + // + // reconnectButton + // + reconnectButton.Anchor = AnchorStyles.Bottom | AnchorStyles.Right; + reconnectButton.Location = new Point(636, 560); + reconnectButton.Name = "reconnectButton"; + reconnectButton.Size = new Size(112, 34); + reconnectButton.TabIndex = 3; + reconnectButton.Text = "Reconnect"; + reconnectButton.UseVisualStyleBackColor = true; + reconnectButton.Click += reconnectButton_Click; + // + // makeDefaultButton + // + makeDefaultButton.Anchor = AnchorStyles.Bottom | AnchorStyles.Right; + makeDefaultButton.Location = new Point(473, 560); + makeDefaultButton.Name = "makeDefaultButton"; + makeDefaultButton.Size = new Size(139, 34); + makeDefaultButton.TabIndex = 2; + makeDefaultButton.Text = "Make Default"; + makeDefaultButton.UseVisualStyleBackColor = true; + makeDefaultButton.Click += makeDefaultButton_Click; + // + // ConnectionManagerDialog + // + AutoScaleDimensions = new SizeF(10F, 25F); + AutoScaleMode = AutoScaleMode.Font; + ClientSize = new Size(1115, 615); + Controls.Add(makeDefaultButton); + Controls.Add(reconnectButton); + Controls.Add(deleteButton); + Controls.Add(editButton); + Controls.Add(connectionsLabel); + Controls.Add(newButton); + Controls.Add(dataGridView1); + Name = "ConnectionManagerDialog"; + Text = "Connection Manager"; + ((System.ComponentModel.ISupportInitialize)dataGridView1).EndInit(); + ResumeLayout(false); + PerformLayout(); + } + + #endregion + + private ColorDialog colorDialog1; + private DataGridView dataGridView1; + private Button newButton; + private Label connectionsLabel; + private Button editButton; + private Button deleteButton; + private Button reconnectButton; + private Button makeDefaultButton; + } +} \ No newline at end of file diff --git a/csharp/ExcelAddIn/views/ConnectionManagerDialog.cs b/csharp/ExcelAddIn/views/ConnectionManagerDialog.cs new file mode 100644 index 00000000000..d56b79f950a --- /dev/null +++ b/csharp/ExcelAddIn/views/ConnectionManagerDialog.cs @@ -0,0 +1,73 @@ +using Deephaven.ExcelAddIn.Viewmodels; + +namespace Deephaven.ExcelAddIn.Views; + +using SelectedRowsAction = Action; + +public partial class ConnectionManagerDialog : Form { + public event Action? OnNewButtonClicked; + public event SelectedRowsAction? OnDeleteButtonClicked; + public event SelectedRowsAction? OnReconnectButtonClicked; + public event SelectedRowsAction? OnMakeDefaultButtonClicked; + public event SelectedRowsAction? OnEditButtonClicked; + + private readonly BindingSource _bindingSource = new(); + + public ConnectionManagerDialog() { + InitializeComponent(); + + _bindingSource.DataSource = typeof(ConnectionManagerDialogRow); + dataGridView1.DataSource = _bindingSource; + } + + public void AddRow(ConnectionManagerDialogRow row) { + if (InvokeRequired) { + Invoke(() => AddRow(row)); + return; + } + _bindingSource.Add(row); + dataGridView1.ClearSelection(); + } + + public void RemoveRow(ConnectionManagerDialogRow row) { + if (InvokeRequired) { + Invoke(() => RemoveRow(row)); + return; + } + _bindingSource.Remove(row); + } + + private void newButton_Click(object sender, EventArgs e) { + OnNewButtonClicked?.Invoke(); + } + + private void reconnectButton_Click(object sender, EventArgs e) { + var selections = GetSelectedRows(); + OnReconnectButtonClicked?.Invoke(selections); + } + + private void editButton_Click(object sender, EventArgs e) { + var selections = GetSelectedRows(); + OnEditButtonClicked?.Invoke(selections); + } + + private void deleteButton_Click(object sender, EventArgs e) { + var selections = GetSelectedRows(); + OnDeleteButtonClicked?.Invoke(selections); + } + + private void makeDefaultButton_Click(object sender, EventArgs e) { + var selections = GetSelectedRows(); + OnMakeDefaultButtonClicked?.Invoke(selections); + } + + private ConnectionManagerDialogRow[] GetSelectedRows() { + var result = new List(); + var sr = dataGridView1.SelectedRows; + var count = sr.Count; + for (var i = 0; i != count; ++i) { + result.Add((ConnectionManagerDialogRow)sr[i].DataBoundItem); + } + return result.ToArray(); + } +} diff --git a/csharp/ExcelAddIn/views/ConnectionManagerDialog.resx b/csharp/ExcelAddIn/views/ConnectionManagerDialog.resx new file mode 100644 index 00000000000..7f2cf2b8014 --- /dev/null +++ b/csharp/ExcelAddIn/views/ConnectionManagerDialog.resx @@ -0,0 +1,123 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + 17, 17 + + \ No newline at end of file diff --git a/csharp/ExcelAddIn/views/CredentialsDialog.Designer.cs b/csharp/ExcelAddIn/views/CredentialsDialog.Designer.cs new file mode 100644 index 00000000000..aca6db6b75a --- /dev/null +++ b/csharp/ExcelAddIn/views/CredentialsDialog.Designer.cs @@ -0,0 +1,400 @@ +namespace ExcelAddIn.views { + partial class CredentialsDialog { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) { + if (disposing && (components != null)) { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() { + corePlusPanel = new Panel(); + label1 = new Label(); + validateCertCheckBox = new CheckBox(); + operateAsBox = new TextBox(); + passwordBox = new TextBox(); + passwordLabel = new Label(); + userIdBox = new TextBox(); + userIdLabel = new Label(); + jsonUrlBox = new TextBox(); + jsonUrlLabel = new Label(); + corePanel = new Panel(); + connectionStringBox = new TextBox(); + label2 = new Label(); + groupBox1 = new GroupBox(); + sessionTypeIsGroovyButton = new RadioButton(); + sessionTypeIsPythonButton = new RadioButton(); + finalPanel = new Panel(); + makeDefaultCheckBox = new CheckBox(); + testResultsTextBox = new TextBox(); + testResultsLabel = new Label(); + testCredentialsButton = new Button(); + setCredentialsButton = new Button(); + isCorePlusRadioButton = new RadioButton(); + isCoreRadioButton = new RadioButton(); + endpointIdBox = new TextBox(); + connectionIdLabel = new Label(); + connectionTypeGroup = new GroupBox(); + connectionIdPanel = new Panel(); + corePlusPanel.SuspendLayout(); + corePanel.SuspendLayout(); + groupBox1.SuspendLayout(); + finalPanel.SuspendLayout(); + connectionTypeGroup.SuspendLayout(); + connectionIdPanel.SuspendLayout(); + SuspendLayout(); + // + // corePlusPanel + // + corePlusPanel.Controls.Add(label1); + corePlusPanel.Controls.Add(validateCertCheckBox); + corePlusPanel.Controls.Add(operateAsBox); + corePlusPanel.Controls.Add(passwordBox); + corePlusPanel.Controls.Add(passwordLabel); + corePlusPanel.Controls.Add(userIdBox); + corePlusPanel.Controls.Add(userIdLabel); + corePlusPanel.Controls.Add(jsonUrlBox); + corePlusPanel.Controls.Add(jsonUrlLabel); + corePlusPanel.Dock = DockStyle.Top; + corePlusPanel.Location = new Point(0, 191); + corePlusPanel.Name = "corePlusPanel"; + corePlusPanel.Size = new Size(1054, 299); + corePlusPanel.TabIndex = 2; + // + // label1 + // + label1.AutoSize = true; + label1.Location = new Point(75, 183); + label1.Name = "label1"; + label1.Size = new Size(96, 25); + label1.TabIndex = 217; + label1.Text = "OperateAs"; + // + // validateCertCheckBox + // + validateCertCheckBox.AutoSize = true; + validateCertCheckBox.Location = new Point(194, 237); + validateCertCheckBox.Name = "validateCertCheckBox"; + validateCertCheckBox.RightToLeft = RightToLeft.No; + validateCertCheckBox.Size = new Size(183, 29); + validateCertCheckBox.TabIndex = 5; + validateCertCheckBox.Text = "Validate Certificate"; + validateCertCheckBox.UseVisualStyleBackColor = true; + // + // operateAsBox + // + operateAsBox.Anchor = AnchorStyles.Top | AnchorStyles.Left | AnchorStyles.Right; + operateAsBox.Location = new Point(194, 180); + operateAsBox.Name = "operateAsBox"; + operateAsBox.PlaceholderText = "Leave blank for default"; + operateAsBox.Size = new Size(795, 31); + operateAsBox.TabIndex = 4; + // + // passwordBox + // + passwordBox.Anchor = AnchorStyles.Top | AnchorStyles.Left | AnchorStyles.Right; + passwordBox.Location = new Point(194, 130); + passwordBox.Name = "passwordBox"; + passwordBox.PasswordChar = '●'; + passwordBox.Size = new Size(795, 31); + passwordBox.TabIndex = 3; + // + // passwordLabel + // + passwordLabel.AutoSize = true; + passwordLabel.Location = new Point(84, 133); + passwordLabel.Name = "passwordLabel"; + passwordLabel.Size = new Size(87, 25); + passwordLabel.TabIndex = 4; + passwordLabel.Text = "Password"; + // + // userIdBox + // + userIdBox.Anchor = AnchorStyles.Top | AnchorStyles.Left | AnchorStyles.Right; + userIdBox.Location = new Point(194, 82); + userIdBox.Name = "userIdBox"; + userIdBox.Size = new Size(795, 31); + userIdBox.TabIndex = 2; + // + // userIdLabel + // + userIdLabel.AutoSize = true; + userIdLabel.Location = new Point(108, 85); + userIdLabel.Name = "userIdLabel"; + userIdLabel.Size = new Size(63, 25); + userIdLabel.TabIndex = 2; + userIdLabel.Text = "UserId"; + // + // jsonUrlBox + // + jsonUrlBox.Anchor = AnchorStyles.Top | AnchorStyles.Left | AnchorStyles.Right; + jsonUrlBox.Location = new Point(194, 27); + jsonUrlBox.Name = "jsonUrlBox"; + jsonUrlBox.Size = new Size(795, 31); + jsonUrlBox.TabIndex = 1; + // + // jsonUrlLabel + // + jsonUrlLabel.AutoSize = true; + jsonUrlLabel.Location = new Point(89, 30); + jsonUrlLabel.Name = "jsonUrlLabel"; + jsonUrlLabel.Size = new Size(91, 25); + jsonUrlLabel.TabIndex = 0; + jsonUrlLabel.Text = "JSON URL"; + // + // corePanel + // + corePanel.Controls.Add(connectionStringBox); + corePanel.Controls.Add(label2); + corePanel.Controls.Add(groupBox1); + corePanel.Dock = DockStyle.Top; + corePanel.Location = new Point(0, 490); + corePanel.Name = "corePanel"; + corePanel.Size = new Size(1054, 170); + corePanel.TabIndex = 3; + // + // connectionStringBox + // + connectionStringBox.Anchor = AnchorStyles.Top | AnchorStyles.Left | AnchorStyles.Right; + connectionStringBox.Location = new Point(194, 20); + connectionStringBox.Name = "connectionStringBox"; + connectionStringBox.Size = new Size(795, 31); + connectionStringBox.TabIndex = 1; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new Point(27, 23); + label2.Name = "label2"; + label2.Size = new Size(153, 25); + label2.TabIndex = 0; + label2.Text = "Connection String"; + // + // groupBox1 + // + groupBox1.Controls.Add(sessionTypeIsGroovyButton); + groupBox1.Controls.Add(sessionTypeIsPythonButton); + groupBox1.Location = new Point(18, 80); + groupBox1.Name = "groupBox1"; + groupBox1.Size = new Size(970, 71); + groupBox1.TabIndex = 2; + groupBox1.TabStop = false; + groupBox1.Text = "Session Type"; + // + // sessionTypeIsGroovyButton + // + sessionTypeIsGroovyButton.AutoSize = true; + sessionTypeIsGroovyButton.Location = new Point(338, 30); + sessionTypeIsGroovyButton.Name = "sessionTypeIsGroovyButton"; + sessionTypeIsGroovyButton.Size = new Size(95, 29); + sessionTypeIsGroovyButton.TabIndex = 2; + sessionTypeIsGroovyButton.TabStop = true; + sessionTypeIsGroovyButton.Text = "Groovy"; + sessionTypeIsGroovyButton.UseVisualStyleBackColor = true; + // + // sessionTypeIsPythonButton + // + sessionTypeIsPythonButton.AutoSize = true; + sessionTypeIsPythonButton.Location = new Point(192, 30); + sessionTypeIsPythonButton.Name = "sessionTypeIsPythonButton"; + sessionTypeIsPythonButton.Size = new Size(93, 29); + sessionTypeIsPythonButton.TabIndex = 1; + sessionTypeIsPythonButton.TabStop = true; + sessionTypeIsPythonButton.Text = "Python"; + sessionTypeIsPythonButton.UseVisualStyleBackColor = true; + // + // finalPanel + // + finalPanel.Controls.Add(makeDefaultCheckBox); + finalPanel.Controls.Add(testResultsTextBox); + finalPanel.Controls.Add(testResultsLabel); + finalPanel.Controls.Add(testCredentialsButton); + finalPanel.Controls.Add(setCredentialsButton); + finalPanel.Dock = DockStyle.Bottom; + finalPanel.Location = new Point(0, 481); + finalPanel.Name = "finalPanel"; + finalPanel.Size = new Size(1054, 106); + finalPanel.TabIndex = 4; + // + // makeDefaultCheckBox + // + makeDefaultCheckBox.Anchor = AnchorStyles.Top | AnchorStyles.Right; + makeDefaultCheckBox.AutoSize = true; + makeDefaultCheckBox.Location = new Point(616, 53); + makeDefaultCheckBox.Name = "makeDefaultCheckBox"; + makeDefaultCheckBox.Size = new Size(143, 29); + makeDefaultCheckBox.TabIndex = 2; + makeDefaultCheckBox.Text = "Make Default"; + makeDefaultCheckBox.UseVisualStyleBackColor = true; + // + // testResultsTextBox + // + testResultsTextBox.Anchor = AnchorStyles.Top | AnchorStyles.Left | AnchorStyles.Right; + testResultsTextBox.Location = new Point(188, 13); + testResultsTextBox.Name = "testResultsTextBox"; + testResultsTextBox.ReadOnly = true; + testResultsTextBox.Size = new Size(801, 31); + testResultsTextBox.TabIndex = 7; + // + // testResultsLabel + // + testResultsLabel.AutoSize = true; + testResultsLabel.Location = new Point(157, 48); + testResultsLabel.Name = "testResultsLabel"; + testResultsLabel.Size = new Size(0, 25); + testResultsLabel.TabIndex = 6; + // + // testCredentialsButton + // + testCredentialsButton.Location = new Point(46, 11); + testCredentialsButton.Name = "testCredentialsButton"; + testCredentialsButton.Size = new Size(134, 34); + testCredentialsButton.TabIndex = 1; + testCredentialsButton.Text = "Test Creds"; + testCredentialsButton.UseVisualStyleBackColor = true; + testCredentialsButton.Click += testCredentialsButton_Click; + // + // setCredentialsButton + // + setCredentialsButton.Anchor = AnchorStyles.Top | AnchorStyles.Right; + setCredentialsButton.Location = new Point(789, 48); + setCredentialsButton.Name = "setCredentialsButton"; + setCredentialsButton.Size = new Size(200, 34); + setCredentialsButton.TabIndex = 3; + setCredentialsButton.Text = "Set Credentials"; + setCredentialsButton.UseVisualStyleBackColor = true; + setCredentialsButton.Click += setCredentialsButton_Click; + // + // isCorePlusRadioButton + // + isCorePlusRadioButton.AutoSize = true; + isCorePlusRadioButton.Location = new Point(192, 30); + isCorePlusRadioButton.Name = "isCorePlusRadioButton"; + isCorePlusRadioButton.Size = new Size(169, 29); + isCorePlusRadioButton.TabIndex = 3; + isCorePlusRadioButton.TabStop = true; + isCorePlusRadioButton.Text = "Enterprise Core+"; + isCorePlusRadioButton.UseVisualStyleBackColor = true; + // + // isCoreRadioButton + // + isCoreRadioButton.AutoSize = true; + isCoreRadioButton.Location = new Point(388, 30); + isCoreRadioButton.Name = "isCoreRadioButton"; + isCoreRadioButton.Size = new Size(172, 29); + isCoreRadioButton.TabIndex = 4; + isCoreRadioButton.TabStop = true; + isCoreRadioButton.Text = "Community Core"; + isCoreRadioButton.UseVisualStyleBackColor = true; + // + // endpointIdBox + // + endpointIdBox.Anchor = AnchorStyles.Top | AnchorStyles.Left | AnchorStyles.Right; + endpointIdBox.Location = new Point(194, 29); + endpointIdBox.Name = "endpointIdBox"; + endpointIdBox.Size = new Size(795, 31); + endpointIdBox.TabIndex = 1; + // + // connectionIdLabel + // + connectionIdLabel.AutoSize = true; + connectionIdLabel.Location = new Point(46, 29); + connectionIdLabel.Name = "connectionIdLabel"; + connectionIdLabel.Size = new Size(125, 25); + connectionIdLabel.TabIndex = 5; + connectionIdLabel.Text = "Connection ID"; + // + // connectionTypeGroup + // + connectionTypeGroup.Controls.Add(isCorePlusRadioButton); + connectionTypeGroup.Controls.Add(isCoreRadioButton); + connectionTypeGroup.Location = new Point(34, 84); + connectionTypeGroup.Name = "connectionTypeGroup"; + connectionTypeGroup.Size = new Size(588, 80); + connectionTypeGroup.TabIndex = 2; + connectionTypeGroup.TabStop = false; + connectionTypeGroup.Text = "Connection Type"; + // + // connectionIdPanel + // + connectionIdPanel.Controls.Add(connectionIdLabel); + connectionIdPanel.Controls.Add(endpointIdBox); + connectionIdPanel.Controls.Add(connectionTypeGroup); + connectionIdPanel.Dock = DockStyle.Top; + connectionIdPanel.Location = new Point(0, 0); + connectionIdPanel.Name = "connectionIdPanel"; + connectionIdPanel.Size = new Size(1054, 191); + connectionIdPanel.TabIndex = 1; + // + // CredentialsDialog + // + AutoScaleDimensions = new SizeF(10F, 25F); + AutoScaleMode = AutoScaleMode.Font; + ClientSize = new Size(1054, 587); + Controls.Add(finalPanel); + Controls.Add(corePanel); + Controls.Add(corePlusPanel); + Controls.Add(connectionIdPanel); + Name = "CredentialsDialog"; + Text = "Credentials Editor"; + corePlusPanel.ResumeLayout(false); + corePlusPanel.PerformLayout(); + corePanel.ResumeLayout(false); + corePanel.PerformLayout(); + groupBox1.ResumeLayout(false); + groupBox1.PerformLayout(); + finalPanel.ResumeLayout(false); + finalPanel.PerformLayout(); + connectionTypeGroup.ResumeLayout(false); + connectionTypeGroup.PerformLayout(); + connectionIdPanel.ResumeLayout(false); + connectionIdPanel.PerformLayout(); + ResumeLayout(false); + } + + #endregion + private RadioButton isCorePlusRadioButton; + private RadioButton isCoreRadioButton; + private Button setCredentialsButton; + private TextBox endpointIdBox; + private Label connectionIdLabel; + private GroupBox connectionTypeGroup; + private Panel corePlusPanel; + private Panel corePanel; + private Label jsonUrlLabel; + private Label label2; + private TextBox jsonUrlBox; + private TextBox connectionStringBox; + private Label passwordLabel; + private TextBox userIdBox; + private Label userIdLabel; + private TextBox operateAsBox; + private TextBox passwordBox; + private Panel finalPanel; + private Button testCredentialsButton; + private Label testResultsLabel; + private TextBox testResultsTextBox; + private CheckBox makeDefaultCheckBox; + private CheckBox validateCertCheckBox; + private GroupBox groupBox1; + private RadioButton sessionTypeIsGroovyButton; + private RadioButton sessionTypeIsPythonButton; + private Panel connectionIdPanel; + private Label label1; + } +} \ No newline at end of file diff --git a/csharp/ExcelAddIn/views/CredentialsDialog.cs b/csharp/ExcelAddIn/views/CredentialsDialog.cs new file mode 100644 index 00000000000..f3b664cab31 --- /dev/null +++ b/csharp/ExcelAddIn/views/CredentialsDialog.cs @@ -0,0 +1,76 @@ +using Deephaven.ExcelAddIn.ViewModels; + +namespace ExcelAddIn.views { + public partial class CredentialsDialog : Form { + public event Action? OnSetCredentialsButtonClicked = null; + public event Action? OnTestCredentialsButtonClicked = null; + + public CredentialsDialog(CredentialsDialogViewModel vm) { + InitializeComponent(); + // Need to fire these bindings on property changed rather than simply on validation, + // because on validation is not responsive enough. Also, painful technical note: + // being a member of connectionTypeGroup *also* ensures that at most one of these buttons + // are checked. So you might think databinding is not necessary. However being in + // a group does nothing for the initial conditions. So the group doesn't care if + // *neither* of them are checked. + isCorePlusRadioButton.DataBindings.Add(new Binding(nameof(isCorePlusRadioButton.Checked), + vm, nameof(vm.IsCorePlus), false, DataSourceUpdateMode.OnPropertyChanged)); + isCoreRadioButton.DataBindings.Add(new Binding(nameof(isCorePlusRadioButton.Checked), + vm, nameof(vm.IsCore), false, DataSourceUpdateMode.OnPropertyChanged)); + + // Make one of the two panels visible, according to the setting of the radio box. + corePlusPanel.DataBindings.Add(nameof(corePlusPanel.Visible), vm, nameof(vm.IsCorePlus)); + corePanel.DataBindings.Add(nameof(corePanel.Visible), vm, nameof(vm.IsCore)); + + // Bind the ID + endpointIdBox.DataBindings.Add(nameof(endpointIdBox.Text), vm, nameof(vm.Id)); + + // Bind the Core+ properties + jsonUrlBox.DataBindings.Add(nameof(jsonUrlBox.Text), vm, nameof(vm.JsonUrl)); + userIdBox.DataBindings.Add(nameof(userIdBox.Text), vm, nameof(vm.UserId)); + passwordBox.DataBindings.Add(nameof(passwordBox.Text), vm, nameof(vm.Password)); + + operateAsBox.DataBindings.Add(nameof(operateAsBox.Text), vm, nameof(vm.OperateAs)); + + validateCertCheckBox.DataBindings.Add(nameof(validateCertCheckBox.Checked), vm, nameof(vm.ValidateCertificate)); + + // Bind the Core property (there's just one) + connectionStringBox.DataBindings.Add(nameof(connectionStringBox.Text), + vm, nameof(vm.ConnectionString)); + + // Bind the SessionType checkboxes + sessionTypeIsPythonButton.DataBindings.Add(nameof(sessionTypeIsPythonButton.Checked), + vm, nameof(vm.SessionTypeIsPython)); + sessionTypeIsGroovyButton.DataBindings.Add(nameof(sessionTypeIsGroovyButton.Checked), + vm, nameof(vm.SessionTypeIsGroovy)); + + // Bind the IsDefault property + makeDefaultCheckBox.DataBindings.Add(nameof(makeDefaultCheckBox.Checked), + vm, nameof(vm.IsDefault)); + } + + public void SetTestResultsBox(string testResultsState) { + if (InvokeRequired) { + try { + Invoke(() => SetTestResultsBox(testResultsState)); + } catch (InvalidOperationException) { + // TODO(kosak): figure out a better way to handle this race + // Invoke will fail if it is called after the form was closed. There is a race + // here that is pretty hard to get rid of. For now we catch the exception and + // throw away the value. + } + return; + } + + testResultsTextBox.Text = testResultsState; + } + + private void setCredentialsButton_Click(object sender, EventArgs e) { + OnSetCredentialsButtonClicked?.Invoke(); + } + + private void testCredentialsButton_Click(object sender, EventArgs e) { + OnTestCredentialsButtonClicked?.Invoke(); + } + } +} diff --git a/csharp/ExcelAddIn/views/CredentialsDialog.resx b/csharp/ExcelAddIn/views/CredentialsDialog.resx new file mode 100644 index 00000000000..4f24d55cd6b --- /dev/null +++ b/csharp/ExcelAddIn/views/CredentialsDialog.resx @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + \ No newline at end of file diff --git a/csharp/ExcelAddIn/views/DeephavenMessageBox.Designer.cs b/csharp/ExcelAddIn/views/DeephavenMessageBox.Designer.cs new file mode 100644 index 00000000000..624a860a9af --- /dev/null +++ b/csharp/ExcelAddIn/views/DeephavenMessageBox.Designer.cs @@ -0,0 +1,111 @@ +namespace ExcelAddIn.views { + partial class DeephavenMessageBox { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) { + if (disposing && (components != null)) { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() { + captionLabel = new Label(); + contentsBox = new TextBox(); + okButton = new Button(); + captionPanel = new Panel(); + cancelButton = new Button(); + captionPanel.SuspendLayout(); + SuspendLayout(); + // + // captionLabel + // + captionLabel.Dock = DockStyle.Fill; + captionLabel.Font = new Font("Segoe UI", 20F, FontStyle.Regular, GraphicsUnit.Point, 0); + captionLabel.Location = new Point(0, 0); + captionLabel.Name = "captionLabel"; + captionLabel.Size = new Size(751, 73); + captionLabel.TabIndex = 0; + captionLabel.Text = "Caption"; + captionLabel.TextAlign = ContentAlignment.MiddleCenter; + // + // contentsBox + // + contentsBox.Anchor = AnchorStyles.Top | AnchorStyles.Bottom | AnchorStyles.Left | AnchorStyles.Right; + contentsBox.Location = new Point(40, 105); + contentsBox.Multiline = true; + contentsBox.Name = "contentsBox"; + contentsBox.ReadOnly = true; + contentsBox.Size = new Size(716, 207); + contentsBox.TabIndex = 1; + contentsBox.TabStop = false; + // + // okButton + // + okButton.Anchor = AnchorStyles.Bottom | AnchorStyles.Right; + okButton.Location = new Point(644, 343); + okButton.Name = "okButton"; + okButton.Size = new Size(112, 34); + okButton.TabIndex = 2; + okButton.Text = "OK"; + okButton.UseVisualStyleBackColor = true; + okButton.Click += okButton_Click; + // + // captionPanel + // + captionPanel.Anchor = AnchorStyles.Top | AnchorStyles.Left | AnchorStyles.Right; + captionPanel.Controls.Add(captionLabel); + captionPanel.Location = new Point(24, 12); + captionPanel.Name = "captionPanel"; + captionPanel.Size = new Size(751, 73); + captionPanel.TabIndex = 3; + // + // cancelButton + // + cancelButton.Anchor = AnchorStyles.Bottom | AnchorStyles.Right; + cancelButton.Location = new Point(513, 343); + cancelButton.Name = "cancelButton"; + cancelButton.Size = new Size(112, 34); + cancelButton.TabIndex = 4; + cancelButton.Text = "Cancel"; + cancelButton.UseVisualStyleBackColor = true; + cancelButton.Click += cancelButton_Click; + // + // DeephavenMessageBox + // + AutoScaleDimensions = new SizeF(10F, 25F); + AutoScaleMode = AutoScaleMode.Font; + ClientSize = new Size(800, 401); + Controls.Add(cancelButton); + Controls.Add(captionPanel); + Controls.Add(okButton); + Controls.Add(contentsBox); + Name = "DeephavenMessageBox"; + Text = "Deephaven Message"; + captionPanel.ResumeLayout(false); + ResumeLayout(false); + PerformLayout(); + } + + #endregion + + private Label captionLabel; + private TextBox contentsBox; + private Button okButton; + private Panel captionPanel; + private Button cancelButton; + } +} \ No newline at end of file diff --git a/csharp/ExcelAddIn/views/DeephavenMessageBox.cs b/csharp/ExcelAddIn/views/DeephavenMessageBox.cs new file mode 100644 index 00000000000..8476f689b05 --- /dev/null +++ b/csharp/ExcelAddIn/views/DeephavenMessageBox.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace ExcelAddIn.views { + public partial class DeephavenMessageBox : Form { + public DeephavenMessageBox(string caption, string text, bool cancelVisible) { + InitializeComponent(); + + captionLabel.Text = caption; + contentsBox.Text = text; + cancelButton.Visible = cancelVisible; + + AcceptButton = AcceptButton; + CancelButton = cancelButton; + } + + private void okButton_Click(object sender, EventArgs e) { + DialogResult = DialogResult.OK; + Close(); + } + + private void cancelButton_Click(object sender, EventArgs e) { + DialogResult = DialogResult.Cancel; + Close(); + } + } +} diff --git a/csharp/ExcelAddIn/views/DeephavenMessageBox.resx b/csharp/ExcelAddIn/views/DeephavenMessageBox.resx new file mode 100644 index 00000000000..4f24d55cd6b --- /dev/null +++ b/csharp/ExcelAddIn/views/DeephavenMessageBox.resx @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + \ No newline at end of file diff --git a/csharp/client/DeephavenClient/Client.cs b/csharp/client/DeephavenClient/Client.cs index f4439a1caae..887f51448e1 100644 --- a/csharp/client/DeephavenClient/Client.cs +++ b/csharp/client/DeephavenClient/Client.cs @@ -17,13 +17,13 @@ public static Client Connect(string target, ClientOptions options) { return new Client(clientResult, manager); } - private Client(NativePtr self, TableHandleManager manager) { + private protected Client(NativePtr self, TableHandleManager manager) { Self = self; Manager = manager; } ~Client() { - ReleaseUnmanagedResources(); + ReleaseUnmanagedResources(true); } public void Close() { @@ -31,14 +31,17 @@ public void Close() { } public void Dispose() { - ReleaseUnmanagedResources(); + ReleaseUnmanagedResources(true); GC.SuppressFinalize(this); } - private void ReleaseUnmanagedResources() { + protected virtual void ReleaseUnmanagedResources(bool destructSelf) { if (!Self.TryRelease(out var old)) { return; } + if (!destructSelf) { + return; + } NativeClient.deephaven_client_Client_dtor(old); } } diff --git a/csharp/client/DeephavenClient/DeephavenClient.csproj b/csharp/client/DeephavenClient/DeephavenClient.csproj index 707e4dad819..ac6d099d291 100644 --- a/csharp/client/DeephavenClient/DeephavenClient.csproj +++ b/csharp/client/DeephavenClient/DeephavenClient.csproj @@ -1,10 +1,16 @@  - net7.0 + net8.0 enable enable True + + + + + + diff --git a/csharp/client/DeephavenClient/TableHandleManager.cs b/csharp/client/DeephavenClient/TableHandleManager.cs index 09af0ba87fc..01dc64660b0 100644 --- a/csharp/client/DeephavenClient/TableHandleManager.cs +++ b/csharp/client/DeephavenClient/TableHandleManager.cs @@ -3,7 +3,7 @@ namespace Deephaven.DeephavenClient; -public sealed class TableHandleManager : IDisposable { +public class TableHandleManager : IDisposable { internal NativePtr Self; private readonly Dictionary _subscriptions; @@ -13,18 +13,21 @@ internal TableHandleManager(NativePtr self) { } ~TableHandleManager() { - ReleaseUnmanagedResources(); + ReleaseUnmanagedResources(true); } public void Dispose() { - ReleaseUnmanagedResources(); + ReleaseUnmanagedResources(true); GC.SuppressFinalize(this); } - private void ReleaseUnmanagedResources() { + protected virtual void ReleaseUnmanagedResources(bool destructSelf) { if (!Self.TryRelease(out var old)) { return; } + if (!destructSelf) { + return; + } NativeTableHandleManager.deephaven_client_TableHandleManager_dtor(old); } diff --git a/csharp/client/DeephavenClient/dhe_client/session/DndClient.cs b/csharp/client/DeephavenClient/dhe_client/session/DndClient.cs new file mode 100644 index 00000000000..e6db330881e --- /dev/null +++ b/csharp/client/DeephavenClient/dhe_client/session/DndClient.cs @@ -0,0 +1,57 @@ +using System.Runtime.InteropServices; +using Deephaven.DeephavenClient; +using Deephaven.DeephavenClient.Interop; + +namespace Deephaven.DheClient.Session; + +public sealed class DndClient : Client { + internal new NativePtr Self; + public Int64 PqSerial; + + internal static DndClient OfNativePtr(NativePtr dndClient) { + NativeDndClient.deephaven_enterprise_session_DndClient_GetManager(dndClient, + out var dndManagerResult, out var status1); + status1.OkOrThrow(); + NativeDndClient.deephaven_enterprise_session_DndClient_PqSerial(dndClient, + out var pqSerial, out var status2); + status2.OkOrThrow(); + var dndManager = new DndTableHandleManager(dndManagerResult); + + return new DndClient(dndClient, dndManager, pqSerial); + } + + private DndClient(NativePtr self, DndTableHandleManager manager, + Int64 pqSerial) + : base(self.UnsafeCast(), manager) { + Self = self; + PqSerial = pqSerial; + } + + protected override void ReleaseUnmanagedResources(bool destructSelf) { + base.ReleaseUnmanagedResources(false); + if (!Self.TryRelease(out var old)) { + return; + } + if (!destructSelf) { + return; + } + NativeDndClient.deephaven_enterprise_session_DndClient_dtor(old); + } +} + + +internal partial class NativeDndClient { + [LibraryImport(LibraryPaths.DhEnterprise, StringMarshalling = StringMarshalling.Utf8)] + public static partial void deephaven_enterprise_session_DndClient_dtor( + NativePtr self); + + [LibraryImport(LibraryPaths.DhEnterprise, StringMarshalling = StringMarshalling.Utf8)] + public static partial void deephaven_enterprise_session_DndClient_GetManager( + NativePtr self, + out NativePtr result, out ErrorStatus status); + + [LibraryImport(LibraryPaths.DhEnterprise, StringMarshalling = StringMarshalling.Utf8)] + public static partial void deephaven_enterprise_session_DndClient_PqSerial( + NativePtr self, + out Int64 result, out ErrorStatus status); +} diff --git a/csharp/client/DeephavenClient/dhe_client/session/DndTableHandleManager.cs b/csharp/client/DeephavenClient/dhe_client/session/DndTableHandleManager.cs new file mode 100644 index 00000000000..59c88df657e --- /dev/null +++ b/csharp/client/DeephavenClient/dhe_client/session/DndTableHandleManager.cs @@ -0,0 +1,32 @@ +using System.Runtime.InteropServices; +using Deephaven.DeephavenClient; +using Deephaven.DeephavenClient.Interop; + +namespace Deephaven.DheClient.Session; + +public sealed class DndTableHandleManager : TableHandleManager { + internal new NativePtr Self; + + internal DndTableHandleManager(NativePtr self) : + base(self.UnsafeCast()) { + Self = self; + } + + protected override void ReleaseUnmanagedResources(bool destructSelf) { + base.ReleaseUnmanagedResources(false); + if (!Self.TryRelease(out var old)) { + return; + } + if (!destructSelf) { + return; + } + NativeDndTableHandleManager.deephaven_enterprise_session_DndTableHandleManager_dtor(old); + } + +} + +internal partial class NativeDndTableHandleManager { + [LibraryImport(LibraryPaths.DhEnterprise, StringMarshalling = StringMarshalling.Utf8)] + public static partial void deephaven_enterprise_session_DndTableHandleManager_dtor( + NativePtr self); +} diff --git a/csharp/client/DeephavenClient/dhe_client/session/SessionManager.cs b/csharp/client/DeephavenClient/dhe_client/session/SessionManager.cs new file mode 100644 index 00000000000..e86f9d9903b --- /dev/null +++ b/csharp/client/DeephavenClient/dhe_client/session/SessionManager.cs @@ -0,0 +1,84 @@ +using System.Runtime.InteropServices; +using Deephaven.DeephavenClient.Interop; + +namespace Deephaven.DheClient.Session; + +public class SessionManager : IDisposable { + internal NativePtr Self; + + public static SessionManager FromUrl(string descriptiveName, string jsonUrl) { + NativeSessionManager.deephaven_enterprise_session_SessionManager_FromUrl(descriptiveName, + jsonUrl, out var sessionResult, out var status); + status.OkOrThrow(); + return new SessionManager(sessionResult); + } + + public static SessionManager FromJson(string descriptiveName, string json) { + NativeSessionManager.deephaven_enterprise_session_SessionManager_FromJson(descriptiveName, + json, out var sessionResult, out var status); + status.OkOrThrow(); + return new SessionManager(sessionResult); + } + + private SessionManager(NativePtr self) { + Self = self; + } + + ~SessionManager() { + ReleaseUnmanagedResources(); + } + + public void Close() { + Dispose(); + } + + public void Dispose() { + ReleaseUnmanagedResources(); + GC.SuppressFinalize(this); + } + + public bool PasswordAuthentication(string user, string password, string operateAs) { + NativeSessionManager.deephaven_enterprise_session_SessionManager_PasswordAuthentication( + Self, user, password, operateAs, out var result, out var status); + status.OkOrThrow(); + return (bool)result; + } + + public DndClient ConnectToPqByName(string pqName, bool removeOnClose) { + NativeSessionManager.deephaven_enterprise_session_SessionManager_ConnectToPqByName( + Self, pqName, (InteropBool)removeOnClose, out var result, out var status); + status.OkOrThrow(); + return DndClient.OfNativePtr(result); + } + + private void ReleaseUnmanagedResources() { + if (!Self.TryRelease(out var old)) { + return; + } + NativeSessionManager.deephaven_enterprise_session_SessionManager_dtor(old); + } +} + +internal partial class NativeSessionManager { + [LibraryImport(LibraryPaths.DhEnterprise, StringMarshalling = StringMarshalling.Utf8)] + public static partial void deephaven_enterprise_session_SessionManager_dtor( + NativePtr self); + + [LibraryImport(LibraryPaths.DhEnterprise, StringMarshalling = StringMarshalling.Utf8)] + public static partial void deephaven_enterprise_session_SessionManager_FromUrl(string descriptiveName, + string jsonUrl, out NativePtr result, out ErrorStatus status); + + [LibraryImport(LibraryPaths.DhEnterprise, StringMarshalling = StringMarshalling.Utf8)] + public static partial void deephaven_enterprise_session_SessionManager_FromJson(string descriptiveName, + string json, out NativePtr result, out ErrorStatus status); + + [LibraryImport(LibraryPaths.DhEnterprise, StringMarshalling = StringMarshalling.Utf8)] + public static partial void deephaven_enterprise_session_SessionManager_PasswordAuthentication( + NativePtr self, string user, string password, string operateAs, + out InteropBool result, out ErrorStatus status); + + [LibraryImport(LibraryPaths.DhEnterprise, StringMarshalling = StringMarshalling.Utf8)] + public static partial void deephaven_enterprise_session_SessionManager_ConnectToPqByName( + NativePtr self, string pqName, InteropBool removeOnClose, + out NativePtr result, out ErrorStatus status); +} diff --git a/csharp/client/DeephavenClient/Interop/InteropSupport.cs b/csharp/client/DeephavenClient/interop/InteropSupport.cs similarity index 90% rename from csharp/client/DeephavenClient/Interop/InteropSupport.cs rename to csharp/client/DeephavenClient/interop/InteropSupport.cs index 83684b28c90..f6e67a3a311 100644 --- a/csharp/client/DeephavenClient/Interop/InteropSupport.cs +++ b/csharp/client/DeephavenClient/interop/InteropSupport.cs @@ -5,9 +5,11 @@ namespace Deephaven.DeephavenClient.Interop; -internal class LibraryPaths { - internal const string Dhcore = "dhcore"; - internal const string Dhclient = "dhclient"; +public class LibraryPaths { + public const string Dhcore = "dhcore"; + public const string Dhclient = "dhclient"; + // public const string DhEnterprise = @"dhe_client"; // does not work + public const string DhEnterprise = @"dhe_client.dll"; // works } /// @@ -32,6 +34,10 @@ public bool TryRelease(out NativePtr oldPtr) { return true; } + public NativePtr UnsafeCast() { + return new NativePtr(ptr); + } + public readonly bool IsNull => ptr == IntPtr.Zero; } diff --git a/csharp/client/DeephavenClient/Interop/TestApi/BasicInteropInteractions.cs b/csharp/client/DeephavenClient/interop/TestApi/BasicInteropInteractions.cs similarity index 100% rename from csharp/client/DeephavenClient/Interop/TestApi/BasicInteropInteractions.cs rename to csharp/client/DeephavenClient/interop/TestApi/BasicInteropInteractions.cs diff --git a/csharp/client/DeephavenClient/Utility/ColumnFactory.cs b/csharp/client/DeephavenClient/utility/ColumnFactory.cs similarity index 100% rename from csharp/client/DeephavenClient/Utility/ColumnFactory.cs rename to csharp/client/DeephavenClient/utility/ColumnFactory.cs diff --git a/csharp/client/DeephavenClient/Utility/DeephavenConstants.cs b/csharp/client/DeephavenClient/utility/DeephavenConstants.cs similarity index 100% rename from csharp/client/DeephavenClient/Utility/DeephavenConstants.cs rename to csharp/client/DeephavenClient/utility/DeephavenConstants.cs diff --git a/csharp/client/DeephavenClient/Utility/Schema.cs b/csharp/client/DeephavenClient/utility/Schema.cs similarity index 100% rename from csharp/client/DeephavenClient/Utility/Schema.cs rename to csharp/client/DeephavenClient/utility/Schema.cs diff --git a/csharp/client/DeephavenClient/Utility/TableMaker.cs b/csharp/client/DeephavenClient/utility/TableMaker.cs similarity index 100% rename from csharp/client/DeephavenClient/Utility/TableMaker.cs rename to csharp/client/DeephavenClient/utility/TableMaker.cs diff --git a/deephaven-changelog b/deephaven-changelog new file mode 100644 index 00000000000..2b1048e34d5 --- /dev/null +++ b/deephaven-changelog @@ -0,0 +1,51 @@ +{# Tera templates are used to generate the changelog content -#} +{# https://keats.github.io/tera/docs/ -#} +{# Based on Cocogittos remote template, but adds breaking changes: https://github.com/cocogitto/cocogitto/blob/main/src/conventional/changelog/template/remote -#} +{# First display all the breaking changes -#} +{% set breaking_commits = commits | filter(attribute="breaking_change", value=true) -%} +{% if breaking_commits | length > 0 -%} +#### ⚠ Breaking Changes + +{% for commit in breaking_commits -%} +{% set commit_link = repository_url ~ "/commit/" ~ commit.id -%} +{% set shorthand = commit.id | truncate(length=7, end="") -%} +{% for footer in commit.footer | filter(attribute="token", value="BREAKING CHANGE") -%} +- {{ footer.content }} - ([{{shorthand}}]({{ commit_link }})) +{% endfor -%} +{% endfor -%} +{% endif %} + +{# Now group the rest of the commits and display them -#} +{% for type, typed_commits in commits | sort(attribute="type")| group_by(attribute="type") -%} +#### {{ type | upper_first }} +{% for scope, scoped_commits in typed_commits | group_by(attribute="scope") -%} + +{% for commit in scoped_commits | sort(attribute="scope") -%} + {% if commit.author and repository_url -%} + {% set author = "@" ~ commit.author -%} + {% set author_link = platform ~ "/" ~ commit.author -%} + {% set author = "[" ~ author ~ "](" ~ author_link ~ ")" -%} + {% else -%} + {% set author = commit.signature -%} + {% endif -%} + {% set commit_link = repository_url ~ "/commit/" ~ commit.id -%} + {% set shorthand = commit.id | truncate(length=7, end="") -%} + - **({{ scope }})** {{ commit.summary }} - ([{{shorthand}}]({{ commit_link }})) - {{ author }} +{% endfor -%} + +{% endfor -%} + +{% for commit in typed_commits | unscoped -%} + {% if commit.author and repository_url -%} + {% set author = "@" ~ commit.author -%} + {% set author_link = platform ~ "/" ~ commit.author -%} + {% set author = "[" ~ author ~ "](" ~ author_link ~ ")" -%} + {% else -%} + {% set author = commit.signature -%} + {% endif -%} + {% set commit_link = repository_url ~ "/commit/" ~ commit.id -%} + {% set shorthand = commit.id | truncate(length=7, end="") -%} + - {{ commit.summary }} - ([{{shorthand}}]({{ commit_link }})) - {{ author }} +{% endfor -%} + +{% endfor -%} \ No newline at end of file diff --git a/docker/registry/cpp-clients-multi-base/gradle.properties b/docker/registry/cpp-clients-multi-base/gradle.properties index 79a0de4f28c..3e482e0ab78 100644 --- a/docker/registry/cpp-clients-multi-base/gradle.properties +++ b/docker/registry/cpp-clients-multi-base/gradle.properties @@ -1,6 +1,5 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=ghcr.io/deephaven/cpp-clients-multi-base:latest # When the image version changes, remember to update cpp-client/build-dependencies.sh to match the image's content -#deephaven.registry.imageId=ghcr.io/deephaven/cpp-clients-multi-base@sha256:5b39f286d94a335890314577faf9211bd1b72299efa328f917ddab92c99a437e -deephaven.registry.imageId=ghcr.io/deephaven/cpp-clients-multi-base:latest +deephaven.registry.imageId=ghcr.io/deephaven/cpp-clients-multi-base@sha256:78e114d9523d02ccd4c0f7f2ce1ef4deb5ce4d653b6fdc6aaf1f14f4ceb2357a deephaven.registry.platform=linux/amd64 diff --git a/docker/registry/fedora/gradle.properties b/docker/registry/fedora/gradle.properties index 2068e113e91..c7ef2bb21b0 100644 --- a/docker/registry/fedora/gradle.properties +++ b/docker/registry/fedora/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=fedora:39 -deephaven.registry.imageId=fedora@sha256:e8792bee618a8d1c2ca8fbcf641ceb828d1b6b69bfac1ff70792f8bd5ed10ddd +deephaven.registry.imageId=fedora@sha256:2922a1237abbb7f8517018e4f5d7a82a618f6ec09f386799e8595f9e1c39f021 diff --git a/docker/registry/go/gradle.properties b/docker/registry/go/gradle.properties index f5c58465c3f..1406a8fa3d1 100644 --- a/docker/registry/go/gradle.properties +++ b/docker/registry/go/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=golang:1 -deephaven.registry.imageId=golang@sha256:829eff99a4b2abffe68f6a3847337bf6455d69d17e49ec1a97dac78834754bd6 +deephaven.registry.imageId=golang@sha256:613a108a4a4b1dfb6923305db791a19d088f77632317cfc3446825c54fb862cd diff --git a/docker/registry/localstack/gradle.properties b/docker/registry/localstack/gradle.properties index 783af51a431..275f102fbd5 100644 --- a/docker/registry/localstack/gradle.properties +++ b/docker/registry/localstack/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=localstack/localstack:3 -deephaven.registry.imageId=localstack/localstack@sha256:54fcf172f6ff70909e1e26652c3bb4587282890aff0d02c20aa7695469476ac0 +deephaven.registry.imageId=localstack/localstack@sha256:231148e6d60d040441ee0b418ab181eaedf30d18bca23ce5b44dfb863c40fb7c diff --git a/docker/registry/manylinux2014_x86_64/gradle.properties b/docker/registry/manylinux2014_x86_64/gradle.properties index ee6d3d3a0a7..b11f6c7070a 100644 --- a/docker/registry/manylinux2014_x86_64/gradle.properties +++ b/docker/registry/manylinux2014_x86_64/gradle.properties @@ -1,4 +1,4 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=quay.io/pypa/manylinux2014_x86_64:latest -deephaven.registry.imageId=quay.io/pypa/manylinux2014_x86_64@sha256:07a0f9d59d8d075bb4f84dffc5dd76d991b90d7e4f315d1ef9f48a4f04390b71 +deephaven.registry.imageId=quay.io/pypa/manylinux2014_x86_64@sha256:545ae43918a0b5a871a09ccb4421c68230a109a21d99cd6f4806b2e4c34543d2 deephaven.registry.platform=linux/amd64 diff --git a/docker/registry/minio/gradle.properties b/docker/registry/minio/gradle.properties index 6d8fedc8c19..1ab1b39c1d1 100644 --- a/docker/registry/minio/gradle.properties +++ b/docker/registry/minio/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=minio/minio:latest -deephaven.registry.imageId=minio/minio@sha256:77ff9f7e12549d269990b167fa21da010fa8b0beb40d6064569b8887e37c456b +deephaven.registry.imageId=minio/minio@sha256:6f23072e3e222e64fe6f86b31a7f7aca971e5129e55cbccef649b109b8e651a1 diff --git a/docker/registry/protoc-base/gradle.properties b/docker/registry/protoc-base/gradle.properties index 14302326f6d..7c534487e3a 100644 --- a/docker/registry/protoc-base/gradle.properties +++ b/docker/registry/protoc-base/gradle.properties @@ -1,5 +1,5 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=ghcr.io/deephaven/protoc-base:latest -deephaven.registry.imageId=ghcr.io/deephaven/protoc-base@sha256:a7f819be945c058a96079960c94fea0ca04c6189aa92c0ed45fcf4131ecbbf23 +deephaven.registry.imageId=ghcr.io/deephaven/protoc-base@sha256:7b43037ce0cf2a82aff73605a689754620791238ed598b7ae41aa5b225a27ccf # TODO(deephaven-base-images#54): arm64 native image for cpp-client-base deephaven.registry.platform=linux/amd64 diff --git a/docker/registry/python/gradle.properties b/docker/registry/python/gradle.properties index 3b94db0e04e..0cec680f6ef 100644 --- a/docker/registry/python/gradle.properties +++ b/docker/registry/python/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=python:3.10 -deephaven.registry.imageId=python@sha256:d47cb1b09be409a0cb5efeb01dd3ff5d44a5e28d371fa1a736c8928ab72fffd2 +deephaven.registry.imageId=python@sha256:09447073b9603858602b4a725ad92e4fd4c8f6979768cee295afae67fd997718 diff --git a/docker/registry/server-base/gradle.properties b/docker/registry/server-base/gradle.properties index 75412cde87f..a9314887d1b 100644 --- a/docker/registry/server-base/gradle.properties +++ b/docker/registry/server-base/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=ghcr.io/deephaven/server-base:edge -deephaven.registry.imageId=ghcr.io/deephaven/server-base@sha256:3304ec00e18bf86aee18b3491c7d92b326773deacfd98d1556facaddb1957a63 +deephaven.registry.imageId=ghcr.io/deephaven/server-base@sha256:66f8cecdac170dfb8bf284e41684480359a15443dc5a5b8a1efc95987f3ddcb5 diff --git a/docker/registry/slim-base/gradle.properties b/docker/registry/slim-base/gradle.properties index cc812a7b768..61745e6630c 100644 --- a/docker/registry/slim-base/gradle.properties +++ b/docker/registry/slim-base/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=ghcr.io/deephaven/server-slim-base:edge -deephaven.registry.imageId=ghcr.io/deephaven/server-slim-base@sha256:ba5ce0113b246c50179e4355564f202840319f09a95cd921158c9042a528e03f +deephaven.registry.imageId=ghcr.io/deephaven/server-slim-base@sha256:0605678d4961227fa0a8e83c0c8b030f56e581c070504dcca49a44a9ab43d8f0 diff --git a/docker/server-jetty/src/main/server-jetty/requirements.txt b/docker/server-jetty/src/main/server-jetty/requirements.txt index a80149cbaaa..72c871eb253 100644 --- a/docker/server-jetty/src/main/server-jetty/requirements.txt +++ b/docker/server-jetty/src/main/server-jetty/requirements.txt @@ -2,7 +2,7 @@ adbc-driver-manager==1.1.0 adbc-driver-postgresql==1.1.0 connectorx==0.3.3; platform.machine == 'x86_64' deephaven-plugin==0.6.0 -importlib_resources==6.4.0 +importlib_resources==6.4.3 java-utilities==0.3.0 jedi==0.19.1 jpy==0.18.0 diff --git a/docker/server/src/main/server-netty/requirements.txt b/docker/server/src/main/server-netty/requirements.txt index a80149cbaaa..72c871eb253 100644 --- a/docker/server/src/main/server-netty/requirements.txt +++ b/docker/server/src/main/server-netty/requirements.txt @@ -2,7 +2,7 @@ adbc-driver-manager==1.1.0 adbc-driver-postgresql==1.1.0 connectorx==0.3.3; platform.machine == 'x86_64' deephaven-plugin==0.6.0 -importlib_resources==6.4.0 +importlib_resources==6.4.3 java-utilities==0.3.0 jedi==0.19.1 jpy==0.18.0 diff --git a/engine/chunk/src/main/resources/io/deephaven/chunk/Chunk.gwt.xml b/engine/chunk/src/main/resources/io/deephaven/chunk/Chunk.gwt.xml new file mode 100644 index 00000000000..1f53956d9ae --- /dev/null +++ b/engine/chunk/src/main/resources/io/deephaven/chunk/Chunk.gwt.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/engine/primitive/src/main/java/io/deephaven/engine/primitive/iterator/CloseablePrimitiveIterator.java b/engine/primitive/src/main/java/io/deephaven/engine/primitive/iterator/CloseablePrimitiveIterator.java index 7f29483e1e8..41fe961e769 100644 --- a/engine/primitive/src/main/java/io/deephaven/engine/primitive/iterator/CloseablePrimitiveIterator.java +++ b/engine/primitive/src/main/java/io/deephaven/engine/primitive/iterator/CloseablePrimitiveIterator.java @@ -5,13 +5,7 @@ import io.deephaven.util.SafeCloseable; -import java.util.Iterator; import java.util.PrimitiveIterator; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.stream.IntStream; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; /** * Interface for {@link SafeCloseable closeable} {@link PrimitiveIterator primitive iterators}. diff --git a/engine/primitive/src/main/resources/io/deephaven/engine/primitive/function/Function.gwt.xml b/engine/primitive/src/main/resources/io/deephaven/engine/primitive/function/Function.gwt.xml new file mode 100644 index 00000000000..c78151ef021 --- /dev/null +++ b/engine/primitive/src/main/resources/io/deephaven/engine/primitive/function/Function.gwt.xml @@ -0,0 +1,3 @@ + + + diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/AbstractFilterExecution.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/AbstractFilterExecution.java index 3659e8bb530..6c388b3ee7d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/AbstractFilterExecution.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/AbstractFilterExecution.java @@ -190,12 +190,12 @@ private void doFilterParallel( // Clean up the row sets created by the filter. try (final RowSet ignored = adds; final RowSet ignored2 = mods) { - if (addedResult != null) { + if (addedResult != null && adds != null) { synchronized (addedResult) { addedResult.insert(adds); } } - if (modifiedResult != null) { + if (modifiedResult != null && mods != null) { synchronized (modifiedResult) { modifiedResult.insert(mods); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/InstrumentedTableListenerBase.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/InstrumentedTableListenerBase.java index acdf3605ac6..c14d5eaf20f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/InstrumentedTableListenerBase.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/InstrumentedTableListenerBase.java @@ -50,7 +50,7 @@ public abstract class InstrumentedTableListenerBase extends LivenessArtifact private final PerformanceEntry entry; private final boolean terminalListener; - private boolean failed = false; + protected boolean failed = false; private static volatile boolean verboseLogging = Configuration .getInstance() .getBooleanWithDefault("InstrumentedTableListenerBase.verboseLogging", false); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/MergedListener.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/MergedListener.java index 3a5b39e898e..6b25ddfbce0 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/MergedListener.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/MergedListener.java @@ -57,6 +57,9 @@ public abstract class MergedListener extends LivenessArtifact implements Notific protected final PerformanceEntry entry; private final String logPrefix; + private boolean failed; + + @SuppressWarnings("FieldMayBeFinal") private volatile long lastCompletedStep = NotificationStepReceiver.NULL_NOTIFICATION_STEP; private volatile long lastEnqueuedStep = NotificationStepReceiver.NULL_NOTIFICATION_STEP; @@ -96,6 +99,10 @@ protected Iterable getRecorders() { return recorders; } + public boolean isFailed() { + return failed; + } + public final void notifyOnUpstreamError( @NotNull final Throwable upstreamError, @Nullable final TableListener.Entry errorSourceEntry) { notifyInternal(upstreamError, errorSourceEntry); @@ -107,6 +114,10 @@ public void notifyChanges() { private void notifyInternal(@Nullable final Throwable upstreamError, @Nullable final TableListener.Entry errorSourceEntry) { + if (failed) { + return; + } + final long currentStep = getUpdateGraph().clock().currentStep(); synchronized (this) { @@ -150,6 +161,7 @@ protected void propagateError( final boolean uncaughtExceptionFromProcess, @NotNull final Throwable error, @Nullable final TableListener.Entry entry) { + failed = true; forceReferenceCountToZero(); propagateErrorDownstream(uncaughtExceptionFromProcess, error, entry); try { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java index ae321271be4..49e6e2e6b14 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java @@ -4,12 +4,11 @@ package io.deephaven.engine.table.impl; import io.deephaven.UncheckedDeephavenException; -import io.deephaven.api.util.NameValidator; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.context.QueryCompilerRequest; -import io.deephaven.engine.context.QueryScope; import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; +import io.deephaven.engine.table.impl.select.codegen.FormulaAnalyzer; import io.deephaven.util.MultiException; import io.deephaven.util.SafeCloseable; import io.deephaven.util.CompletionStageFuture; @@ -18,43 +17,43 @@ import org.jetbrains.annotations.VisibleForTesting; import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -public interface QueryCompilerRequestProcessor { +public abstract class QueryCompilerRequestProcessor { /** * @return An immediate QueryCompilerRequestProcessor */ - static QueryCompilerRequestProcessor.ImmediateProcessor immediate() { + public static QueryCompilerRequestProcessor.ImmediateProcessor immediate() { return new ImmediateProcessor(); } /** * @return A batch QueryCompilerRequestProcessor */ - static QueryCompilerRequestProcessor.BatchProcessor batch() { + public static QueryCompilerRequestProcessor.BatchProcessor batch() { return new BatchProcessor(); } /** - * @return a CachingSupplier that supplies a snapshot of the current query scope variables + * @return a CachingSupplier that supplies a snapshot of current query scope variables and query library imports */ @VisibleForTesting - static CachingSupplier> newQueryScopeVariableSupplier() { - final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); - return new CachingSupplier<>(() -> Collections.unmodifiableMap( - queryScope.toMap((name, value) -> NameValidator.isValidQueryParameterName(name)))); + public static CachingSupplier newFormulaImportsSupplier() { + return new CachingSupplier<>(FormulaAnalyzer.Imports::new); } + private final CachingSupplier formulaImportsSupplier = newFormulaImportsSupplier(); + /** - * @return a lazily cached snapshot of the current query scope variables + * @return a lazily cached snapshot of current query scope variables and query library imports */ - Map getQueryScopeVariables(); + public final FormulaAnalyzer.Imports getFormulaImports() { + return formulaImportsSupplier.get(); + } /** * Submit a request for compilation. The QueryCompilerRequestProcessor is not required to immediately compile this @@ -62,24 +61,16 @@ static CachingSupplier> newQueryScopeVariableSupplier() { * * @param request the request to compile */ - CompletionStageFuture> submit(@NotNull QueryCompilerRequest request); + public abstract CompletionStageFuture> submit(@NotNull QueryCompilerRequest request); /** * A QueryCompilerRequestProcessor that immediately compiles requests. */ - class ImmediateProcessor implements QueryCompilerRequestProcessor { - - private final CachingSupplier> queryScopeVariableSupplier = newQueryScopeVariableSupplier(); - + public static class ImmediateProcessor extends QueryCompilerRequestProcessor { private ImmediateProcessor() { // force use of static factory method } - @Override - public Map getQueryScopeVariables() { - return queryScopeVariableSupplier.get(); - } - @Override public CompletionStageFuture> submit(@NotNull final QueryCompilerRequest request) { final String desc = "Compile: " + request.description(); @@ -108,20 +99,14 @@ public CompletionStageFuture> submit(@NotNull final QueryCompilerReques *

* The compile method must be called to actually compile the requests. */ - class BatchProcessor implements QueryCompilerRequestProcessor { + public static class BatchProcessor extends QueryCompilerRequestProcessor { private final List requests = new ArrayList<>(); private final List>> resolvers = new ArrayList<>(); - private final CachingSupplier> queryScopeVariableSupplier = newQueryScopeVariableSupplier(); private BatchProcessor() { // force use of static factory method } - @Override - public Map getQueryScopeVariables() { - return queryScopeVariableSupplier.get(); - } - @Override public CompletionStageFuture> submit(@NotNull final QueryCompilerRequest request) { final CompletionStageFuture.Resolver> resolver = CompletionStageFuture.make(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java index be12408a63b..1c227ea3ae7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java @@ -43,7 +43,6 @@ import io.deephaven.engine.table.impl.remote.ConstructSnapshot; import io.deephaven.engine.table.impl.select.*; import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; -import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzerWrapper; import io.deephaven.engine.table.impl.snapshot.SnapshotIncrementalListener; import io.deephaven.engine.table.impl.snapshot.SnapshotInternalListener; import io.deephaven.engine.table.impl.snapshot.SnapshotUtils; @@ -1498,10 +1497,9 @@ public Table update(final Collection newColumns) { */ public SelectValidationResult validateSelect(final SelectColumn... selectColumns) { final SelectColumn[] clones = SelectColumn.copyFrom(selectColumns); - SelectAndViewAnalyzerWrapper analyzerWrapper = SelectAndViewAnalyzer.create( - this, SelectAndViewAnalyzer.Mode.SELECT_STATIC, columns, rowSet, getModifiedColumnSetForUpdates(), true, - false, clones); - return new SelectValidationResult(analyzerWrapper.getAnalyzer(), clones); + SelectAndViewAnalyzer.AnalyzerContext analyzerContext = SelectAndViewAnalyzer.createContext( + this, SelectAndViewAnalyzer.Mode.SELECT_STATIC, true, false, clones); + return new SelectValidationResult(analyzerContext.createAnalyzer(), clones); } private Table selectOrUpdate(Flavor flavor, final SelectColumn... selectColumns) { @@ -1526,18 +1524,16 @@ private Table selectOrUpdate(Flavor flavor, final SelectColumn... selectColumns) } } final boolean publishTheseSources = flavor == Flavor.Update; - final SelectAndViewAnalyzerWrapper analyzerWrapper = SelectAndViewAnalyzer.create( - this, mode, columns, rowSet, getModifiedColumnSetForUpdates(), publishTheseSources, true, - selectColumns); + final SelectAndViewAnalyzer.AnalyzerContext analyzerContext = SelectAndViewAnalyzer.createContext( + this, mode, publishTheseSources, true, selectColumns); - final SelectAndViewAnalyzer analyzer = analyzerWrapper.getAnalyzer(); - final SelectColumn[] processedColumns = analyzerWrapper.getProcessedColumns() + final SelectAndViewAnalyzer analyzer = analyzerContext.createAnalyzer(); + final SelectColumn[] processedColumns = analyzerContext.getProcessedColumns() .toArray(SelectColumn[]::new); // Init all the rows by cooking up a fake Update final TableUpdate fakeUpdate = new TableUpdateImpl( - analyzer.alreadyFlattenedSources() ? RowSetFactory.flat(rowSet.size()) : rowSet.copy(), - RowSetFactory.empty(), RowSetFactory.empty(), + rowSet.copy(), RowSetFactory.empty(), RowSetFactory.empty(), RowSetShiftData.EMPTY, ModifiedColumnSet.ALL); final CompletableFuture waitForResult = new CompletableFuture<>(); @@ -1558,8 +1554,10 @@ this, mode, columns, rowSet, getModifiedColumnSetForUpdates(), publishTheseSourc new SelectAndViewAnalyzer.UpdateHelper(emptyRowSet, fakeUpdate)) { try { - analyzer.applyUpdate(fakeUpdate, emptyRowSet, updateHelper, jobScheduler, - liveResultCapture, analyzer.futureCompletionHandler(waitForResult)); + analyzer.applyUpdate( + fakeUpdate, emptyRowSet, updateHelper, jobScheduler, liveResultCapture, + () -> waitForResult.complete(null), + waitForResult::completeExceptionally); } catch (Exception e) { waitForResult.completeExceptionally(e); } @@ -1580,14 +1578,15 @@ this, mode, columns, rowSet, getModifiedColumnSetForUpdates(), publishTheseSourc } } - final TrackingRowSet resultRowSet = - analyzer.flattenedResult() ? RowSetFactory.flat(rowSet.size()).toTracking() : rowSet; - resultTable = new QueryTable(resultRowSet, analyzerWrapper.getPublishedColumnResources()); + final TrackingRowSet resultRowSet = analyzer.flatResult() && !rowSet.isFlat() + ? RowSetFactory.flat(rowSet.size()).toTracking() + : rowSet; + resultTable = new QueryTable(resultRowSet, analyzerContext.getPublishedColumnSources()); if (liveResultCapture != null) { analyzer.startTrackingPrev(); - final Map effects = analyzerWrapper.calcEffects(); - final SelectOrUpdateListener soul = new SelectOrUpdateListener(updateDescription, this, - resultTable, effects, analyzer); + final Map effects = analyzerContext.calcEffects(); + final SelectOrUpdateListener soul = new SelectOrUpdateListener( + updateDescription, this, resultTable, effects, analyzer); liveResultCapture.transferTo(soul); addUpdateListener(soul); ConstituentDependency.install(resultTable, soul); @@ -1596,11 +1595,6 @@ this, mode, columns, rowSet, getModifiedColumnSetForUpdates(), publishTheseSourc resultTable.setFlat(); } propagateDataIndexes(processedColumns, resultTable); - for (final ColumnSource columnSource : analyzer.getNewColumnSources().values()) { - if (columnSource instanceof PossiblyImmutableColumnSource) { - ((PossiblyImmutableColumnSource) columnSource).setImmutable(); - } - } } } propagateFlatness(resultTable); @@ -1610,10 +1604,10 @@ this, mode, columns, rowSet, getModifiedColumnSetForUpdates(), publishTheseSourc } else { maybeCopyColumnDescriptions(resultTable); } - SelectAndViewAnalyzerWrapper.UpdateFlavor updateFlavor = flavor == Flavor.Update - ? SelectAndViewAnalyzerWrapper.UpdateFlavor.Update - : SelectAndViewAnalyzerWrapper.UpdateFlavor.Select; - return analyzerWrapper.applyShiftsAndRemainingColumns(this, resultTable, updateFlavor); + SelectAndViewAnalyzer.UpdateFlavor updateFlavor = flavor == Flavor.Update + ? SelectAndViewAnalyzer.UpdateFlavor.Update + : SelectAndViewAnalyzer.UpdateFlavor.Select; + return analyzerContext.applyShiftsAndRemainingColumns(this, resultTable, updateFlavor); })); } @@ -1761,15 +1755,16 @@ updateDescription, sizeForInstrumentation(), () -> { createSnapshotControlIfRefreshing(OperationSnapshotControl::new); initializeWithSnapshot(humanReadablePrefix, sc, (usePrev, beforeClockValue) -> { final boolean publishTheseSources = flavor == Flavor.UpdateView; - final SelectAndViewAnalyzerWrapper analyzerWrapper = SelectAndViewAnalyzer.create( - this, SelectAndViewAnalyzer.Mode.VIEW_EAGER, columns, rowSet, - getModifiedColumnSetForUpdates(), publishTheseSources, true, viewColumns); - final SelectColumn[] processedViewColumns = analyzerWrapper.getProcessedColumns() + final SelectAndViewAnalyzer.AnalyzerContext analyzerContext = + SelectAndViewAnalyzer.createContext( + this, SelectAndViewAnalyzer.Mode.VIEW_EAGER, + publishTheseSources, true, viewColumns); + final SelectColumn[] processedViewColumns = analyzerContext.getProcessedColumns() .toArray(SelectColumn[]::new); QueryTable queryTable = new QueryTable( - rowSet, analyzerWrapper.getPublishedColumnResources()); + rowSet, analyzerContext.getPublishedColumnSources()); if (sc != null) { - final Map effects = analyzerWrapper.calcEffects(); + final Map effects = analyzerContext.calcEffects(); final TableUpdateListener listener = new ViewOrUpdateViewListener(updateDescription, this, queryTable, effects); sc.setListenerAndResult(listener, queryTable); @@ -1786,11 +1781,11 @@ updateDescription, sizeForInstrumentation(), () -> { } else { maybeCopyColumnDescriptions(queryTable); } - final SelectAndViewAnalyzerWrapper.UpdateFlavor updateFlavor = + final SelectAndViewAnalyzer.UpdateFlavor updateFlavor = flavor == Flavor.UpdateView - ? SelectAndViewAnalyzerWrapper.UpdateFlavor.UpdateView - : SelectAndViewAnalyzerWrapper.UpdateFlavor.View; - queryTable = analyzerWrapper.applyShiftsAndRemainingColumns( + ? SelectAndViewAnalyzer.UpdateFlavor.UpdateView + : SelectAndViewAnalyzer.UpdateFlavor.View; + queryTable = analyzerContext.applyShiftsAndRemainingColumns( this, queryTable, updateFlavor); result.setValue(queryTable); @@ -1851,14 +1846,13 @@ public Table lazyUpdate(final Collection newColumns) { sizeForInstrumentation(), () -> { checkInitiateOperation(); - final SelectAndViewAnalyzerWrapper analyzerWrapper = SelectAndViewAnalyzer.create( - this, SelectAndViewAnalyzer.Mode.VIEW_LAZY, columns, rowSet, - getModifiedColumnSetForUpdates(), - true, true, selectColumns); - final SelectColumn[] processedColumns = analyzerWrapper.getProcessedColumns() + final SelectAndViewAnalyzer.AnalyzerContext analyzerContext = + SelectAndViewAnalyzer.createContext( + this, SelectAndViewAnalyzer.Mode.VIEW_LAZY, true, true, selectColumns); + final SelectColumn[] processedColumns = analyzerContext.getProcessedColumns() .toArray(SelectColumn[]::new); final QueryTable result = new QueryTable( - rowSet, analyzerWrapper.getPublishedColumnResources()); + rowSet, analyzerContext.getPublishedColumnSources()); if (isRefreshing()) { addUpdateListener(new ListenerImpl( "lazyUpdate(" + Arrays.deepToString(processedColumns) + ')', this, result)); @@ -1868,8 +1862,8 @@ public Table lazyUpdate(final Collection newColumns) { copySortableColumns(result, processedColumns); maybeCopyColumnDescriptions(result, processedColumns); - return analyzerWrapper.applyShiftsAndRemainingColumns( - this, result, SelectAndViewAnalyzerWrapper.UpdateFlavor.LazyUpdate); + return analyzerContext.applyShiftsAndRemainingColumns( + this, result, SelectAndViewAnalyzer.UpdateFlavor.LazyUpdate); }); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/SelectOrUpdateListener.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/SelectOrUpdateListener.java index 285cdeabb94..d7236ce6010 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/SelectOrUpdateListener.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/SelectOrUpdateListener.java @@ -15,8 +15,8 @@ import io.deephaven.engine.table.impl.util.JobScheduler; import io.deephaven.engine.table.impl.util.UpdateGraphJobScheduler; -import java.util.BitSet; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; /** * A Shift-Aware listener for Select or Update. It uses the SelectAndViewAnalyzer to calculate how columns affect other @@ -29,8 +29,6 @@ class SelectOrUpdateListener extends BaseTable.ListenerImpl { private final SelectAndViewAnalyzer analyzer; private volatile boolean updateInProgress = false; - private final BitSet completedColumns = new BitSet(); - private final BitSet allNewColumns = new BitSet(); private final boolean enableParallelUpdate; /** @@ -61,7 +59,6 @@ class SelectOrUpdateListener extends BaseTable.ListenerImpl { (QueryTable.ENABLE_PARALLEL_SELECT_AND_UPDATE && getUpdateGraph().parallelismFactor() > 1)) && analyzer.allowCrossColumnParallelization(); - analyzer.setAllNewColumns(allNewColumns); } @Override @@ -76,7 +73,6 @@ public void onUpdate(final TableUpdate upstream) { // - create parallel arrays of pre-shift-keys and post-shift-keys so we can move them in chunks updateInProgress = true; - completedColumns.clear(); final TableUpdate acquiredUpdate = upstream.acquire(); final WritableRowSet toClear = resultRowSet.copyPrev(); @@ -91,15 +87,16 @@ public void onUpdate(final TableUpdate upstream) { jobScheduler = new ImmediateJobScheduler(); } + // do not allow a double-notify + final AtomicBoolean hasNotified = new AtomicBoolean(); analyzer.applyUpdate(acquiredUpdate, toClear, updateHelper, jobScheduler, this, - new SelectAndViewAnalyzer.SelectLayerCompletionHandler(allNewColumns, completedColumns) { - @Override - public void onAllRequiredColumnsCompleted() { + () -> { + if (!hasNotified.getAndSet(true)) { completionRoutine(acquiredUpdate, jobScheduler, toClear, updateHelper); } - - @Override - protected void onError(Exception error) { + }, + error -> { + if (!hasNotified.getAndSet(true)) { handleException(error); } }); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/ShiftedColumnsFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/ShiftedColumnsFactory.java index 4ab1c70248c..c84deddd93e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/ShiftedColumnsFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/ShiftedColumnsFactory.java @@ -117,7 +117,7 @@ import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.select.WhereFilter; import io.deephaven.engine.table.impl.select.WhereFilterFactory; -import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzerWrapper; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.util.mutable.MutableInt; import org.jetbrains.annotations.NotNull; @@ -260,7 +260,7 @@ private static Pair getShiftedTableFilterPair( public static Table getShiftedColumnsTable( @NotNull final Table source, @NotNull FormulaColumn formulaColumn, - @NotNull SelectAndViewAnalyzerWrapper.UpdateFlavor updateFlavor) { + @NotNull SelectAndViewAnalyzer.UpdateFlavor updateFlavor) { String nuggetName = "getShiftedColumnsTable( " + formulaColumn + ", " + updateFlavor + ") "; return QueryPerformanceRecorder.withNugget(nuggetName, source.sizeForInstrumentation(), () -> { Table tableSoFar = source; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java index 5cd2418c83b..d457a4ab0b3 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java @@ -182,6 +182,8 @@ public final class QueryLanguageParser extends GenericVisitorAdapter, Q * Create a QueryLanguageParser and parse the given {@code expression}. After construction, the * {@link QueryLanguageParser.Result result} of parsing the {@code expression} is available with the * {@link #getResult()}} method. + *

+ * Note that the provided Collections and Maps must not be mutated concurrently with or after construction. * * @param expression The query language expression to parse * @param packageImports Wildcard package imports @@ -190,9 +192,10 @@ public final class QueryLanguageParser extends GenericVisitorAdapter, Q * imported. * @param variables A map of the names of scope variables to their types * @param variableTypeArguments A map of the names of scope variables to their type arguments - * @param unboxArguments If true it will unbox the query scope arguments - * @param queryScopeVariables A mutable map of the names of query scope variables to their values + * @param queryScopeVariables A map of the names of query scope variables to their values * @param columnVariables A set of column variable names + * @param unboxArguments If true it will unbox the query scope arguments + * @param timeConversionResult The result of converting time literals in the expression * @throws QueryLanguageParseException If any exception or error is encountered */ public QueryLanguageParser( @@ -225,6 +228,8 @@ public QueryLanguageParser( * Create a QueryLanguageParser and parse the given {@code expression}. After construction, the * {@link QueryLanguageParser.Result result} of parsing the {@code expression} is available with the * {@link #getResult()}} method. + *

+ * Note that the provided Collections and Maps must not be mutated concurrently with or after construction. * * @param expression The query language expression to parse * @param packageImports Wildcard package imports @@ -247,6 +252,28 @@ public QueryLanguageParser( variableTypeArguments, null, null, true, null); } + /** + * Create a QueryLanguageParser and parse the given {@code expression}. After construction, the + * {@link QueryLanguageParser.Result result} of parsing the {@code expression} is available with the + * {@link #getResult()}} method. + *

+ * Note that the provided Collections and Maps must not be mutated concurrently with or after construction. + * + * @param expression The query language expression to parse + * @param packageImports Wildcard package imports + * @param classImports Individual class imports + * @param staticImports Wildcard static imports. All static variables and methods for the given classes are + * imported. + * @param variables A map of the names of scope variables to their types + * @param variableTypeArguments A map of the names of scope variables to their type arguments + * @param queryScopeVariables A map of the names of query scope variables to their values + * @param columnVariables A set of column variable names + * @param unboxArguments If true it will unbox the query scope arguments + * @param verifyIdempotence If true, the parser will verify that the result expression will not mutate when parsed + * @param pyCallableWrapperImplName The name of the PyCallableWrapper implementation to use + * @param timeConversionResult The result of converting time literals in the expression + * @throws QueryLanguageParseException If any exception or error is encountered + */ @VisibleForTesting QueryLanguageParser( String expression, @@ -264,9 +291,8 @@ public QueryLanguageParser( this.packageImports = packageImports == null ? Collections.emptySet() : Set.copyOf(packageImports); this.classImports = classImports == null ? Collections.emptySet() : Set.copyOf(classImports); this.staticImports = staticImports == null ? Collections.emptySet() : Set.copyOf(staticImports); - this.variables = variables == null ? Collections.emptyMap() : Map.copyOf(variables); - this.variableTypeArguments = - variableTypeArguments == null ? Collections.emptyMap() : Map.copyOf(variableTypeArguments); + this.variables = variables == null ? Collections.emptyMap() : variables; + this.variableTypeArguments = variableTypeArguments == null ? Collections.emptyMap() : variableTypeArguments; this.queryScopeVariables = queryScopeVariables == null ? new HashMap<>() : queryScopeVariables; this.columnVariables = columnVariables == null ? Collections.emptySet() : columnVariables; this.unboxArguments = unboxArguments; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/TableDataService.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/TableDataService.java index ccd392534be..8d3f9638d6d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/TableDataService.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/TableDataService.java @@ -20,6 +20,22 @@ public interface TableDataService { @NotNull TableLocationProvider getTableLocationProvider(@NotNull TableKey tableKey); + /** + * Request the single raw {@link TableLocationProvider} from this service that has the {@link TableLocation} for + * {@code tableKey} and {@code tableLocationKey}. A raw {@link TableLocationProvider} does not compose multiple + * {@link TableLocationProvider TableLocationProviders} or delegate to other implementations. + * + * @param tableKey The {@link TableKey} to lookup + * @param tableLocationKey The {@link TableLocationKey} to lookup + * @return The raw {@link TableLocationProvider} that has the {@link TableLocation} for {@code tableKey} and + * {@code tableLocationKey}, or {@code null} if there is none + * @throws TableDataException If more than one {@link TableLocationProvider} has the {@link TableLocation} + * + */ + @Nullable + TableLocationProvider getRawTableLocationProvider(@NotNull final TableKey tableKey, + @NotNull final TableLocationKey tableLocationKey); + /** * Forget all state for subsequent requests for all tables. */ diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/AbstractTableDataService.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/AbstractTableDataService.java index 3be464123c4..a7d7ed2dd39 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/AbstractTableDataService.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/AbstractTableDataService.java @@ -5,6 +5,7 @@ import io.deephaven.engine.table.impl.locations.TableDataService; import io.deephaven.engine.table.impl.locations.TableKey; +import io.deephaven.engine.table.impl.locations.TableLocationKey; import io.deephaven.engine.table.impl.locations.TableLocationProvider; import io.deephaven.hash.KeyedObjectHashMap; import io.deephaven.hash.KeyedObjectKey; @@ -41,6 +42,18 @@ public final TableLocationProvider getTableLocationProvider(@NotNull final Table return tableLocationProviders.putIfAbsent(tableKey, this::makeTableLocationProvider); } + @Override + @Nullable + public TableLocationProvider getRawTableLocationProvider(@NotNull TableKey tableKey, + @NotNull TableLocationKey tableLocationKey) { + final TableLocationProvider tableLocationProvider = tableLocationProviders.get(tableKey); + if (tableLocationProvider == null || !tableLocationProvider.hasTableLocationKey(tableLocationKey)) { + return null; + } + + return tableLocationProvider; + } + @Override public void reset() { tableLocationProviders.clear(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/CompositeTableDataService.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/CompositeTableDataService.java index e9dbbc747ce..d5ad0e1faf0 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/CompositeTableDataService.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/CompositeTableDataService.java @@ -53,6 +53,36 @@ public CompositeTableDataService(@NotNull String name, @NotNull final ServiceSel this.serviceSelector = Require.neqNull(serviceSelector, "serviceSelector"); } + @Override + @Nullable + public TableLocationProvider getRawTableLocationProvider(@NotNull final TableKey tableKey, + @NotNull final TableLocationKey tableLocationKey) { + final TableDataService[] services = serviceSelector.call(tableKey); + + if (services == null || services.length == 0) { + return null; + } + + TableLocationProvider tlp = null; + for (final TableDataService service : services) { + final TableLocationProvider tlpCandidate = service.getRawTableLocationProvider(tableKey, tableLocationKey); + if (tlpCandidate == null) { + continue; + } + + if (tlp != null) { + throw new TableDataException( + "TableDataService elements " + tlpCandidate.getName() + " and " + tlp.getName() + + " both contain " + tableLocationKey + ". Full TableDataService configuration:\n" + + Formatter.formatTableDataService(CompositeTableDataService.this.toString())); + } + + tlp = tlpCandidate; + } + + return tlp; + } + @Override public void reset() { super.reset(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/FilteredTableDataService.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/FilteredTableDataService.java index 8bfbeed184a..21c03635091 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/FilteredTableDataService.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/FilteredTableDataService.java @@ -47,6 +47,17 @@ public FilteredTableDataService(@NotNull final TableDataService serviceToFilter, this.locationKeyFilter = Require.neqNull(locationKeyFilter, "locationKeyFilter"); } + @Override + @Nullable + public TableLocationProvider getRawTableLocationProvider(@NotNull final TableKey tableKey, + @NotNull final TableLocationKey tableLocationKey) { + if (!locationKeyFilter.accept(tableLocationKey)) { + return null; + } + + return serviceToFilter.getRawTableLocationProvider(tableKey, tableLocationKey); + } + @Override public void reset() { super.reset(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/local/URIStreamKeyValuePartitionLayout.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/local/URIStreamKeyValuePartitionLayout.java index 2defb0a762b..1fb27e1d6b4 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/local/URIStreamKeyValuePartitionLayout.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/local/URIStreamKeyValuePartitionLayout.java @@ -90,7 +90,8 @@ protected final void findKeys(@NotNull final Stream uriStream, buildLocationKeys(locationTable, targetURIs, locationKeyObserver); } - private void getPartitions(@NotNull final URI relativePath, + private void getPartitions( + @NotNull final URI relativePath, @NotNull final Set partitionKeys, @NotNull final Collection partitionValues, @NotNull final TIntObjectMap partitionColInfo, diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/preview/ColumnPreviewManager.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/preview/ColumnPreviewManager.java index a8e82e2a9e2..b77f8619bf9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/preview/ColumnPreviewManager.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/preview/ColumnPreviewManager.java @@ -15,6 +15,8 @@ import org.jpy.PyListWrapper; import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; import java.time.ZonedDateTime; import java.util.*; import java.util.function.Function; @@ -159,6 +161,7 @@ public static boolean isColumnTypeDisplayable(Class type) { || io.deephaven.util.type.TypeUtils.isString(type) || NumericTypeUtils.isBigNumeric(type) || Instant.class == type || ZonedDateTime.class == type + || LocalDate.class == type || LocalTime.class == type || isOnWhiteList(type); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java index e96bb529f1e..2007eee6526 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java @@ -89,7 +89,7 @@ public synchronized void init( try { final QueryLanguageParser.Result result = FormulaAnalyzer.parseFormula( formula, tableDefinition.getColumnNameMap(), outerToInnerNames, - compilationProcessor.getQueryScopeVariables(), unboxArguments); + compilationProcessor.getFormulaImports(), unboxArguments); formulaShiftColPair = result.getFormulaShiftColPair(); if (formulaShiftColPair != null) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java index 06cbd79fd70..ace064b9353 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java @@ -51,7 +51,7 @@ public abstract class AbstractFormulaColumn implements FormulaColumn { private Formula formula; protected QueryScopeParam[] params; protected Map> columnSources; - protected Map> columnDefinitions; + protected Map> columnDefinitions; private TrackingRowSet rowSet; protected Class returnedType; public static final String COLUMN_SUFFIX = "_"; @@ -90,12 +90,28 @@ public List initInputs( @NotNull final Map> columnsOfInterest) { this.rowSet = rowSet; - this.columnSources = columnsOfInterest; - if (usedColumns != null) { - return usedColumns; + if (usedColumns == null) { + initDef(extractDefinitions(columnsOfInterest), QueryCompilerRequestProcessor.immediate()); } + this.columnSources = filterColumnSources(columnsOfInterest); - return initDef(extractDefinitions(columnsOfInterest), QueryCompilerRequestProcessor.immediate()); + return usedColumns; + } + + private Map> filterColumnSources( + final Map> columnsOfInterest) { + if (usedColumns.isEmpty() && usedColumnArrays.isEmpty()) { + return Map.of(); + } + + final HashMap> sources = new HashMap<>(); + for (String columnName : usedColumns) { + sources.put(columnName, columnsOfInterest.get(columnName)); + } + for (String columnName : usedColumnArrays) { + sources.put(columnName, columnsOfInterest.get(columnName)); + } + return sources; } @Override @@ -119,28 +135,32 @@ public void validateSafeForRefresh(BaseTable sourceTable) { } protected void applyUsedVariables( - @NotNull final Map> columnDefinitionMap, + @NotNull final Map> parentColumnDefinitions, @NotNull final Set variablesUsed, @NotNull final Map possibleParams) { // the column definition map passed in is being mutated by the caller, so we need to make a copy - columnDefinitions = Map.copyOf(columnDefinitionMap); + columnDefinitions = new HashMap<>(); final List> paramsList = new ArrayList<>(); usedColumns = new ArrayList<>(); usedColumnArrays = new ArrayList<>(); for (String variable : variablesUsed) { + ColumnDefinition columnDefinition = parentColumnDefinitions.get(variable); if (variable.equals("i")) { usesI = true; } else if (variable.equals("ii")) { usesII = true; } else if (variable.equals("k")) { usesK = true; - } else if (columnDefinitions.get(variable) != null) { + } else if (columnDefinition != null) { + columnDefinitions.put(variable, columnDefinition); usedColumns.add(variable); } else { String strippedColumnName = variable.substring(0, Math.max(0, variable.length() - COLUMN_SUFFIX.length())); - if (variable.endsWith(COLUMN_SUFFIX) && columnDefinitions.get(strippedColumnName) != null) { + columnDefinition = parentColumnDefinitions.get(strippedColumnName); + if (variable.endsWith(COLUMN_SUFFIX) && columnDefinition != null) { + columnDefinitions.put(strippedColumnName, columnDefinition); usedColumnArrays.add(strippedColumnName); } else if (possibleParams.containsKey(variable)) { paramsList.add(new QueryScopeParam<>(variable, possibleParams.get(variable))); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java index 56a8aecfddc..7ed70800dfa 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java @@ -34,8 +34,7 @@ import io.deephaven.io.logger.Logger; import io.deephaven.util.CompletionStageFuture; import io.deephaven.util.type.TypeUtils; -import io.deephaven.vector.ObjectVector; -import io.deephaven.vector.Vector; +import io.deephaven.vector.VectorFactory; import org.jetbrains.annotations.NotNull; import org.jpy.PyObject; @@ -161,21 +160,7 @@ private static Map> makeNameToTypeDict(final String[] names, } public static Class getVectorType(Class declaredType) { - if (!io.deephaven.util.type.TypeUtils.isConvertibleToPrimitive(declaredType) || declaredType == boolean.class - || declaredType == Boolean.class) { - return ObjectVector.class; - } else { - final String declaredTypeSimpleName = - io.deephaven.util.type.TypeUtils.getUnboxedType(declaredType).getSimpleName(); - try { - return Class.forName(Vector.class.getPackage().getName() + '.' - + Character.toUpperCase(declaredTypeSimpleName.charAt(0)) - + declaredTypeSimpleName.substring(1) - + "Vector"); - } catch (ClassNotFoundException e) { - throw new RuntimeException("Unexpected exception for type " + declaredType, e); - } - } + return VectorFactory.forElementType(declaredType).vectorType(); } @Override @@ -195,7 +180,7 @@ public List initDef( try { final QueryLanguageParser.Result result = FormulaAnalyzer.parseFormula( formulaString, columnDefinitionMap, Collections.emptyMap(), - compilationRequestProcessor.getQueryScopeVariables()); + compilationRequestProcessor.getFormulaImports()); analyzedFormula = FormulaAnalyzer.analyze(formulaString, columnDefinitionMap, result); hasConstantValue = result.isConstantValueExpression(); formulaShiftColPair = result.getFormulaShiftColPair(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index d4026f087aa..4a7921e39b1 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -223,7 +223,8 @@ public synchronized void init( return; } final List valueList = new ArrayList<>(); - final Map queryScopeVariables = compilationProcessor.getQueryScopeVariables(); + final Map queryScopeVariables = + compilationProcessor.getFormulaImports().getQueryScopeVariables(); final ColumnTypeConvertor convertor = ColumnTypeConvertorFactory.getConvertor(column.getDataType()); for (String strValue : strValues) { convertor.convertValue(column, tableDefinition, strValue, queryScopeVariables, valueList::add); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java index 65d65b75e03..5236862c6d5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java @@ -183,7 +183,7 @@ public void init( try { boolean wasAnArrayType = convertor.convertValue( - def, tableDefinition, value, compilationProcessor.getQueryScopeVariables(), + def, tableDefinition, value, compilationProcessor.getFormulaImports().getQueryScopeVariables(), realValue::setValue); if (wasAnArrayType) { conversionError = diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java deleted file mode 100644 index fb79fc32e01..00000000000 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java +++ /dev/null @@ -1,97 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.engine.table.impl.select.analyzers; - -import io.deephaven.base.log.LogOutput; -import io.deephaven.engine.liveness.LivenessNode; -import io.deephaven.engine.table.TableUpdate; -import io.deephaven.engine.table.ModifiedColumnSet; -import io.deephaven.engine.table.ColumnSource; -import io.deephaven.engine.rowset.RowSet; -import io.deephaven.engine.table.impl.util.JobScheduler; -import org.jetbrains.annotations.Nullable; - -import java.util.*; - -public class BaseLayer extends SelectAndViewAnalyzer { - private final Map> sources; - private final boolean publishTheseSources; - - BaseLayer(Map> sources, boolean publishTheseSources) { - super(BASE_LAYER_INDEX); - this.sources = sources; - this.publishTheseSources = publishTheseSources; - } - - @Override - int getLayerIndexFor(String column) { - if (sources.containsKey(column)) { - return BASE_LAYER_INDEX; - } - throw new IllegalArgumentException("Unknown column: " + column); - } - - @Override - void setBaseBits(BitSet bitset) { - bitset.set(BASE_LAYER_INDEX); - } - - @Override - public void setAllNewColumns(BitSet bitset) { - bitset.set(BASE_LAYER_INDEX); - } - - @Override - void populateModifiedColumnSetRecurse(ModifiedColumnSet mcsBuilder, Set remainingDepsToSatisfy) { - mcsBuilder.setAll(remainingDepsToSatisfy.toArray(String[]::new)); - } - - @Override - final Map> getColumnSourcesRecurse(GetMode mode) { - // We specifically return a LinkedHashMap so the columns get populated in order - final Map> result = new LinkedHashMap<>(); - if (mode == GetMode.All || (mode == GetMode.Published && publishTheseSources)) { - result.putAll(sources); - } - return result; - } - - @Override - public void applyUpdate(TableUpdate upstream, RowSet toClear, UpdateHelper helper, JobScheduler jobScheduler, - @Nullable LivenessNode liveResultOwner, SelectLayerCompletionHandler onCompletion) { - // nothing to do at the base layer - onCompletion.onLayerCompleted(BASE_LAYER_INDEX); - } - - @Override - final Map> calcDependsOnRecurse(boolean forcePublishAllSources) { - final Map> result = new HashMap<>(); - if (publishTheseSources || forcePublishAllSources) { - for (final String col : sources.keySet()) { - result.computeIfAbsent(col, dummy -> new HashSet<>()).add(col); - } - } - return result; - } - - @Override - public SelectAndViewAnalyzer getInner() { - return null; - } - - @Override - public void startTrackingPrev() { - // nothing to do - } - - @Override - public LogOutput append(LogOutput logOutput) { - return logOutput.append("{BaseLayer").append(", layerIndex=").append(getLayerIndex()).append("}"); - } - - @Override - public boolean allowCrossColumnParallelization() { - return true; - } -} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/ConstantColumnLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/ConstantColumnLayer.java index c5eff7f3132..4738d944825 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/ConstantColumnLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/ConstantColumnLayer.java @@ -5,41 +5,23 @@ import io.deephaven.base.log.LogOutput; import io.deephaven.chunk.attributes.Values; -import io.deephaven.engine.liveness.LivenessNode; import io.deephaven.engine.rowset.RowSequence; -import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.ChunkSource; import io.deephaven.engine.table.ModifiedColumnSet; -import io.deephaven.engine.table.TableUpdate; import io.deephaven.engine.table.WritableColumnSource; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.VectorChunkAdapter; -import io.deephaven.engine.table.impl.util.JobScheduler; -import org.jetbrains.annotations.Nullable; - -import java.util.Arrays; -import java.util.BitSet; public class ConstantColumnLayer extends SelectOrViewColumnLayer { - private final BitSet dependencyBitSet; - private final boolean flattenedResult; - private final boolean alreadyFlattenedSources; ConstantColumnLayer( - SelectAndViewAnalyzer inner, - String name, - SelectColumn sc, - WritableColumnSource ws, - String[] deps, - ModifiedColumnSet mcsBuilder, - boolean flattenedResult, - boolean alreadyFlattenedSources) { - super(inner, name, sc, ws, null, deps, mcsBuilder); - this.dependencyBitSet = new BitSet(); - this.flattenedResult = flattenedResult; - this.alreadyFlattenedSources = alreadyFlattenedSources; - Arrays.stream(deps).mapToInt(inner::getLayerIndexFor).forEach(dependencyBitSet::set); + final SelectAndViewAnalyzer.AnalyzerContext context, + final SelectColumn sc, + final WritableColumnSource ws, + final String[] deps, + final ModifiedColumnSet mcsBuilder) { + super(context, sc, ws, null, deps, mcsBuilder); initialize(ws); } @@ -60,38 +42,17 @@ private void initialize(final WritableColumnSource writableSource) { } @Override - public void applyUpdate(final TableUpdate upstream, final RowSet toClear, final UpdateHelper helper, - final JobScheduler jobScheduler, @Nullable final LivenessNode liveResultOwner, - final SelectLayerCompletionHandler onCompletion) { - // Nothing to do at this level, but need to recurse because my inner layers might need to be called (e.g. - // because they are SelectColumnLayers) - inner.applyUpdate(upstream, toClear, helper, jobScheduler, liveResultOwner, - new SelectLayerCompletionHandler(dependencyBitSet, onCompletion) { - @Override - public void onAllRequiredColumnsCompleted() { - // we don't need to do anything specific here; our result value is constant - onCompletion.onLayerCompleted(getLayerIndex()); - } - }); - } - - @Override - public LogOutput append(LogOutput logOutput) { - return logOutput.append("{ConstantColumnLayer: ").append(selectColumn.toString()).append("}"); + public boolean hasRefreshingLogic() { + return false; } @Override - public boolean flattenedResult() { - return flattenedResult; + boolean allowCrossColumnParallelization() { + return true; } @Override - public boolean alreadyFlattenedSources() { - return alreadyFlattenedSources; - } - - @Override - public boolean allowCrossColumnParallelization() { - return inner.allowCrossColumnParallelization(); + public LogOutput append(LogOutput logOutput) { + return logOutput.append("{ConstantColumnLayer: ").append(selectColumn.toString()).append("}"); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java index 67fa89b424a..7d00107d17e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java @@ -3,7 +3,6 @@ // package io.deephaven.engine.table.impl.select.analyzers; -import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.vector.Vector; import io.deephaven.engine.table.ModifiedColumnSet; import io.deephaven.engine.table.impl.select.SelectColumn; @@ -11,75 +10,38 @@ import java.util.*; -public abstract class DependencyLayerBase extends SelectAndViewAnalyzer { - final SelectAndViewAnalyzer inner; +public abstract class DependencyLayerBase extends SelectAndViewAnalyzer.Layer { final String name; final SelectColumn selectColumn; final boolean selectColumnHoldsVector; final ColumnSource columnSource; - // probably don't need this any more - private final String[] dependencies; final ModifiedColumnSet myModifiedColumnSet; - - DependencyLayerBase(SelectAndViewAnalyzer inner, String name, SelectColumn selectColumn, - ColumnSource columnSource, - String[] dependencies, ModifiedColumnSet mcsBuilder) { - super(inner.getLayerIndex() + 1); - this.inner = inner; - this.name = name; + final BitSet myLayerDependencySet; + + DependencyLayerBase( + final SelectAndViewAnalyzer.AnalyzerContext context, + final SelectColumn selectColumn, + final ColumnSource columnSource, + final String[] dependencies, + final ModifiedColumnSet mcsBuilder) { + super(context.getNextLayerIndex()); + this.name = selectColumn.getName(); this.selectColumn = selectColumn; selectColumnHoldsVector = Vector.class.isAssignableFrom(selectColumn.getReturnedType()); this.columnSource = columnSource; - this.dependencies = dependencies; - final Set remainingDepsToSatisfy = new HashSet<>(Arrays.asList(dependencies)); - inner.populateModifiedColumnSetRecurse(mcsBuilder, remainingDepsToSatisfy); + context.populateParentDependenciesMCS(mcsBuilder, dependencies); this.myModifiedColumnSet = mcsBuilder; + this.myLayerDependencySet = new BitSet(); + context.populateLayerDependencySet(myLayerDependencySet, dependencies); } @Override - void populateModifiedColumnSetRecurse(ModifiedColumnSet mcsBuilder, Set remainingDepsToSatisfy) { - // Later-defined columns override earlier-defined columns. So we satisfy column dependencies "on the way - // down" the recursion. - if (remainingDepsToSatisfy.remove(name)) { - // Caller had a dependency on us, so caller gets our dependencies - mcsBuilder.setAll(myModifiedColumnSet); - } - inner.populateModifiedColumnSetRecurse(mcsBuilder, remainingDepsToSatisfy); - } - - @Override - final Map> calcDependsOnRecurse(boolean forcePublishAllResources) { - final Map> result = inner.calcDependsOnRecurse(forcePublishAllResources); - final Set thisResult = new HashSet<>(); - for (final String dep : dependencies) { - final Set innerDependencies = result.get(dep); - if (innerDependencies == null) { - // There are no further expansions of 'dep', so add it as a dependency. - thisResult.add(dep); - } else { - // Instead of adding 'dep', add what 'dep' expands to. - thisResult.addAll(innerDependencies); - } - } - result.put(name, thisResult); - return result; - } - - @Override - public SelectAndViewAnalyzer getInner() { - return inner; - } - - @Override - int getLayerIndexFor(String column) { - if (name.equals(column)) { - return getLayerIndex(); - } - return inner.getLayerIndexFor(column); + Set getLayerColumnNames() { + return Set.of(name); } @Override - void setBaseBits(BitSet bitset) { - inner.setBaseBits(bitset); + public ModifiedColumnSet getModifiedColumnSet() { + return myModifiedColumnSet; } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/PreserveColumnLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/PreserveColumnLayer.java index 8d687cdc8c8..9b0f4b690f7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/PreserveColumnLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/PreserveColumnLayer.java @@ -4,69 +4,41 @@ package io.deephaven.engine.table.impl.select.analyzers; import io.deephaven.base.log.LogOutput; -import io.deephaven.engine.liveness.LivenessNode; -import io.deephaven.engine.table.TableUpdate; import io.deephaven.engine.table.ModifiedColumnSet; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.ColumnSource; -import io.deephaven.engine.rowset.RowSet; -import io.deephaven.engine.table.impl.util.JobScheduler; -import org.jetbrains.annotations.Nullable; -import java.util.Arrays; -import java.util.BitSet; import java.util.Map; /** * A layer that copies a column from our input to our output. - * + *

* {@implNote This class is part of the Deephaven engine, and not intended for direct use.} */ final public class PreserveColumnLayer extends DependencyLayerBase { - private final BitSet dependencyBitSet; - PreserveColumnLayer(SelectAndViewAnalyzer inner, String name, SelectColumn sc, ColumnSource cs, String[] deps, - ModifiedColumnSet mcsBuilder) { - super(inner, name, sc, cs, deps, mcsBuilder); - this.dependencyBitSet = new BitSet(); - Arrays.stream(deps).mapToInt(inner::getLayerIndexFor).forEach(dependencyBitSet::set); + PreserveColumnLayer( + final SelectAndViewAnalyzer.AnalyzerContext context, + final SelectColumn sc, + final ColumnSource cs, + final String[] deps, + final ModifiedColumnSet mcsBuilder) { + super(context, sc, cs, deps, mcsBuilder); } @Override - public void applyUpdate(TableUpdate upstream, RowSet toClear, UpdateHelper helper, JobScheduler jobScheduler, - @Nullable LivenessNode liveResultOwner, SelectLayerCompletionHandler onCompletion) { - // Nothing to do at this level, but need to recurse because my inner layers might need to be called (e.g. - // because they are SelectColumnLayers) - inner.applyUpdate(upstream, toClear, helper, jobScheduler, liveResultOwner, - new SelectLayerCompletionHandler(dependencyBitSet, onCompletion) { - @Override - public void onAllRequiredColumnsCompleted() { - // we don't need to do anything specific here - onCompletion.onLayerCompleted(getLayerIndex()); - } - }); + public boolean hasRefreshingLogic() { + return false; } @Override - Map> getColumnSourcesRecurse(GetMode mode) { - // our column is not a new column, so we need to make sure that we do not double enable previous tracking - final Map> result = inner.getColumnSourcesRecurse(mode); - switch (mode) { - case New: - // we have no new sources - break; - case Published: - case All: - result.put(name, columnSource); - break; - } - return result; + void populateColumnSources(final Map> result) { + result.put(name, columnSource); } @Override - public void startTrackingPrev() { - // nothing to do, here but the inner needs to be called - inner.startTrackingPrev(); + boolean allowCrossColumnParallelization() { + return true; } @Override @@ -74,24 +46,4 @@ public LogOutput append(LogOutput logOutput) { return logOutput.append("{PreserveColumnLayer: ").append(name).append(", layerIndex=").append(getLayerIndex()) .append("}"); } - - @Override - public boolean flattenedResult() { - // preserve layer is only flattened if the inner is flattened - // the "flattenedResult" means that we are flattening the table as part of select. For a pre-existing column, we - // could not preserve a layer while flattening, but if we are preserving a newly generated column; it is valid - // for the result to have been flattened as part of select. - return inner.flattenedResult(); - } - - @Override - public boolean alreadyFlattenedSources() { - // a preserve layer is only already flattened if the inner is already flattened - return inner.alreadyFlattenedSources(); - } - - @Override - public boolean allowCrossColumnParallelization() { - return inner.allowCrossColumnParallelization(); - } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java index 8a33acfa00d..46fe6ea4c31 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java @@ -4,82 +4,83 @@ package io.deephaven.engine.table.impl.select.analyzers; import io.deephaven.base.log.LogOutput; -import io.deephaven.base.verify.Assert; import io.deephaven.engine.liveness.LivenessNode; import io.deephaven.engine.rowset.*; import io.deephaven.engine.rowset.RowSetFactory; -import io.deephaven.engine.table.TableUpdate; -import io.deephaven.engine.table.ModifiedColumnSet; import io.deephaven.engine.table.ColumnSource; +import io.deephaven.engine.table.ModifiedColumnSet; +import io.deephaven.engine.table.TableUpdate; import io.deephaven.engine.table.impl.util.*; import io.deephaven.util.mutable.MutableLong; import org.apache.commons.lang3.mutable.MutableObject; import org.jetbrains.annotations.Nullable; import java.util.*; +import java.util.function.Consumer; /** * A layer that maintains the row redirection for future SelectColumnLayers. - * + *

* {@implNote This class is part of the Deephaven engine, and not intended for direct use.} */ -public final class RedirectionLayer extends SelectAndViewAnalyzer { - private final SelectAndViewAnalyzer inner; +public final class RedirectionLayer extends SelectAndViewAnalyzer.Layer { private final TrackingRowSet resultRowSet; private final WritableRowRedirection rowRedirection; private final WritableRowSet freeValues = RowSetFactory.empty(); + private final BitSet layerDependencySet = new BitSet(); private long maxInnerIndex; - RedirectionLayer(SelectAndViewAnalyzer inner, TrackingRowSet resultRowSet, WritableRowRedirection rowRedirection) { - super(REDIRECTION_LAYER_INDEX); - Assert.eq(inner.getLayerIndex(), "inner.getLayerIndex()", BASE_LAYER_INDEX); - this.inner = inner; + RedirectionLayer( + final SelectAndViewAnalyzer.AnalyzerContext context, + final TrackingRowSet resultRowSet, + final WritableRowRedirection rowRedirection) { + super(context.getNextLayerIndex()); this.resultRowSet = resultRowSet; this.rowRedirection = rowRedirection; this.maxInnerIndex = -1; } @Override - int getLayerIndexFor(String column) { - // Result columns' applyUpdate depend on the result of the redirection. - Assert.eq(inner.getLayerIndexFor(column), "inner.getLayerIndexFor(column)", BASE_LAYER_INDEX); - return REDIRECTION_LAYER_INDEX; + Set getLayerColumnNames() { + return Set.of(); } @Override - void setBaseBits(BitSet bitset) { - inner.setBaseBits(bitset); - bitset.set(REDIRECTION_LAYER_INDEX); + void populateColumnSources(final Map> result) { + // we don't generate any column sources, so we don't need to do anything here } @Override - public void populateModifiedColumnSetRecurse(ModifiedColumnSet mcsBuilder, Set remainingDepsToSatisfy) { - inner.populateModifiedColumnSetRecurse(mcsBuilder, remainingDepsToSatisfy); + ModifiedColumnSet getModifiedColumnSet() { + return ModifiedColumnSet.EMPTY; } @Override - public Map> getColumnSourcesRecurse(GetMode mode) { - return inner.getColumnSourcesRecurse(mode); + BitSet getLayerDependencySet() { + return layerDependencySet; } @Override - public void applyUpdate(TableUpdate upstream, RowSet toClear, UpdateHelper helper, JobScheduler jobScheduler, - @Nullable LivenessNode liveResultOwner, SelectLayerCompletionHandler onCompletion) { - final BitSet baseLayerBitSet = new BitSet(); - inner.setBaseBits(baseLayerBitSet); - inner.applyUpdate(upstream, toClear, helper, jobScheduler, liveResultOwner, - new SelectLayerCompletionHandler(baseLayerBitSet, onCompletion) { - @Override - public void onAllRequiredColumnsCompleted() { - // we only have a base layer underneath us, so we do not care about the bitSet; it is always - // empty - doApplyUpdate(upstream, toClear, helper, onCompletion); - } - }); + boolean allowCrossColumnParallelization() { + return true; + } + + @Override + public Runnable createUpdateHandler( + final TableUpdate upstream, + final RowSet toClear, + final SelectAndViewAnalyzer.UpdateHelper helper, + final JobScheduler jobScheduler, + @Nullable final LivenessNode liveResultOwner, + final Runnable onSuccess, + final Consumer onError) { + // note that we process this layer directly because all subsequent layers depend on it + return () -> doApplyUpdate(upstream, onSuccess); } - private void doApplyUpdate(TableUpdate upstream, RowSet toClear, UpdateHelper helper, - SelectLayerCompletionHandler onCompletion) { + private void doApplyUpdate( + final TableUpdate upstream, + final Runnable onSuccess) { // we need to remove the removed values from our row redirection, and add them to our free RowSet; so that // updating tables will not consume more space over the course of a day for abandoned rows final RowSetBuilderRandom innerToFreeBuilder = RowSetFactory.builderRandom(); @@ -150,32 +151,16 @@ private void doApplyUpdate(TableUpdate upstream, RowSet toClear, UpdateHelper he freeValues.removeRange(0, lastAllocated.get()); } - onCompletion.onLayerCompleted(REDIRECTION_LAYER_INDEX); - } - - @Override - public Map> calcDependsOnRecurse(boolean forcePublishAllResources) { - return inner.calcDependsOnRecurse(forcePublishAllResources); - } - - @Override - public SelectAndViewAnalyzer getInner() { - return inner; + onSuccess.run(); } @Override public void startTrackingPrev() { rowRedirection.startTrackingPrevValues(); - inner.startTrackingPrev(); } @Override public LogOutput append(LogOutput logOutput) { return logOutput.append("{RedirectionLayer").append(", layerIndex=").append(getLayerIndex()).append("}"); } - - @Override - public boolean allowCrossColumnParallelization() { - return inner.allowCrossColumnParallelization(); - } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java index 099d70d4eb6..6efaf2425b2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java @@ -3,46 +3,60 @@ // package io.deephaven.engine.table.impl.select.analyzers; +import gnu.trove.map.TObjectIntMap; +import gnu.trove.map.hash.TObjectIntHashMap; import io.deephaven.base.Pair; +import io.deephaven.base.log.LogOutput; import io.deephaven.base.log.LogOutputAppendable; +import io.deephaven.base.verify.Assert; import io.deephaven.engine.liveness.LivenessNode; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; +import io.deephaven.engine.rowset.RowSetShiftData; import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.QueryTable; +import io.deephaven.engine.table.impl.ShiftedColumnsFactory; +import io.deephaven.engine.table.impl.TableUpdateImpl; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.SourceColumn; import io.deephaven.engine.table.impl.select.SwitchColumn; import io.deephaven.engine.table.impl.sources.InMemoryColumnSource; +import io.deephaven.engine.table.impl.sources.PossiblyImmutableColumnSource; +import io.deephaven.engine.table.impl.sources.RedirectedColumnSource; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; import io.deephaven.engine.table.impl.sources.WritableRedirectedColumnSource; import io.deephaven.engine.table.impl.util.InverseWrappedRowSetRowRedirection; import io.deephaven.engine.table.impl.util.JobScheduler; import io.deephaven.engine.table.impl.util.RowRedirection; +import io.deephaven.engine.table.impl.util.WrappedRowSetRowRedirection; import io.deephaven.engine.table.impl.util.WritableRowRedirection; import io.deephaven.engine.updategraph.UpdateGraph; import io.deephaven.io.log.impl.LogOutputStringImpl; import io.deephaven.util.SafeCloseable; import io.deephaven.util.SafeCloseablePair; import io.deephaven.vector.Vector; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.util.*; -import java.util.concurrent.CompletableFuture; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.stream.Stream; -public abstract class SelectAndViewAnalyzer implements LogOutputAppendable { +public class SelectAndViewAnalyzer implements LogOutputAppendable { private static final Consumer> NOOP = ignore -> { }; public enum Mode { VIEW_LAZY, VIEW_EAGER, SELECT_STATIC, SELECT_REFRESHING, SELECT_REDIRECTED_REFRESHING, SELECT_REDIRECTED_STATIC } + public enum UpdateFlavor { + Select, View, Update, UpdateView, LazyUpdate + } public static void initializeSelectColumns( final Map> parentColumnMap, @@ -65,26 +79,23 @@ public static void initializeSelectColumns( } } - public static SelectAndViewAnalyzerWrapper create( - QueryTable sourceTable, Mode mode, Map> columnSources, - TrackingRowSet rowSet, ModifiedColumnSet parentMcs, boolean publishTheseSources, boolean useShiftedColumns, - SelectColumn... selectColumns) { - return create(sourceTable, mode, columnSources, rowSet, parentMcs, publishTheseSources, useShiftedColumns, - true, selectColumns); - } - - public static SelectAndViewAnalyzerWrapper create( - final QueryTable sourceTable, + public static AnalyzerContext createContext( + final QueryTable parentTable, final Mode mode, - final Map> columnSources, - TrackingRowSet rowSet, - final ModifiedColumnSet parentMcs, - final boolean publishTheseSources, + final boolean publishParentSources, boolean useShiftedColumns, - final boolean allowInternalFlatten, final SelectColumn... selectColumns) { - final UpdateGraph updateGraph = sourceTable.getUpdateGraph(); - SelectAndViewAnalyzer analyzer = createBaseLayer(columnSources, publishTheseSources); + final UpdateGraph updateGraph = parentTable.getUpdateGraph(); + + final Map> columnSources = parentTable.getColumnSourceMap(); + final TrackingRowSet rowSet = parentTable.getRowSet(); + + final boolean parentIsFlat = parentTable.isFlat(); + final boolean flatResult = !parentIsFlat + && (columnSources.isEmpty() || !publishParentSources) + && mode == Mode.SELECT_STATIC; + final AnalyzerContext context = new AnalyzerContext(parentTable, publishParentSources, flatResult); + final Map> columnDefinitions = new LinkedHashMap<>(); final RowRedirection rowRedirection; if (mode == Mode.SELECT_REDIRECTED_STATIC) { @@ -92,19 +103,12 @@ public static SelectAndViewAnalyzerWrapper create( } else if (mode == Mode.SELECT_REDIRECTED_REFRESHING && rowSet.size() < Integer.MAX_VALUE) { final WritableRowRedirection writableRowRedirection = WritableRowRedirection.FACTORY.createRowRedirection(rowSet.intSize()); - analyzer = analyzer.createRedirectionLayer(rowSet, writableRowRedirection); + context.addLayer(new RedirectionLayer(context, rowSet, writableRowRedirection)); rowRedirection = writableRowRedirection; } else { rowRedirection = null; } - List processedCols = new LinkedList<>(); - List remainingCols = null; - FormulaColumn shiftColumn = null; - boolean shiftColumnHasPositiveOffset = false; - - final HashSet resultColumns = new HashSet<>(); - // First pass to initialize all columns and to compile formulas in one batch. final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); for (Map.Entry> entry : columnSources.entrySet()) { @@ -114,9 +118,10 @@ public static SelectAndViewAnalyzerWrapper create( columnDefinitions.put(name, cd); } + final Set resultColumnNames = new HashSet<>(); for (final SelectColumn sc : selectColumns) { - if (remainingCols != null) { - remainingCols.add(sc); + if (context.remainingCols != null) { + context.remainingCols.add(sc); continue; } @@ -126,55 +131,50 @@ public static SelectAndViewAnalyzerWrapper create( columnDefinitions.put(sc.getName(), cd); if (useShiftedColumns && hasConstantArrayAccess(sc)) { - remainingCols = new LinkedList<>(); - shiftColumn = sc instanceof FormulaColumn + context.remainingCols = new LinkedList<>(); + context.shiftColumn = sc instanceof FormulaColumn ? (FormulaColumn) sc : (FormulaColumn) ((SwitchColumn) sc).getRealColumn(); - shiftColumnHasPositiveOffset = hasPositiveOffsetConstantArrayAccess(sc); + context.shiftColumnHasPositiveOffset = hasPositiveOffsetConstantArrayAccess(sc); continue; } - processedCols.add(sc); + // In our first pass, determine whether any columns will be preserved so that we don't prematurely flatten. + final SourceColumn realColumn = tryToGetSourceColumn(sc); + + if (realColumn != null && !resultColumnNames.contains(realColumn.getSourceName())) { + // if we are preserving a column, then we cannot change key space + context.flatResult &= !shouldPreserve(columnSources.get(realColumn.getSourceName())); + } + + // TODO (deephaven#5760): If layers may define more than one column, we'll need to add all of them here. + resultColumnNames.add(sc.getName()); + + context.processedCols.add(sc); } compilationProcessor.compile(); // Second pass builds the analyzer and destination columns - final TrackingRowSet originalRowSet = rowSet; - boolean flatResult = rowSet.isFlat(); - // if we preserve a column, we set this to false - boolean flattenedResult = !flatResult - && allowInternalFlatten - && (columnSources.isEmpty() || !publishTheseSources) - && mode == Mode.SELECT_STATIC; - int numberOfInternallyFlattenedColumns = 0; - final HashMap> resultAlias = new HashMap<>(); - for (final SelectColumn sc : processedCols) { - - sc.initInputs(rowSet, analyzer.getAllColumnSources()); + for (final SelectColumn sc : context.processedCols) { - // When flattening the result, intermediate columns generate results in position space. When we discover - // that a select column depends on an intermediate result, then we must flatten all parent columns so - // that all dependent columns are in the same result-key space. - if (!flatResult && flattenedResult && Stream.concat(sc.getColumns().stream(), sc.getColumnArrays().stream()) - .anyMatch(resultColumns::contains)) { - analyzer = analyzer.createStaticFlattenLayer(rowSet); - rowSet = RowSetFactory.flat(rowSet.size()).toTracking(); - flatResult = true; + // if this select column depends on result column then its updates must happen in result-key-space + // note: if flatResult is true then we are not preserving any parent columns + final boolean useResultKeySpace = context.flatResult + && Stream.concat(sc.getColumns().stream(), sc.getColumnArrays().stream()) + .anyMatch(columnName -> context.getLayerIndexFor(columnName) != Layer.PARENT_TABLE_INDEX); - // we must re-initialize the column inputs as they may have changed post-flatten - sc.initInputs(rowSet, analyzer.getAllColumnSources()); - } + sc.initInputs(rowSet, useResultKeySpace ? context.allSourcesInResultKeySpace : context.allSources); - resultColumns.add(sc.getName()); - // this shadows any known alias + // TODO (deephaven-core#5760): If layers may define more than one column, we'll need to fix resultAlias. + // new columns shadow known aliases resultAlias.remove(sc.getName()); final Stream allDependencies = Stream.concat(sc.getColumns().stream(), sc.getColumnArrays().stream()); final String[] distinctDeps = allDependencies.distinct().toArray(String[]::new); - final ModifiedColumnSet mcsBuilder = new ModifiedColumnSet(parentMcs); + final ModifiedColumnSet mcsBuilder = new ModifiedColumnSet(parentTable.getModifiedColumnSetForUpdates()); if (useShiftedColumns && hasConstantArrayAccess(sc)) { // we use the first shifted column to split between processed columns and remaining columns @@ -183,104 +183,78 @@ public static SelectAndViewAnalyzerWrapper create( // shifted columns appear to not be safe for refresh, so we do not validate them until they are rewritten // using the intermediary shifted column - if (sourceTable.isRefreshing()) { - sc.validateSafeForRefresh(sourceTable); + if (parentTable.isRefreshing()) { + sc.validateSafeForRefresh(parentTable); } if (hasConstantValue(sc)) { final WritableColumnSource constViewSource = SingleValueColumnSource.getSingleValueColumnSource(sc.getReturnedType()); - analyzer = analyzer.createLayerForConstantView( - sc.getName(), sc, constViewSource, distinctDeps, mcsBuilder, flattenedResult, - flatResult && flattenedResult); + context.addLayer(new ConstantColumnLayer(context, sc, constViewSource, distinctDeps, mcsBuilder)); continue; } - final SourceColumn realColumn; - if (sc instanceof SourceColumn) { - realColumn = (SourceColumn) sc; - } else if ((sc instanceof SwitchColumn) && ((SwitchColumn) sc).getRealColumn() instanceof SourceColumn) { - realColumn = (SourceColumn) ((SwitchColumn) sc).getRealColumn(); - } else { - realColumn = null; - } - - if (realColumn != null && shouldPreserve(sc)) { - boolean sourceIsNew = resultColumns.contains(realColumn.getSourceName()); - if (!sourceIsNew) { - if (numberOfInternallyFlattenedColumns > 0) { - // we must preserve this column, but have already created an analyzer for the internally - // flattened column, therefore must start over without permitting internal flattening - return create(sourceTable, mode, columnSources, originalRowSet, parentMcs, publishTheseSources, - useShiftedColumns, false, selectColumns); - } else { - // we can not flatten future columns because we are preserving a column that may not be flat - flattenedResult = false; - } - } - - analyzer = analyzer.createLayerForPreserve( - sc.getName(), sc, sc.getDataView(), distinctDeps, mcsBuilder); - - continue; - } - - // look for an existing alias that can be preserved instead + final SourceColumn realColumn = tryToGetSourceColumn(sc); if (realColumn != null) { + if (shouldPreserve(sc.getDataView())) { + context.addLayer(new PreserveColumnLayer(context, sc, sc.getDataView(), distinctDeps, mcsBuilder)); + continue; + } + // look for an existing alias that can be preserved instead final ColumnSource alias = resultAlias.get(realColumn.getSourceName()); if (alias != null) { - analyzer = analyzer.createLayerForPreserve(sc.getName(), sc, alias, distinctDeps, mcsBuilder); + context.addLayer(new PreserveColumnLayer(context, sc, alias, distinctDeps, mcsBuilder)); continue; } } - // if this is a source column, then results are eligible for aliasing + // if this is a SourceColumn, then results are eligible for aliasing final Consumer> maybeCreateAlias = realColumn == null ? NOOP : cs -> resultAlias.put(realColumn.getSourceName(), cs); final long targetDestinationCapacity = - rowSet.isEmpty() ? 0 : (flattenedResult ? rowSet.size() : rowSet.lastRowKey() + 1); + rowSet.isEmpty() ? 0 : (context.flatResult ? rowSet.size() : rowSet.lastRowKey() + 1); switch (mode) { case VIEW_LAZY: { final ColumnSource viewCs = sc.getLazyView(); maybeCreateAlias.accept(viewCs); - analyzer = analyzer.createLayerForView(sc.getName(), sc, viewCs, distinctDeps, mcsBuilder); + context.addLayer(new ViewColumnLayer(context, sc, viewCs, distinctDeps, mcsBuilder)); break; } case VIEW_EAGER: { final ColumnSource viewCs = sc.getDataView(); maybeCreateAlias.accept(viewCs); - analyzer = analyzer.createLayerForView(sc.getName(), sc, viewCs, distinctDeps, mcsBuilder); + context.addLayer(new ViewColumnLayer(context, sc, viewCs, distinctDeps, mcsBuilder)); break; } case SELECT_STATIC: { // We need to call newDestInstance because only newDestInstance has the knowledge to endow our // created array with the proper componentType (in the case of Vectors). - final WritableColumnSource scs = - flatResult || flattenedResult ? sc.newFlatDestInstance(targetDestinationCapacity) - : sc.newDestInstance(targetDestinationCapacity); + final WritableColumnSource scs = parentIsFlat || context.flatResult + ? sc.newFlatDestInstance(targetDestinationCapacity) + : sc.newDestInstance(targetDestinationCapacity); + maybeSetStaticColumnSourceImmutable(scs); maybeCreateAlias.accept(scs); - analyzer = analyzer.createLayerForSelect(updateGraph, rowSet, sc.getName(), sc, scs, null, - distinctDeps, mcsBuilder, false, flattenedResult, flatResult && flattenedResult); - if (flattenedResult) { - numberOfInternallyFlattenedColumns++; - } + context.addLayer(new SelectColumnLayer( + updateGraph, rowSet, context, sc, scs, null, distinctDeps, mcsBuilder, false, + useResultKeySpace)); break; } case SELECT_REDIRECTED_STATIC: { final WritableColumnSource underlyingSource = sc.newDestInstance(rowSet.size()); final WritableColumnSource scs = WritableRedirectedColumnSource.maybeRedirect( rowRedirection, underlyingSource, rowSet.size()); + maybeSetStaticColumnSourceImmutable(scs); maybeCreateAlias.accept(scs); - analyzer = analyzer.createLayerForSelect(updateGraph, rowSet, sc.getName(), sc, scs, - underlyingSource, distinctDeps, mcsBuilder, true, false, false); + context.addLayer(new SelectColumnLayer( + updateGraph, rowSet, context, sc, scs, underlyingSource, distinctDeps, mcsBuilder, true, + useResultKeySpace)); break; } case SELECT_REDIRECTED_REFRESHING: case SELECT_REFRESHING: { // We need to call newDestInstance because only newDestInstance has the knowledge to endow our // created array with the proper componentType (in the case of Vectors). - // TODO: use DeltaAwareColumnSource WritableColumnSource scs = sc.newDestInstance(targetDestinationCapacity); WritableColumnSource underlyingSource = null; if (rowRedirection != null) { @@ -289,8 +263,9 @@ public static SelectAndViewAnalyzerWrapper create( rowRedirection, underlyingSource, rowSet.intSize()); } maybeCreateAlias.accept(scs); - analyzer = analyzer.createLayerForSelect(updateGraph, rowSet, sc.getName(), sc, scs, - underlyingSource, distinctDeps, mcsBuilder, rowRedirection != null, false, false); + context.addLayer(new SelectColumnLayer( + updateGraph, rowSet, context, sc, scs, underlyingSource, distinctDeps, mcsBuilder, + rowRedirection != null, useResultKeySpace)); break; } default: @@ -298,8 +273,25 @@ public static SelectAndViewAnalyzerWrapper create( } } - return new SelectAndViewAnalyzerWrapper(analyzer, shiftColumn, shiftColumnHasPositiveOffset, remainingCols, - processedCols); + return context; + } + + private static void maybeSetStaticColumnSourceImmutable(final ColumnSource columnSource) { + if (columnSource instanceof PossiblyImmutableColumnSource) { + ((PossiblyImmutableColumnSource) columnSource).setImmutable(); + } + } + + private static @Nullable SourceColumn tryToGetSourceColumn(final SelectColumn sc) { + final SourceColumn realColumn; + if (sc instanceof SourceColumn) { + realColumn = (SourceColumn) sc; + } else if ((sc instanceof SwitchColumn) && ((SwitchColumn) sc).getRealColumn() instanceof SourceColumn) { + realColumn = (SourceColumn) ((SwitchColumn) sc).getRealColumn(); + } else { + realColumn = null; + } + return realColumn; } private static boolean hasConstantArrayAccess(final SelectColumn sc) { @@ -343,110 +335,439 @@ private static boolean hasConstantValue(final SelectColumn sc) { return false; } - private static boolean shouldPreserve(final SelectColumn sc) { - // we already know sc is a SourceColumn or switches to a SourceColumn - final ColumnSource sccs = sc.getDataView(); - return sccs instanceof InMemoryColumnSource && ((InMemoryColumnSource) sccs).isInMemory() - && !Vector.class.isAssignableFrom(sc.getReturnedType()); + private static boolean shouldPreserve(final ColumnSource columnSource) { + return columnSource instanceof InMemoryColumnSource && ((InMemoryColumnSource) columnSource).isInMemory() + && !Vector.class.isAssignableFrom(columnSource.getType()); } - static final int BASE_LAYER_INDEX = 0; - static final int REDIRECTION_LAYER_INDEX = 1; + /** The layers that make up this analyzer. */ + private final Layer[] layers; - /** - * The layerIndex is used to identify each layer uniquely within the bitsets for completion. - */ - private final int layerIndex; + /** Whether the result should be flat. */ + private final boolean flatResult; - public SelectAndViewAnalyzer(int layerIndex) { - this.layerIndex = layerIndex; - } + private final BitSet requiredLayers = new BitSet(); + private final BitSet remainingLayers = new BitSet(); - int getLayerIndex() { - return layerIndex; + private SelectAndViewAnalyzer( + final Layer[] layers, + final boolean flatResult) { + this.layers = layers; + this.flatResult = flatResult; + for (final Layer layer : layers) { + if (layer.hasRefreshingLogic()) { + requiredLayers.set(layer.getLayerIndex()); + } else { + this.layers[layer.getLayerIndex()] = null; + } + } } - /** - * Set the bits in bitset that represent the base layer and optional redirection layer. No other jobs can be - * executed until all of these bits are set. - * - * @param bitset the bitset to manipulate. - */ - abstract void setBaseBits(BitSet bitset); + public final static class AnalyzerContext { - /** - * Set the bits in bitset that represent all the new columns. This is used to identify when the select or update - * operation is complete. - * - * @param bitset the bitset to manipulate. - */ - public void setAllNewColumns(BitSet bitset) { - getInner().setAllNewColumns(bitset); - bitset.set(getLayerIndex()); - } + /** The analyzer that we are building. */ + private final List layers = new ArrayList<>(); + /** + * The sources that are available to the analyzer, including parent columns. Parent columns are in parent key + * space, others are in result key space. + */ + private final Map> allSources = new LinkedHashMap<>(); + /** The sources that are available to the analyzer, including parent columns, in result key space. */ + private final Map> allSourcesInResultKeySpace; + /** The sources that are published to the child table. */ + private final Map> publishedSources = new LinkedHashMap<>(); + /** A mapping from result column name to the layer index that created it. */ + private final TObjectIntMap columnToLayerIndex; + /** The select columns that have been processed so far. */ + private final List processedCols = new ArrayList<>(); + + /** A holder for the shift column, if any. */ + private FormulaColumn shiftColumn; + /** Whether the shift column has a positive offset. */ + private boolean shiftColumnHasPositiveOffset; + /** The columns that will need to be processed after the shift column. */ + private List remainingCols; + /** Whether the result should be flat. */ + private boolean flatResult; + /** The layer that will be used to process redirection, if we have one. */ + private int redirectionLayer = Layer.UNSET_INDEX; + + AnalyzerContext( + final QueryTable parentTable, + final boolean publishParentSources, + final boolean flatResult) { + final Map> parentSources = parentTable.getColumnSourceMap(); + columnToLayerIndex = new TObjectIntHashMap<>(parentSources.size(), 0.5f, Layer.UNSET_INDEX); + + this.flatResult = flatResult; + + allSources.putAll(parentSources); + for (final String columnName : allSources.keySet()) { + columnToLayerIndex.put(columnName, Layer.PARENT_TABLE_INDEX); + } - private static SelectAndViewAnalyzer createBaseLayer(Map> sources, - boolean publishTheseSources) { - return new BaseLayer(sources, publishTheseSources); - } + if (publishParentSources) { + publishedSources.putAll(parentSources); + } - private RedirectionLayer createRedirectionLayer(TrackingRowSet resultRowSet, - WritableRowRedirection rowRedirection) { - return new RedirectionLayer(this, resultRowSet, rowRedirection); - } + if (!flatResult) { + // result key space is the same as parent key space + allSourcesInResultKeySpace = allSources; + } else { + allSourcesInResultKeySpace = new HashMap<>(); - private StaticFlattenLayer createStaticFlattenLayer(TrackingRowSet parentRowSet) { - return new StaticFlattenLayer(this, parentRowSet); - } + final RowRedirection rowRedirection = new WrappedRowSetRowRedirection(parentTable.getRowSet()); + allSources.forEach((name, cs) -> allSourcesInResultKeySpace.put(name, + RedirectedColumnSource.maybeRedirect(rowRedirection, cs))); + } + } - private SelectAndViewAnalyzer createLayerForSelect( - UpdateGraph updateGraph, RowSet parentRowset, String name, SelectColumn sc, WritableColumnSource cs, - WritableColumnSource underlyingSource, String[] parentColumnDependencies, ModifiedColumnSet mcsBuilder, - boolean isRedirected, boolean flattenResult, boolean alreadyFlattened) { - return new SelectColumnLayer(updateGraph, parentRowset, this, name, sc, cs, underlyingSource, - parentColumnDependencies, - mcsBuilder, isRedirected, flattenResult, alreadyFlattened); - } + /** + * Add a layer to the analyzer. + * + * @param layer the layer to add + */ + void addLayer(final Layer layer) { + if (layer instanceof RedirectionLayer) { + if (redirectionLayer != Layer.UNSET_INDEX) { + throw new IllegalStateException("Cannot have more than one redirection layer"); + } + redirectionLayer = layers.size(); + } - private SelectAndViewAnalyzer createLayerForConstantView(String name, SelectColumn sc, WritableColumnSource cs, - String[] parentColumnDependencies, ModifiedColumnSet mcsBuilder, boolean flattenResult, - boolean alreadyFlattened) { - return new ConstantColumnLayer(this, name, sc, cs, parentColumnDependencies, mcsBuilder, flattenResult, - alreadyFlattened); - } + layer.populateColumnSources(allSources); + if (flatResult) { + layer.populateColumnSources(allSourcesInResultKeySpace); + } + layer.populateColumnSources(publishedSources); - private SelectAndViewAnalyzer createLayerForView(String name, SelectColumn sc, ColumnSource cs, - String[] parentColumnDependencies, ModifiedColumnSet mcsBuilder) { - return new ViewColumnLayer(this, name, sc, cs, parentColumnDependencies, mcsBuilder); - } + layers.add(layer); - private SelectAndViewAnalyzer createLayerForPreserve(String name, SelectColumn sc, ColumnSource cs, - String[] parentColumnDependencies, ModifiedColumnSet mcsBuilder) { - return new PreserveColumnLayer(this, name, sc, cs, parentColumnDependencies, mcsBuilder); - } + for (final String columnName : layer.getLayerColumnNames()) { + columnToLayerIndex.put(columnName, layer.getLayerIndex()); + } + } - abstract void populateModifiedColumnSetRecurse(ModifiedColumnSet mcsBuilder, Set remainingDepsToSatisfy); + /** + * @return the next layerIndex to use + */ + int getNextLayerIndex() { + return layers.size(); + } - enum GetMode { - All, New, Published - } + /** + * Return the layerIndex for a given string column. + * + * @param column the name of the column + * + * @return the layerIndex + */ + int getLayerIndexFor(String column) { + final int layerIndex = columnToLayerIndex.get(column); + if (layerIndex == Layer.UNSET_INDEX) { + throw new IllegalStateException("Column " + column + " not found in any layer of the analyzer"); + } + return layerIndex; + } - public final Map> getAllColumnSources() { - return getColumnSourcesRecurse(GetMode.All); - } + /** + * Populate the ModifiedColumnSet with all indirect/direct dependencies on the parent table. + * + * @param mcsBuilder the result ModifiedColumnSet to populate + * @param dependencies the immediate dependencies + */ + void populateParentDependenciesMCS( + final ModifiedColumnSet mcsBuilder, + final String[] dependencies) { + for (final String dep : dependencies) { + final int layerIndex = getLayerIndexFor(dep); + if (layerIndex == Layer.PARENT_TABLE_INDEX) { + // this is a preserved parent column + mcsBuilder.setAll(dep); + } else { + mcsBuilder.setAll(layers.get(layerIndex).getModifiedColumnSet()); + } + } + } - public final Map> getNewColumnSources() { - return getColumnSourcesRecurse(GetMode.New); - } + /** + * Populate the layer dependency set with the layer indices that the dependencies are in. + * + * @param layerDependencySet the result bitset to populate + * @param dependencies the dependencies + */ + void populateLayerDependencySet( + final BitSet layerDependencySet, + final String[] dependencies) { + for (final String dep : dependencies) { + final int layerIndex = getLayerIndexFor(dep); + if (layerIndex != Layer.PARENT_TABLE_INDEX) { + // note that implicitly preserved columns do not belong to a layer. + layerDependencySet.or(layers.get(layerIndex).getLayerDependencySet()); + } + } + } + + /** + * Set the redirection layer in the bitset if the analyzer has any redirection. + * + * @param layerDependencies the result bitset to populate + */ + void setRedirectionLayer(final BitSet layerDependencies) { + if (redirectionLayer != Layer.UNSET_INDEX) { + layerDependencies.set(redirectionLayer); + } + } + + /** + * @return the column sources that are published through the child table + */ + public Map> getPublishedColumnSources() { + // Note that if we have a shift column that we forcibly publish all columns. + return shiftColumn == null ? publishedSources : allSources; + } + + /** + * @return the final analyzer + */ + public SelectAndViewAnalyzer createAnalyzer() { + return new SelectAndViewAnalyzer(layers.toArray(Layer[]::new), flatResult); + } + + /** + * @return which select columns were included in the result (not including the shift, or post-shift, columns) + */ + public List getProcessedColumns() { + return processedCols; + } + + /** + * @return whether the result should be flat + */ + public boolean isFlatResult() { + return flatResult; + } + + /** + * Our job here is to calculate the effects: a map from incoming column to a list of columns that it effects. We + * do this in two stages. In the first stage we create a map from column to (set of dependent columns). In the + * second stage we reverse that map. + * + * @return the effects map + */ + public Map calcEffects() { + final Map> resultMap = getPublishedColumnSources(); + + // Create the mapping from result column to dependent source columns. + final Map dependsOn = new HashMap<>(); + for (final String columnName : resultMap.keySet()) { + final int layerIndex = getLayerIndexFor(columnName); + final String[] dependencies; + if (layerIndex == Layer.PARENT_TABLE_INDEX) { + dependencies = new String[] {columnName}; + } else { + dependencies = layers.get(layerIndex).getModifiedColumnSet().dirtyColumnNames(); + } + dependsOn.put(columnName, dependencies); + } + + // Now create the mapping from source column to result columns. + final Map> effects = new HashMap<>(); + for (Map.Entry entry : dependsOn.entrySet()) { + final String depender = entry.getKey(); + for (final String dependee : entry.getValue()) { + effects.computeIfAbsent(dependee, dummy -> new ArrayList<>()).add(depender); + } + } + + // Convert effects type into result type + final Map result = new HashMap<>(); + for (Map.Entry> entry : effects.entrySet()) { + final String[] value = entry.getValue().toArray(String[]::new); + result.put(entry.getKey(), value); + } + return result; + } + + /** + * Shift columns introduce intermediary table operations. This method applies remaining work to the result built + * so far. + * + * @param parentTable the source table + * @param resultSoFar the intermediate result + * @param updateFlavor the update flavor + * @return the final result + */ + public QueryTable applyShiftsAndRemainingColumns( + @NotNull final QueryTable parentTable, + @NotNull QueryTable resultSoFar, + final UpdateFlavor updateFlavor) { + if (shiftColumn != null) { + resultSoFar = (QueryTable) ShiftedColumnsFactory.getShiftedColumnsTable( + resultSoFar, shiftColumn, updateFlavor); + } + + // shift columns may introduce modifies that are not present in the original table; set these before using + if (parentTable.isRefreshing()) { + if (shiftColumn == null && parentTable.isAddOnly()) { + resultSoFar.setAttribute(Table.ADD_ONLY_TABLE_ATTRIBUTE, true); + } + if ((shiftColumn == null || !shiftColumnHasPositiveOffset) && parentTable.isAppendOnly()) { + // note if the shift offset is non-positive, then this result is still append-only + resultSoFar.setAttribute(Table.APPEND_ONLY_TABLE_ATTRIBUTE, true); + } + if (parentTable.hasAttribute(Table.TEST_SOURCE_TABLE_ATTRIBUTE)) { + // be convenient for test authors by propagating the test source table attribute + resultSoFar.setAttribute(Table.TEST_SOURCE_TABLE_ATTRIBUTE, true); + } + if (parentTable.isBlink()) { + // blink tables, although possibly not useful, can have shift columns + resultSoFar.setAttribute(Table.BLINK_TABLE_ATTRIBUTE, true); + } + } - public final Map> getPublishedColumnSources() { - return getColumnSourcesRecurse(GetMode.Published); + boolean isMultiStateSelect = shiftColumn != null || remainingCols != null; + if (isMultiStateSelect && (updateFlavor == UpdateFlavor.Select || updateFlavor == UpdateFlavor.View)) { + List newResultColumns = new LinkedList<>(); + for (SelectColumn processed : processedCols) { + newResultColumns.add(new SourceColumn(processed.getName())); + } + if (shiftColumn != null) { + newResultColumns.add(new SourceColumn(shiftColumn.getName())); + } + if (remainingCols != null) { + newResultColumns.addAll(remainingCols); + } + + if (updateFlavor == UpdateFlavor.Select) { + resultSoFar = (QueryTable) resultSoFar.select(newResultColumns); + } else { + resultSoFar = (QueryTable) resultSoFar.view(newResultColumns); + } + } else if (remainingCols != null) { + switch (updateFlavor) { + case Update: { + resultSoFar = (QueryTable) resultSoFar.update(remainingCols); + break; + } + case UpdateView: { + resultSoFar = (QueryTable) resultSoFar.updateView(remainingCols); + break; + } + case LazyUpdate: { + resultSoFar = (QueryTable) resultSoFar.lazyUpdate(remainingCols); + break; + } + default: + throw new IllegalStateException("Unexpected update flavor: " + updateFlavor); + } + } + + return resultSoFar; + } } - abstract Map> getColumnSourcesRecurse(GetMode mode); + static abstract class Layer implements LogOutputAppendable { + private static final BitSet EMPTY_BITSET = new BitSet(); + + public static final int UNSET_INDEX = -1; + public static final int PARENT_TABLE_INDEX = -2; + + /** + * The layerIndex is used to identify each layer uniquely within the bitsets for completion. + */ + private final int layerIndex; + + Layer(int layerIndex) { + this.layerIndex = layerIndex; + } + + /** + * @return which index in the layer stack this layer is + */ + int getLayerIndex() { + return layerIndex; + } + + /** + * @return whether this layer has refreshing logic and needs to be updated + */ + boolean hasRefreshingLogic() { + return true; + } + + /** + * @return the modified column set of the parent table that this layer indirectly depends on + */ + ModifiedColumnSet getModifiedColumnSet() { + return failNoRefreshingLogic(); + } + + /** + * @return the layer dependency set indicating which layers this layer depends on + */ + BitSet getLayerDependencySet() { + return EMPTY_BITSET; + } + + @Override + public String toString() { + return new LogOutputStringImpl().append(this).toString(); + } + + void startTrackingPrev() { + // default is that there is nothing to do + } + + /** + * @return the column names created by this layer + */ + abstract Set getLayerColumnNames(); + + /** + * Populate the column sources for this layer. + * + * @param result the map to populate + */ + abstract void populateColumnSources(Map> result); + + /** + * @return true if this layer allows parallelization across columns + */ + abstract boolean allowCrossColumnParallelization(); + + /** + * Apply this update to this Layer. + * + * @param upstream the upstream update + * @param toClear rows that used to exist and no longer exist + * @param helper convenience class that memoizes reusable calculations for this update + * @param jobScheduler scheduler for parallel sub-tasks + * @param liveResultOwner {@link LivenessNode node} to be used to manage/unmanage results that happen to be + * {@link io.deephaven.engine.liveness.LivenessReferent liveness referents} + * @param onSuccess called when the update completed successfully + * @param onError called when the update failed + */ + Runnable createUpdateHandler( + TableUpdate upstream, + RowSet toClear, + UpdateHelper helper, + JobScheduler jobScheduler, + @Nullable LivenessNode liveResultOwner, + Runnable onSuccess, + Consumer onError) { + return failNoRefreshingLogic(); + } + + private T failNoRefreshingLogic() { + throw new UnsupportedOperationException(String.format( + "%s does not have any refreshing logic", this.getClass().getSimpleName())); + } + } public static class UpdateHelper implements SafeCloseable { private RowSet existingRows; + private TableUpdate upstreamInResultSpace; private SafeCloseablePair shiftedWithModifies; private SafeCloseablePair shiftedWithoutModifies; @@ -458,6 +779,21 @@ public UpdateHelper(RowSet parentRowSet, TableUpdate upstream) { this.upstream = upstream; } + /** + * Flatten the upstream update from the parent key space to the destination key space. We are guaranteed to be + * in STATIC_SELECT mode. + * + * @return the flattened update + */ + TableUpdate resultKeySpaceUpdate() { + if (upstreamInResultSpace == null) { + upstreamInResultSpace = new TableUpdateImpl( + RowSetFactory.flat(upstream.added().size()), RowSetFactory.empty(), RowSetFactory.empty(), + RowSetShiftData.EMPTY, ModifiedColumnSet.EMPTY); + } + return upstreamInResultSpace; + } + private RowSet getExisting() { if (existingRows == null) { existingRows = parentRowSet.minus(upstream.added()); @@ -507,6 +843,10 @@ public void close() { shiftedWithoutModifies.close(); shiftedWithoutModifies = null; } + if (upstreamInResultSpace != null) { + upstreamInResultSpace.release(); + upstreamInResultSpace = null; + } } } @@ -519,195 +859,174 @@ public void close() { * @param jobScheduler scheduler for parallel sub-tasks * @param liveResultOwner {@link LivenessNode node} to be used to manage/unmanage results that happen to be * {@link io.deephaven.engine.liveness.LivenessReferent liveness referents} - * @param onCompletion Called when an inner column is complete. The outer layer should pass the {@code onCompletion} - */ - public abstract void applyUpdate(TableUpdate upstream, RowSet toClear, UpdateHelper helper, - JobScheduler jobScheduler, @Nullable LivenessNode liveResultOwner, - SelectLayerCompletionHandler onCompletion); - - /** - * Our job here is to calculate the effects: a map from incoming column to a list of columns that it effects. We do - * this in two stages. In the first stage we create a map from column to (set of dependent columns). In the second - * stage we reverse that map. + * @param onSuccess called when the update completed successfully + * @param onError called when the update failed */ - public final Map calcEffects(boolean forcePublishAllResources) { - final Map> dependsOn = calcDependsOnRecurse(forcePublishAllResources); - - // Now create effects, which is the inverse of dependsOn: - // An entry W -> [X, Y, Z] in effects means that W affects X, Y, and Z - final Map> effects = new HashMap<>(); - for (Map.Entry> entry : dependsOn.entrySet()) { - final String depender = entry.getKey(); - for (final String dependee : entry.getValue()) { - effects.computeIfAbsent(dependee, dummy -> new ArrayList<>()).add(depender); + public void applyUpdate( + final TableUpdate upstream, + final RowSet toClear, + final UpdateHelper helper, + final JobScheduler jobScheduler, + @Nullable final LivenessNode liveResultOwner, + final Runnable onSuccess, + final Consumer onError) { + + Assert.assertion(remainingLayers.isEmpty(), "remainingLayers.isEmpty()"); + remainingLayers.or(requiredLayers); + + final Runnable[] runners = new Runnable[layers.length]; + final UpdateScheduler scheduler = new UpdateScheduler(runners, onSuccess, onError); + + for (int ii = 0; ii < layers.length; ++ii) { + final Layer layer = layers[ii]; + if (layer != null) { + // TODO (deephaven-core#4896): this error handling allows concurrent layers to fail without ensuring + // that other tasks are finished. + runners[ii] = layer.createUpdateHandler( + upstream, toClear, helper, jobScheduler, liveResultOwner, + () -> scheduler.onLayerComplete(layer.getLayerIndex()), onError); } } - // Convert effects type into result type - final Map result = new HashMap<>(); - for (Map.Entry> entry : effects.entrySet()) { - final String[] value = entry.getValue().toArray(String[]::new); - result.put(entry.getKey(), value); - } - return result; + + scheduler.tryToKickOffWork(); } - abstract Map> calcDependsOnRecurse(boolean forcePublishAllResources); + private class UpdateScheduler { + private final ReentrantLock runLock = new ReentrantLock(); - public abstract SelectAndViewAnalyzer getInner(); + private final Runnable[] runners; + private final Runnable onSuccess; + private final Consumer onError; - public abstract void startTrackingPrev(); + private volatile boolean needsRun; + /** whether we have already invoked onSuccess */ + private boolean updateComplete; - /** - * Was the result internally flattened? Only the STATIC_SELECT case flattens the result. If the result preserves any - * columns, then flattening is not permitted. Because all the other layers cannot internally flatten, the default - * implementation returns false. - */ - public boolean flattenedResult() { - return false; - } + public UpdateScheduler( + final Runnable[] runners, + final Runnable onSuccess, + final Consumer onError) { + this.runners = runners; + this.onSuccess = onSuccess; + this.onError = onError; + } - /** - * Have the column sources already been flattened? Only the STATIC_SELECT case flattens the result. A static flatten - * layer is only added if SelectColumn depends on an intermediate result. - */ - public boolean alreadyFlattenedSources() { - return false; - } + public void onLayerComplete(final int layerIndex) { + synchronized (remainingLayers) { + remainingLayers.set(layerIndex, false); + } - /** - * Return the layerIndex for a given string column. - * - *

- * This is executed recursively, because later columns in a select statement hide earlier columns. - *

- * - * @param column the name of the column - * - * @return the layerIndex - */ - abstract int getLayerIndexFor(String column); + tryToKickOffWork(); + } - /** - * Can all of our columns permit parallel updates? - */ - abstract public boolean allowCrossColumnParallelization(); + private void tryToKickOffWork() { + needsRun = true; + while (true) { + if (runLock.isHeldByCurrentThread() || !runLock.tryLock()) { + // do not permit re-entry or waiting on another thread doing exactly this work + return; + } - /** - * A class that handles the completion of one select column. The handlers are chained together; all downstream - * dependencies may execute when a column completes. - */ - public static abstract class SelectLayerCompletionHandler { - /** - * Note that the completed columns are shared among the entire chain of completion handlers. - */ - private final BitSet completedColumns; - private final SelectLayerCompletionHandler nextHandler; - private final BitSet requiredColumns; - private volatile boolean fired = false; + try { + if (needsRun) { + needsRun = false; + doKickOffWork(); + } + } catch (final Exception exception) { + try { + onError.accept(exception); + } catch (final Exception ignored) { + } + } finally { + runLock.unlock(); + } - /** - * Create a new completion handler that calls nextHandler after its own processing. The completedColumns BitSet - * is shared among all handlers. - * - * @param requiredColumns the columns required for this layer - * @param nextHandler the next handler to call - */ - SelectLayerCompletionHandler(BitSet requiredColumns, SelectLayerCompletionHandler nextHandler) { - this.requiredColumns = requiredColumns; - this.completedColumns = nextHandler.completedColumns; - this.nextHandler = nextHandler; + if (!needsRun) { + return; + } + } } - /** - * Create the final completion handler, which has no next handler. - * - * @param requiredColumns the columns required for this handler to fire - * @param completedColumns the set of completed columns, shared with all the other handlers - */ - public SelectLayerCompletionHandler(BitSet requiredColumns, BitSet completedColumns) { - this.requiredColumns = requiredColumns; - this.completedColumns = completedColumns; - this.nextHandler = null; - } + private void doKickOffWork() { + if (updateComplete) { + // we may have already completed the update, but are checking again due to the potential of a race + return; + } - /** - * Called when a single column is completed. - *

- * If we are ready, then we call {@link #onAllRequiredColumnsCompleted()}. - *

- * We may not be ready, but other columns downstream of us may be ready, so they are also notified (the - * nextHandler). - * - * @param completedColumn the layerIndex of the completedColumn - */ - void onLayerCompleted(int completedColumn) { - if (!fired) { + int nextLayer = 0; + while (nextLayer >= 0) { + boolean complete; boolean readyToFire = false; - synchronized (completedColumns) { - if (!fired) { - completedColumns.set(completedColumn); - if (requiredColumns.get(completedColumn) || requiredColumns.isEmpty()) { - readyToFire = requiredColumns.stream().allMatch(completedColumns::get); - if (readyToFire) { - fired = true; - } + Runnable runner = null; + synchronized (remainingLayers) { + complete = remainingLayers.isEmpty(); + nextLayer = complete ? -1 : remainingLayers.nextSetBit(nextLayer); + + if (nextLayer != -1) { + if ((runner = runners[nextLayer]) != null) { + readyToFire = !layers[nextLayer].getLayerDependencySet().intersects(remainingLayers); + } + + if (readyToFire) { + runners[nextLayer] = null; + } else { + ++nextLayer; } } } + if (readyToFire) { - onAllRequiredColumnsCompleted(); + runner.run(); + } else if (complete) { + updateComplete = true; + onSuccess.run(); + return; } } - if (nextHandler != null) { - nextHandler.onLayerCompleted(completedColumn); - } } + } - protected void onError(Exception error) { - if (nextHandler != null) { - nextHandler.onError(error); + public void startTrackingPrev() { + for (final Layer layer : layers) { + if (layer != null) { + layer.startTrackingPrev(); } } - - /** - * Called when all required columns are completed. - */ - protected abstract void onAllRequiredColumnsCompleted(); } /** - * Create a completion handler that signals a future when the update is completed. - * - * @param waitForResult a void future indicating success or failure - * - * @return a completion handler that will signal the future + * Is the result of this select/view flat? */ - public SelectLayerCompletionHandler futureCompletionHandler(CompletableFuture waitForResult) { - final BitSet completedColumns = new BitSet(); - final BitSet requiredColumns = new BitSet(); - - setAllNewColumns(requiredColumns); + public boolean flatResult() { + return flatResult; + } - return new SelectLayerCompletionHandler(requiredColumns, completedColumns) { - boolean errorOccurred = false; + /** + * Can all of our columns permit parallel updates? + */ + public boolean allowCrossColumnParallelization() { + return Arrays.stream(layers) + .filter(Objects::nonNull) + .allMatch(Layer::allowCrossColumnParallelization); + } - @Override - public void onAllRequiredColumnsCompleted() { - if (errorOccurred) { - return; - } - waitForResult.complete(null); + @Override + public LogOutput append(LogOutput logOutput) { + logOutput = logOutput.append("SelectAndViewAnalyzer{"); + boolean first = true; + for (final Layer layer : layers) { + if (layer == null) { + continue; } - - @Override - protected void onError(Exception error) { - if (errorOccurred) { - return; - } - errorOccurred = true; - waitForResult.completeExceptionally(error); + if (first) { + first = false; + } else { + logOutput = logOutput.append(", "); } - }; + logOutput = logOutput.append(layer); + + } + return logOutput.append("}"); } @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzerWrapper.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzerWrapper.java deleted file mode 100644 index ec4fcf6f534..00000000000 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzerWrapper.java +++ /dev/null @@ -1,128 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.engine.table.impl.select.analyzers; - -import io.deephaven.engine.table.ColumnSource; -import io.deephaven.engine.table.Table; -import io.deephaven.engine.table.impl.QueryTable; -import io.deephaven.engine.table.impl.ShiftedColumnsFactory; -import io.deephaven.engine.table.impl.select.FormulaColumn; -import io.deephaven.engine.table.impl.select.SelectColumn; -import io.deephaven.engine.table.impl.select.SourceColumn; -import org.jetbrains.annotations.NotNull; - -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -public class SelectAndViewAnalyzerWrapper { - public enum UpdateFlavor { - Select, View, Update, UpdateView, LazyUpdate - } - - private final SelectAndViewAnalyzer analyzer; - private final FormulaColumn shiftColumn; - private final boolean shiftColumnHasPositiveOffset; - private final List remainingCols; - private final List processedColumns; - - SelectAndViewAnalyzerWrapper( - SelectAndViewAnalyzer analyzer, - FormulaColumn shiftColumn, - boolean shiftColumnHasPositiveOffset, - List remainingCols, - List processedColumns) { - this.analyzer = analyzer; - this.shiftColumn = shiftColumn; - this.shiftColumnHasPositiveOffset = shiftColumnHasPositiveOffset; - this.remainingCols = remainingCols; - this.processedColumns = processedColumns; - } - - public final Map> getPublishedColumnResources() { - if (shiftColumn == null) { - return analyzer.getPublishedColumnSources(); - } else { - return analyzer.getAllColumnSources(); - } - } - - public final Map calcEffects() { - return analyzer.calcEffects(shiftColumn != null); - } - - public SelectAndViewAnalyzer getAnalyzer() { - return analyzer; - } - - public List getProcessedColumns() { - return processedColumns; - } - - public QueryTable applyShiftsAndRemainingColumns( - @NotNull QueryTable sourceTable, @NotNull QueryTable queryTable, UpdateFlavor updateFlavor) { - if (shiftColumn != null) { - queryTable = (QueryTable) ShiftedColumnsFactory.getShiftedColumnsTable( - queryTable, shiftColumn, updateFlavor); - } - - // shift columns may introduce modifies that are not present in the original table; set these before using - if (sourceTable.isRefreshing()) { - if (shiftColumn == null && sourceTable.isAddOnly()) { - queryTable.setAttribute(Table.ADD_ONLY_TABLE_ATTRIBUTE, true); - } - if ((shiftColumn == null || !shiftColumnHasPositiveOffset) && sourceTable.isAppendOnly()) { - // note if the shift offset is non-positive, then this result is still append-only - queryTable.setAttribute(Table.APPEND_ONLY_TABLE_ATTRIBUTE, true); - } - if (sourceTable.hasAttribute(Table.TEST_SOURCE_TABLE_ATTRIBUTE)) { - // be convenient for test authors by propagating the test source table attribute - queryTable.setAttribute(Table.TEST_SOURCE_TABLE_ATTRIBUTE, true); - } - if (sourceTable.isBlink()) { - // blink tables, although possibly not useful, can have shift columns - queryTable.setAttribute(Table.BLINK_TABLE_ATTRIBUTE, true); - } - } - - boolean isMultiStateSelect = shiftColumn != null || remainingCols != null; - if (isMultiStateSelect && (updateFlavor == UpdateFlavor.Select || updateFlavor == UpdateFlavor.View)) { - List newResultColumns = new LinkedList<>(); - for (SelectColumn processed : processedColumns) { - newResultColumns.add(new SourceColumn(processed.getName())); - } - if (shiftColumn != null) { - newResultColumns.add(new SourceColumn(shiftColumn.getName())); - } - if (remainingCols != null) { - newResultColumns.addAll(remainingCols); - } - - if (updateFlavor == UpdateFlavor.Select) { - queryTable = (QueryTable) queryTable.select(newResultColumns); - } else { - queryTable = (QueryTable) queryTable.view(newResultColumns); - } - } else if (remainingCols != null) { - switch (updateFlavor) { - case Update: { - queryTable = (QueryTable) queryTable.update(remainingCols); - break; - } - case UpdateView: { - queryTable = (QueryTable) queryTable.updateView(remainingCols); - break; - } - case LazyUpdate: { - queryTable = (QueryTable) queryTable.lazyUpdate(remainingCols); - break; - } - default: - throw new IllegalStateException("Unexpected update flavor: " + updateFlavor); - } - } - - return queryTable; - } -} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectColumnLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectColumnLayer.java index b7177e9fe39..431d00ee7e7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectColumnLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectColumnLayer.java @@ -40,24 +40,22 @@ import static io.deephaven.chunk.util.pools.ChunkPoolConstants.LARGEST_POOLED_CHUNK_CAPACITY; final public class SelectColumnLayer extends SelectOrViewColumnLayer { - /** - * The same reference as super.columnSource, but as a WritableColumnSource and maybe reinterpreted - */ + /** The same reference as super.columnSource, but as a WritableColumnSource and maybe reinterpreted */ private final WritableColumnSource writableSource; - /** - * The execution context the select column layer was constructed in - */ + /** The execution context the select column layer was constructed in */ private final ExecutionContext executionContext; private final UpdateGraph updateGraph; - /** - * Our parent row set, used for ensuring capacity. - */ + /** Our parent row set, used for ensuring capacity */ private final RowSet parentRowSet; + /** Whether our result is redirected */ private final boolean isRedirected; + /** Whether our result is flattened */ private final boolean flattenedResult; - private final boolean alreadyFlattenedSources; + /** Whether our dependencies are in the result key space instead of parent key space */ + private final boolean sourcesAreInResultKeySpace; + /** Which layers we depend on */ private final BitSet dependencyBitSet; private final boolean canParallelizeThisColumn; private final boolean isSystemic; @@ -73,14 +71,22 @@ final public class SelectColumnLayer extends SelectOrViewColumnLayer { private ChunkSource.WithPrev chunkSource; SelectColumnLayer( - UpdateGraph updateGraph, RowSet parentRowSet, SelectAndViewAnalyzer inner, String name, SelectColumn sc, - WritableColumnSource ws, WritableColumnSource underlying, String[] deps, ModifiedColumnSet mcsBuilder, - boolean isRedirected, boolean flattenedResult, boolean alreadyFlattenedSources) { - super(inner, name, sc, ws, underlying, deps, mcsBuilder); + final UpdateGraph updateGraph, + final RowSet parentRowSet, + final SelectAndViewAnalyzer.AnalyzerContext context, + final SelectColumn sc, + final WritableColumnSource ws, + final WritableColumnSource underlying, + final String[] deps, + final ModifiedColumnSet mcsBuilder, + final boolean isRedirected, + final boolean sourcesAreInResultKeySpace) { + super(context, sc, ws, underlying, deps, mcsBuilder); this.updateGraph = updateGraph; this.parentRowSet = parentRowSet; this.writableSource = ReinterpretUtils.maybeConvertToWritablePrimitive(ws); this.isRedirected = isRedirected; + this.sourcesAreInResultKeySpace = sourcesAreInResultKeySpace; final ExecutionContext userSuppliedContext = ExecutionContext.getContextToRecord(); if (userSuppliedContext != null) { @@ -91,10 +97,16 @@ final public class SelectColumnLayer extends SelectOrViewColumnLayer { } dependencyBitSet = new BitSet(); - Arrays.stream(deps).mapToInt(inner::getLayerIndexFor).forEach(dependencyBitSet::set); + Arrays.stream(deps) + .mapToInt(context::getLayerIndexFor) + .filter(layerIndex -> layerIndex >= 0) + .forEach(dependencyBitSet::set); + if (isRedirected) { + // we cannot write to the redirected column until after the redirection has been updated + context.setRedirectionLayer(dependencyBitSet); + } - this.flattenedResult = flattenedResult; - this.alreadyFlattenedSources = alreadyFlattenedSources; + this.flattenedResult = context.isFlatResult(); // We can only parallelize this column if we are not redirected, our destination provides ensure previous, and // the select column is stateless @@ -136,9 +148,33 @@ private ChunkSource getChunkSource() { } @Override - public void applyUpdate(final TableUpdate upstream, final RowSet toClear, - final UpdateHelper helper, final JobScheduler jobScheduler, @Nullable final LivenessNode liveResultOwner, - final SelectLayerCompletionHandler onCompletion) { + public BitSet getLayerDependencySet() { + return dependencyBitSet; + } + + @Override + public Runnable createUpdateHandler( + final TableUpdate originalUpdate, + final RowSet toClear, + final SelectAndViewAnalyzer.UpdateHelper helper, + final JobScheduler jobScheduler, + @Nullable final LivenessNode liveResultOwner, + final Runnable onSuccess, + final Consumer onError) { + final TableUpdate upstream; + if (!sourcesAreInResultKeySpace) { + upstream = originalUpdate; + } else { + // This better be the static fake update. + Assert.eqTrue(originalUpdate.added().size() == parentRowSet.size(), + "originalUpdate.added().size() == parentRowSet.size()"); + Assert.eqTrue(originalUpdate.removed().isEmpty(), "originalUpdate.removed.isEmpty()"); + Assert.eqTrue(originalUpdate.modified().isEmpty(), "originalUpdate.modified.isEmpty()"); + Assert.eqTrue(originalUpdate.shifted().empty(), "originalUpdate.shifted.empty()"); + + upstream = helper.resultKeySpaceUpdate(); + } + if (upstream.removed().isNonempty()) { if (isRedirected) { clearObjectsAtThisLevel(upstream.removed()); @@ -148,80 +184,80 @@ public void applyUpdate(final TableUpdate upstream, final RowSet toClear, } } - // recurse so that dependent intermediate columns are already updated - inner.applyUpdate(upstream, toClear, helper, jobScheduler, liveResultOwner, - new SelectLayerCompletionHandler(dependencyBitSet, onCompletion) { - @Override - public void onAllRequiredColumnsCompleted() { - // We don't want to bother with threads if we are going to process a small update - final long totalSize = upstream.added().size() + upstream.modified().size(); - - // If we have shifts, that makes everything nasty; so we do not want to deal with it - final boolean hasShifts = upstream.shifted().nonempty(); - - final boolean serialTableOperationsSafe = updateGraph.serialTableOperationsSafe() - || updateGraph.sharedLock().isHeldByCurrentThread() - || updateGraph.exclusiveLock().isHeldByCurrentThread(); - - if (canParallelizeThisColumn && jobScheduler.threadCount() > 1 && !hasShifts && - ((resultTypeIsTableOrRowSet && totalSize > 0) - || totalSize >= QueryTable.MINIMUM_PARALLEL_SELECT_ROWS)) { - final long divisionSize = resultTypeIsTableOrRowSet ? 1 - : Math.max(QueryTable.MINIMUM_PARALLEL_SELECT_ROWS, - (totalSize + jobScheduler.threadCount() - 1) / jobScheduler.threadCount()); - final List updates = new ArrayList<>(); - // divide up the additions and modifications - try (final RowSequence.Iterator rsAddIt = upstream.added().getRowSequenceIterator(); - final RowSequence.Iterator rsModIt = upstream.modified().getRowSequenceIterator()) { - while (rsAddIt.hasMore() || rsModIt.hasMore()) { - final TableUpdateImpl update = new TableUpdateImpl(); - update.modifiedColumnSet = upstream.modifiedColumnSet(); - update.shifted = RowSetShiftData.EMPTY; - update.removed = RowSetFactory.empty(); - - if (rsAddIt.hasMore()) { - update.added = rsAddIt.getNextRowSequenceWithLength(divisionSize).asRowSet(); - } else { - update.added = RowSetFactory.empty(); - } - - if (update.added.size() < divisionSize && rsModIt.hasMore()) { - update.modified = rsModIt - .getNextRowSequenceWithLength(divisionSize - update.added().size()) - .asRowSet(); - } else { - update.modified = RowSetFactory.empty(); - } - - updates.add(update); - } - } - - if (updates.isEmpty()) { - throw new IllegalStateException(); - } + return () -> { + // We don't want to bother with threads if we are going to process a small update + final long totalSize = upstream.added().size() + upstream.modified().size(); + + // If we have shifts, that makes everything nasty; so we do not want to deal with it + final boolean hasShifts = upstream.shifted().nonempty(); + + final boolean serialTableOperationsSafe = updateGraph.serialTableOperationsSafe() + || updateGraph.sharedLock().isHeldByCurrentThread() + || updateGraph.exclusiveLock().isHeldByCurrentThread(); + + if (canParallelizeThisColumn && jobScheduler.threadCount() > 1 && !hasShifts && + ((resultTypeIsTableOrRowSet && totalSize > 0) + || totalSize >= QueryTable.MINIMUM_PARALLEL_SELECT_ROWS)) { + final long divisionSize = resultTypeIsTableOrRowSet ? 1 + : Math.max(QueryTable.MINIMUM_PARALLEL_SELECT_ROWS, + (totalSize + jobScheduler.threadCount() - 1) / jobScheduler.threadCount()); + final List updates = new ArrayList<>(); + // divide up the additions and modifications + try (final RowSequence.Iterator rsAddIt = upstream.added().getRowSequenceIterator(); + final RowSequence.Iterator rsModIt = upstream.modified().getRowSequenceIterator()) { + while (rsAddIt.hasMore() || rsModIt.hasMore()) { + final TableUpdateImpl update = new TableUpdateImpl(); + update.modifiedColumnSet = upstream.modifiedColumnSet(); + update.shifted = RowSetShiftData.EMPTY; + update.removed = RowSetFactory.empty(); + + if (rsAddIt.hasMore()) { + update.added = rsAddIt.getNextRowSequenceWithLength(divisionSize).asRowSet(); + } else { + update.added = RowSetFactory.empty(); + } - jobScheduler.submit( - executionContext, - () -> prepareParallelUpdate(jobScheduler, upstream, toClear, helper, - liveResultOwner, onCompletion, this::onError, updates, - serialTableOperationsSafe), - SelectColumnLayer.this, this::onError); + if (update.added.size() < divisionSize && rsModIt.hasMore()) { + update.modified = rsModIt + .getNextRowSequenceWithLength(divisionSize - update.added().size()) + .asRowSet(); } else { - jobScheduler.submit( - executionContext, - () -> doSerialApplyUpdate(upstream, toClear, helper, liveResultOwner, onCompletion, - serialTableOperationsSafe), - SelectColumnLayer.this, this::onError); + update.modified = RowSetFactory.empty(); } + + updates.add(update); } - }); + } + + if (updates.isEmpty()) { + throw new IllegalStateException(); + } + + jobScheduler.submit( + executionContext, + () -> prepareParallelUpdate(jobScheduler, upstream, toClear, helper, liveResultOwner, onSuccess, + onError, updates, serialTableOperationsSafe), + SelectColumnLayer.this, onError); + } else { + jobScheduler.submit( + executionContext, + () -> doSerialApplyUpdate(upstream, toClear, helper, liveResultOwner, onSuccess, + serialTableOperationsSafe), + SelectColumnLayer.this, onError); + } + }; } - private void prepareParallelUpdate(final JobScheduler jobScheduler, final TableUpdate upstream, - final RowSet toClear, final UpdateHelper helper, @Nullable final LivenessNode liveResultOwner, - final SelectLayerCompletionHandler onCompletion, final Consumer onError, - final List splitUpdates, final boolean serialTableOperationsSafe) { + private void prepareParallelUpdate( + final JobScheduler jobScheduler, + final TableUpdate upstream, + final RowSet toClear, + final SelectAndViewAnalyzer.UpdateHelper helper, + @Nullable final LivenessNode liveResultOwner, + final Runnable onSuccess, + final Consumer onError, + final List splitUpdates, + final boolean serialTableOperationsSafe) { // we have to do removal and previous initialization before we can do any of the actual filling in multiple // threads to avoid concurrency problems with our destination column sources doEnsureCapacity(); @@ -250,13 +286,17 @@ private void prepareParallelUpdate(final JobScheduler jobScheduler, final TableU if (!isRedirected) { clearObjectsAtThisLevel(toClear); } - onCompletion.onLayerCompleted(getLayerIndex()); + onSuccess.run(); }, onError); } - private void doSerialApplyUpdate(final TableUpdate upstream, final RowSet toClear, final UpdateHelper helper, - @Nullable final LivenessNode liveResultOwner, final SelectLayerCompletionHandler onCompletion, + private void doSerialApplyUpdate( + final TableUpdate upstream, + final RowSet toClear, + final SelectAndViewAnalyzer.UpdateHelper helper, + @Nullable final LivenessNode liveResultOwner, + final Runnable onSuccess, final boolean serialTableOperationsSafe) { doEnsureCapacity(); final boolean oldSafe = updateGraph.setSerialTableOperationsSafe(serialTableOperationsSafe); @@ -269,11 +309,14 @@ private void doSerialApplyUpdate(final TableUpdate upstream, final RowSet toClea if (!isRedirected) { clearObjectsAtThisLevel(toClear); } - onCompletion.onLayerCompleted(getLayerIndex()); + onSuccess.run(); } - private void doParallelApplyUpdate(final TableUpdate upstream, final UpdateHelper helper, - @Nullable final LivenessNode liveResultOwner, final boolean serialTableOperationsSafe, + private void doParallelApplyUpdate( + final TableUpdate upstream, + final SelectAndViewAnalyzer.UpdateHelper helper, + @Nullable final LivenessNode liveResultOwner, + final boolean serialTableOperationsSafe, final long startOffset) { final boolean oldSafe = updateGraph.setSerialTableOperationsSafe(serialTableOperationsSafe); try { @@ -285,8 +328,11 @@ private void doParallelApplyUpdate(final TableUpdate upstream, final UpdateHelpe upstream.release(); } - private Boolean doApplyUpdate(final TableUpdate upstream, final UpdateHelper helper, - @Nullable final LivenessNode liveResultOwner, final long startOffset) { + private Boolean doApplyUpdate( + final TableUpdate upstream, + final SelectAndViewAnalyzer.UpdateHelper helper, + @Nullable final LivenessNode liveResultOwner, + final long startOffset) { final int PAGE_SIZE = 4096; final LongToIntFunction contextSize = (long size) -> size > PAGE_SIZE ? PAGE_SIZE : (int) size; @@ -594,16 +640,6 @@ private void clearObjectsAtThisLevel(RowSet keys) { } } - @Override - public boolean flattenedResult() { - return flattenedResult; - } - - @Override - public boolean alreadyFlattenedSources() { - return alreadyFlattenedSources; - } - @Override public LogOutput append(LogOutput logOutput) { return logOutput.append("{SelectColumnLayer: ").append(selectColumn.toString()).append(", layerIndex=") @@ -612,6 +648,6 @@ public LogOutput append(LogOutput logOutput) { @Override public boolean allowCrossColumnParallelization() { - return selectColumn.isStateless() && inner.allowCrossColumnParallelization(); + return selectColumn.isStateless(); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectOrViewColumnLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectOrViewColumnLayer.java index 5fbef5b9d74..ebf2ac05ec2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectOrViewColumnLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectOrViewColumnLayer.java @@ -12,18 +12,20 @@ public abstract class SelectOrViewColumnLayer extends DependencyLayerBase { private final ColumnSource optionalUnderlying; - SelectOrViewColumnLayer(SelectAndViewAnalyzer inner, String name, SelectColumn sc, - ColumnSource ws, ColumnSource optionalUnderlying, - String[] deps, ModifiedColumnSet mcsBuilder) { - super(inner, name, sc, ws, deps, mcsBuilder); + SelectOrViewColumnLayer( + final SelectAndViewAnalyzer.AnalyzerContext context, + final SelectColumn sc, + final ColumnSource ws, + final ColumnSource optionalUnderlying, + final String[] deps, + final ModifiedColumnSet mcsBuilder) { + super(context, sc, ws, deps, mcsBuilder); this.optionalUnderlying = optionalUnderlying; } @Override - final Map> getColumnSourcesRecurse(GetMode mode) { - final Map> result = inner.getColumnSourcesRecurse(mode); + void populateColumnSources(final Map> result) { result.put(name, columnSource); - return result; } @Override @@ -32,6 +34,5 @@ public void startTrackingPrev() { if (optionalUnderlying != null) { optionalUnderlying.startTrackingPrevValues(); } - inner.startTrackingPrev(); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java deleted file mode 100644 index 25827b2ca19..00000000000 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java +++ /dev/null @@ -1,146 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.engine.table.impl.select.analyzers; - -import io.deephaven.base.log.LogOutput; -import io.deephaven.base.verify.Assert; -import io.deephaven.engine.liveness.LivenessNode; -import io.deephaven.engine.rowset.RowSet; -import io.deephaven.engine.rowset.RowSetFactory; -import io.deephaven.engine.rowset.RowSetShiftData; -import io.deephaven.engine.rowset.TrackingRowSet; -import io.deephaven.engine.table.ColumnDefinition; -import io.deephaven.engine.table.ColumnSource; -import io.deephaven.engine.table.ModifiedColumnSet; -import io.deephaven.engine.table.TableUpdate; -import io.deephaven.engine.table.impl.TableUpdateImpl; -import io.deephaven.engine.table.impl.sources.RedirectedColumnSource; -import io.deephaven.engine.table.impl.util.RowRedirection; -import io.deephaven.engine.table.impl.util.WrappedRowSetRowRedirection; -import io.deephaven.engine.table.impl.util.JobScheduler; -import org.jetbrains.annotations.Nullable; - -import java.util.BitSet; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Set; - -final public class StaticFlattenLayer extends SelectAndViewAnalyzer { - private final SelectAndViewAnalyzer inner; - private final TrackingRowSet parentRowSet; - private final Map> overriddenColumns; - - StaticFlattenLayer(SelectAndViewAnalyzer inner, TrackingRowSet parentRowSet) { - super(inner.getLayerIndex() + 1); - this.inner = inner; - this.parentRowSet = parentRowSet; - final HashSet alreadyFlattenedColumns = new HashSet<>(); - inner.getNewColumnSources().forEach((name, cs) -> { - alreadyFlattenedColumns.add(name); - }); - - final RowRedirection rowRedirection = new WrappedRowSetRowRedirection(parentRowSet); - overriddenColumns = new HashMap<>(); - inner.getAllColumnSources().forEach((name, cs) -> { - if (alreadyFlattenedColumns.contains(name)) { - return; - } - - overriddenColumns.put(name, RedirectedColumnSource.maybeRedirect(rowRedirection, cs)); - }); - } - - @Override - void setBaseBits(BitSet bitset) { - inner.setBaseBits(bitset); - } - - @Override - void populateModifiedColumnSetRecurse(ModifiedColumnSet mcsBuilder, Set remainingDepsToSatisfy) { - inner.populateModifiedColumnSetRecurse(mcsBuilder, remainingDepsToSatisfy); - } - - @Override - Map> getColumnSourcesRecurse(GetMode mode) { - final Map> innerColumns = inner.getColumnSourcesRecurse(mode); - - if (overriddenColumns.keySet().stream().noneMatch(innerColumns::containsKey)) { - return innerColumns; - } - - final Map> columns = new LinkedHashMap<>(); - innerColumns.forEach((name, cs) -> columns.put(name, overriddenColumns.getOrDefault(name, cs))); - return columns; - } - - @Override - public void applyUpdate(TableUpdate upstream, RowSet toClear, UpdateHelper helper, JobScheduler jobScheduler, - @Nullable LivenessNode liveResultOwner, SelectLayerCompletionHandler onCompletion) { - // this must be the fake update used to initialize the result table - Assert.eqTrue(upstream.added().isFlat(), "upstream.added.isFlat()"); - Assert.eq(upstream.added().size(), "upstream.added.size()", parentRowSet.size(), "parentRowSet.size()"); - Assert.eqTrue(upstream.removed().isEmpty(), "upstream.removed.isEmpty()"); - Assert.eqTrue(upstream.modified().isEmpty(), "upstream.modified.isEmpty()"); - - final BitSet baseLayerBitSet = new BitSet(); - inner.setBaseBits(baseLayerBitSet); - final TableUpdate innerUpdate = new TableUpdateImpl( - parentRowSet.copy(), RowSetFactory.empty(), RowSetFactory.empty(), - RowSetShiftData.EMPTY, ModifiedColumnSet.EMPTY); - inner.applyUpdate(innerUpdate, toClear, helper, jobScheduler, liveResultOwner, - new SelectLayerCompletionHandler(baseLayerBitSet, onCompletion) { - @Override - public void onAllRequiredColumnsCompleted() { - onCompletion.onLayerCompleted(getLayerIndex()); - } - }); - } - - @Override - Map> calcDependsOnRecurse(boolean forcePublishAllResources) { - return inner.calcDependsOnRecurse(forcePublishAllResources); - } - - @Override - public SelectAndViewAnalyzer getInner() { - return inner; - } - - @Override - int getLayerIndexFor(String column) { - if (overriddenColumns.containsKey(column)) { - return getLayerIndex(); - } - return inner.getLayerIndexFor(column); - } - - @Override - public void startTrackingPrev() { - throw new UnsupportedOperationException("StaticFlattenLayer is used in only non-refreshing scenarios"); - } - - @Override - public LogOutput append(LogOutput logOutput) { - return logOutput.append("{StaticFlattenLayer").append(", layerIndex=").append(getLayerIndex()).append("}"); - } - - @Override - public boolean allowCrossColumnParallelization() { - return inner.allowCrossColumnParallelization(); - } - - @Override - public boolean flattenedResult() { - // this layer performs a flatten, so the result is flattened - return true; - } - - @Override - public boolean alreadyFlattenedSources() { - // this layer performs a flatten, so the sources are now flattened - return true; - } -} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/ViewColumnLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/ViewColumnLayer.java index 84bdda755a5..3019b5277b1 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/ViewColumnLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/ViewColumnLayer.java @@ -4,35 +4,30 @@ package io.deephaven.engine.table.impl.select.analyzers; import io.deephaven.base.log.LogOutput; -import io.deephaven.base.verify.Assert; import io.deephaven.configuration.Configuration; -import io.deephaven.engine.liveness.LivenessNode; import io.deephaven.engine.liveness.LivenessReferent; -import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.ModifiedColumnSet; -import io.deephaven.engine.table.TableUpdate; import io.deephaven.engine.table.impl.select.SelectColumn; -import io.deephaven.engine.table.impl.util.JobScheduler; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; final public class ViewColumnLayer extends SelectOrViewColumnLayer { private static final boolean ALLOW_LIVENESS_REFERENT_RESULTS = Configuration.getInstance() .getBooleanForClassWithDefault(ViewColumnLayer.class, "allowLivenessReferentResults", false); - ViewColumnLayer(SelectAndViewAnalyzer inner, String name, SelectColumn sc, ColumnSource cs, String[] deps, - ModifiedColumnSet mcsBuilder) { - super(inner, name, sc, checkResultType(cs), null, deps, mcsBuilder); + ViewColumnLayer( + final SelectAndViewAnalyzer.AnalyzerContext context, + final SelectColumn sc, + final ColumnSource cs, + final String[] deps, + final ModifiedColumnSet mcsBuilder) { + super(context, sc, checkResultType(cs), null, deps, mcsBuilder); } @Override - public void applyUpdate(TableUpdate upstream, RowSet toClear, UpdateHelper helper, JobScheduler jobScheduler, - @Nullable LivenessNode liveResultOwner, SelectLayerCompletionHandler completionHandler) { - // To be parallel with SelectColumnLayer, we would recurse here, but since this is ViewColumnLayer - // (and all my inner layers are ViewColumnLayer), there's nothing to do. - Assert.eqNull(completionHandler, "completionHandler"); + public boolean hasRefreshingLogic() { + return false; } @Override @@ -47,7 +42,7 @@ public boolean allowCrossColumnParallelization() { return false; } - private static ColumnSource checkResultType(@NotNull final ColumnSource cs) { + private static ColumnSource checkResultType(@NotNull final ColumnSource cs) { final Class resultType = cs.getType(); if (!ALLOW_LIVENESS_REFERENT_RESULTS && LivenessReferent.class.isAssignableFrom(resultType)) { throw new UnsupportedOperationException(String.format( diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java index fe162c1f305..0ef5b12f59e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java @@ -3,6 +3,7 @@ // package io.deephaven.engine.table.impl.select.codegen; +import io.deephaven.api.util.NameValidator; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryLibrary; import io.deephaven.engine.table.ColumnDefinition; @@ -12,8 +13,6 @@ import io.deephaven.vector.ObjectVector; import io.deephaven.engine.table.impl.select.DhFormulaColumn; import io.deephaven.engine.table.impl.select.formula.FormulaSourceDescriptor; -import io.deephaven.engine.table.WritableColumnSource; -import io.deephaven.engine.rowset.TrackingWritableRowSet; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import org.jetbrains.annotations.NotNull; @@ -28,9 +27,45 @@ public class FormulaAnalyzer { private static final Logger log = LoggerFactory.getLogger(FormulaAnalyzer.class); + /** + * A container to hold a single copy of imports required to compile formulas for one operation. + */ + public static final class Imports { + private final Map queryScopeVariables; + private final Collection packageImports; + private final Collection> classImports; + private final Collection> staticImports; + + public Imports() { + final ExecutionContext context = ExecutionContext.getContext(); + queryScopeVariables = Collections.unmodifiableMap( + context.getQueryScope().toMap((name, value) -> NameValidator.isValidQueryParameterName(name))); + final QueryLibrary queryLibrary = context.getQueryLibrary(); + packageImports = Set.copyOf(queryLibrary.getPackageImports()); + classImports = Set.copyOf(queryLibrary.getClassImports()); + staticImports = Set.copyOf(queryLibrary.getStaticImports()); + } + + public Map getQueryScopeVariables() { + return queryScopeVariables; + } + + public Collection getPackageImports() { + return packageImports; + } + + public Collection> getClassImports() { + return classImports; + } + + public Collection> getStaticImports() { + return staticImports; + } + } + public static Result analyze(final String rawFormulaString, final Map> columnDefinitionMap, - final QueryLanguageParser.Result queryLanguageResult) throws Exception { + final QueryLanguageParser.Result queryLanguageResult) { log.debug().append("Expression (after language conversion) : ") .append(queryLanguageResult.getConvertedExpression()) @@ -75,7 +110,7 @@ public static Result analyze(final String rawFormulaString, * @param formulaString The raw formula string * @param availableColumns The columns available for use in the formula * @param columnRenames Outer to inner column name mapping - * @param queryScopeVariables The query scope variables + * @param imports The query scope variables, package, class, and static imports * @return The parsed formula {@link QueryLanguageParser.Result result} * @throws Exception If the formula cannot be parsed */ @@ -83,8 +118,8 @@ public static QueryLanguageParser.Result parseFormula( @NotNull final String formulaString, @NotNull final Map> availableColumns, @NotNull final Map columnRenames, - @NotNull final Map queryScopeVariables) throws Exception { - return parseFormula(formulaString, availableColumns, columnRenames, queryScopeVariables, true); + @NotNull final Imports imports) throws Exception { + return parseFormula(formulaString, availableColumns, columnRenames, imports, true); } /** @@ -93,7 +128,7 @@ public static QueryLanguageParser.Result parseFormula( * @param formulaString The raw formula string * @param availableColumns The columns available for use in the formula * @param columnRenames Outer to inner column name mapping - * @param queryScopeVariables The query scope variables + * @param imports The query scope variables, package, class, and static imports * @param unboxArguments If true it will unbox the query scope arguments * @return The parsed formula {@link QueryLanguageParser.Result result} * @throws Exception If the formula cannot be parsed @@ -102,7 +137,7 @@ public static QueryLanguageParser.Result parseFormula( @NotNull final String formulaString, @NotNull final Map> availableColumns, @NotNull final Map columnRenames, - @NotNull final Map queryScopeVariables, + @NotNull final Imports imports, final boolean unboxArguments) throws Exception { final TimeLiteralReplacedExpression timeConversionResult = @@ -177,7 +212,7 @@ public static QueryLanguageParser.Result parseFormula( } // Parameters come last. - for (Map.Entry param : queryScopeVariables.entrySet()) { + for (Map.Entry param : imports.queryScopeVariables.entrySet()) { if (possibleVariables.containsKey(param.getKey())) { // Columns and column arrays take precedence over parameters. continue; @@ -200,13 +235,10 @@ public static QueryLanguageParser.Result parseFormula( possibleVariables.putAll(timeConversionResult.getNewVariables()); - final QueryLibrary queryLibrary = ExecutionContext.getContext().getQueryLibrary(); - final Set> classImports = new HashSet<>(queryLibrary.getClassImports()); - classImports.add(TrackingWritableRowSet.class); - classImports.add(WritableColumnSource.class); - return new QueryLanguageParser(timeConversionResult.getConvertedFormula(), queryLibrary.getPackageImports(), - classImports, queryLibrary.getStaticImports(), possibleVariables, possibleVariableParameterizedTypes, - queryScopeVariables, columnVariables, unboxArguments, timeConversionResult).getResult(); + return new QueryLanguageParser(timeConversionResult.getConvertedFormula(), imports.getPackageImports(), + imports.getClassImports(), imports.getStaticImports(), possibleVariables, + possibleVariableParameterizedTypes, imports.getQueryScopeVariables(), columnVariables, unboxArguments, + timeConversionResult).getResult(); } public static class Result { diff --git a/engine/table/src/main/java/io/deephaven/engine/util/BigDecimalUtils.java b/engine/table/src/main/java/io/deephaven/engine/util/BigDecimalUtils.java index fe192c436da..eab692ec961 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/BigDecimalUtils.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/BigDecimalUtils.java @@ -3,31 +3,37 @@ // package io.deephaven.engine.util; -import io.deephaven.chunk.ObjectChunk; -import io.deephaven.chunk.attributes.Values; -import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.primitive.iterator.CloseableIterator; import io.deephaven.engine.rowset.RowSet; -import io.deephaven.engine.table.ChunkSource; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.Table; +import io.deephaven.engine.table.vectors.ObjectVectorColumnWrapper; +import io.deephaven.vector.ObjectVector; +import io.deephaven.vector.Vector; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.math.BigDecimal; +import java.util.Iterator; import java.util.Properties; /** * Utilities to support BigDecimal exhaust. - * + *

* Parquet and Avro decimal types make a whole column decimal type have a fixed precision and scale; BigDecimal columns * in Deephaven are, each value, arbitrary precision (its own precision and scale). - * + *

* For static tables, it is possible to compute overall precision and scale values that fit every existing value. For * refreshing tables, we need the user to tell us. */ public class BigDecimalUtils { - private static final PrecisionAndScale EMPTY_TABLE_PRECISION_AND_SCALE = new PrecisionAndScale(1, 1); - private static final int TARGET_CHUNK_SIZE = 4096; + public static final int INVALID_PRECISION_OR_SCALE = -1; + private static final PrecisionAndScale EMPTY_TABLE_PRECISION_AND_SCALE = new PrecisionAndScale(1, 1); + private static final int INIT_MAX_PRECISION_MINUS_SCALE = -1; + private static final int INIT_MAX_SCALE = -1; + /** * Immutable way to store and pass precision and scale values. */ @@ -44,14 +50,16 @@ public PrecisionAndScale(final int precision, final int scale) { /** * Compute an overall precision and scale that would fit all existing values in a table. * - * @param t a Deephaven table - * @param colName a Column for {@code t}, which should be of {@code BigDecimal} type - * @return a {@code PrecisionAndScale} object result. + * @param table A Deephaven table + * @param colName Column for {@code table}, which should be of {@code BigDecimal} {@link ColumnSource#getType type} + * or {@link ColumnSource#getComponentType component type} + * @return A {@link PrecisionAndScale} object result. */ public static PrecisionAndScale computePrecisionAndScale( - final Table t, final String colName) { - final ColumnSource src = t.getColumnSource(colName, BigDecimal.class); - return computePrecisionAndScale(t.getRowSet(), src); + final Table table, + final String colName) { + final ColumnSource src = table.getColumnSource(colName); + return computePrecisionAndScale(table.getRowSet(), src); } /** @@ -59,12 +67,13 @@ public static PrecisionAndScale computePrecisionAndScale( * requires a full table scan to ensure the correct values are determined. * * @param rowSet The rowset for the provided column - * @param source a {@code ColumnSource} of {@code BigDecimal} type - * @return a {@code PrecisionAndScale} object result. + * @param columnSource A {@code ColumnSource} of {@code BigDecimal} {@link ColumnSource#getType type} or + * {@link ColumnSource#getComponentType component type} + * @return A {@link PrecisionAndScale} object result. */ public static PrecisionAndScale computePrecisionAndScale( final RowSet rowSet, - final ColumnSource source) { + final ColumnSource columnSource) { if (rowSet.isEmpty()) { return EMPTY_TABLE_PRECISION_AND_SCALE; } @@ -72,39 +81,94 @@ public static PrecisionAndScale computePrecisionAndScale( // We will walk the entire table to determine the max(precision - scale) and // max(scale), which corresponds to max(digits left of the decimal point), max(digits right of the decimal // point). Then we convert to (precision, scale) before returning. - int maxPrecisionMinusScale = -1; - int maxScale = -1; - try (final ChunkSource.GetContext context = source.makeGetContext(TARGET_CHUNK_SIZE); - final RowSequence.Iterator it = rowSet.getRowSequenceIterator()) { - while (it.hasMore()) { - final RowSequence rowSeq = it.getNextRowSequenceWithLength(TARGET_CHUNK_SIZE); - final ObjectChunk chunk = - source.getChunk(context, rowSeq).asObjectChunk(); - for (int i = 0; i < chunk.size(); ++i) { - final BigDecimal x = chunk.get(i); - if (x == null) { - continue; - } - - final int precision = x.precision(); - final int scale = x.scale(); - final int precisionMinusScale = precision - scale; - if (precisionMinusScale > maxPrecisionMinusScale) { - maxPrecisionMinusScale = precisionMinusScale; - } - if (scale > maxScale) { - maxScale = scale; - } + final BigDecimalParameters result = new BigDecimalParameters(INIT_MAX_PRECISION_MINUS_SCALE, INIT_MAX_SCALE); + final ObjectVector columnVector = new ObjectVectorColumnWrapper<>(columnSource, rowSet); + try (final CloseableIterator columnIterator = columnVector.iterator()) { + final Class columnType = columnSource.getType(); + if (columnType == BigDecimal.class) { + // noinspection unchecked + processFlatColumn((Iterator) columnIterator, result); + } else if (columnSource.getComponentType() == BigDecimal.class) { + if (columnType.isArray()) { + // noinspection unchecked + processArrayColumn((Iterator) columnIterator, result); + } else if (Vector.class.isAssignableFrom(columnType)) { + // noinspection unchecked + processVectorColumn((Iterator>) columnIterator, result); } + } else { + throw new IllegalArgumentException("Column source is not of type BigDecimal or an array/vector of " + + "BigDecimal, but of type " + columnType + " and component type " + + columnSource.getComponentType()); } } - // If these are < 0, then every value we visited was null - if (maxPrecisionMinusScale < 0 && maxScale < 0) { + // If these are same as initial values, then every value we visited was null + if (result.maxPrecisionMinusScale == INIT_MAX_PRECISION_MINUS_SCALE && result.maxScale == INIT_MAX_SCALE) { return EMPTY_TABLE_PRECISION_AND_SCALE; } - return new PrecisionAndScale(maxPrecisionMinusScale + maxScale, maxScale); + return new PrecisionAndScale(result.maxPrecisionMinusScale + result.maxScale, result.maxScale); + } + + private static class BigDecimalParameters { + private int maxPrecisionMinusScale; + private int maxScale; + + private BigDecimalParameters(final int maxPrecisionMinusScale, final int maxScale) { + this.maxPrecisionMinusScale = maxPrecisionMinusScale; + this.maxScale = maxScale; + } + + /** + * Update the maximum values for the parameters based on the given value. + */ + private void updateMaximum(@Nullable final BigDecimal value) { + if (value == null) { + return; + } + final int precision = value.precision(); + final int scale = value.scale(); + final int precisionMinusScale = precision - scale; + if (precisionMinusScale > maxPrecisionMinusScale) { + maxPrecisionMinusScale = precisionMinusScale; + } + if (scale > maxScale) { + maxScale = scale; + } + } + } + + private static void processFlatColumn( + @NotNull final Iterator columnIterator, + @NotNull final BigDecimalParameters result) { + columnIterator.forEachRemaining(result::updateMaximum); + } + + private static void processVectorColumn( + @NotNull final Iterator> columnIterator, + @NotNull final BigDecimalParameters result) { + columnIterator.forEachRemaining(values -> { + if (values == null) { + return; + } + try (final CloseableIterator valuesIterator = values.iterator()) { + valuesIterator.forEachRemaining(result::updateMaximum); + } + }); + } + + private static void processArrayColumn( + @NotNull final Iterator columnIterator, + @NotNull final BigDecimalParameters result) { + columnIterator.forEachRemaining(values -> { + if (values == null) { + return; + } + for (final BigDecimal value : values) { + result.updateMaximum(value); + } + }); } /** diff --git a/engine/table/src/main/java/io/deephaven/engine/util/TableDiff.java b/engine/table/src/main/java/io/deephaven/engine/util/TableDiff.java index 179a3b9fc32..156b8219c58 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/TableDiff.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/TableDiff.java @@ -182,7 +182,7 @@ public enum DiffItems { */ DoublesExact, /** - * Columns that exist in both tables, but in diferent orders are not treated as differences. + * Columns that exist in both tables, but in different orders are not treated as differences. */ ColumnsOrder, /** diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/DummyTableLocation.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/DummyTableLocation.java new file mode 100644 index 00000000000..752c74b9414 --- /dev/null +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/DummyTableLocation.java @@ -0,0 +1,131 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.table.impl; + +import io.deephaven.api.SortColumn; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.table.BasicDataIndex; +import io.deephaven.engine.table.ColumnDefinition; +import io.deephaven.engine.table.impl.locations.ColumnLocation; +import io.deephaven.engine.table.impl.locations.TableKey; +import io.deephaven.engine.table.impl.locations.TableLocation; +import io.deephaven.engine.table.impl.locations.TableLocationKey; +import io.deephaven.engine.table.impl.locations.impl.AbstractTableLocation; +import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionByte; +import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionChar; +import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionDouble; +import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionFloat; +import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionInt; +import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionLong; +import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionObject; +import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionShort; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.List; + +public final class DummyTableLocation extends AbstractTableLocation { + + public DummyTableLocation(@NotNull final TableKey tableKey, + @NotNull final TableLocationKey tableLocationKey) { + super(tableKey, tableLocationKey, false); + } + + @Override + public void refresh() {} + + @Override + public @NotNull List getSortedColumns() { + return List.of(); + } + + @Override + @NotNull + public List getDataIndexColumns() { + return List.of(); + } + + @Override + public boolean hasDataIndex(@NotNull final String... columns) { + return false; + } + + @NotNull + @Override + protected ColumnLocation makeColumnLocation(@NotNull final String name) { + return new ColumnLocation() { + @NotNull + @Override + public TableLocation getTableLocation() { + return DummyTableLocation.this; + } + + @NotNull + @Override + public String getName() { + return name; + } + + @Override + public boolean exists() { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnRegionChar makeColumnRegionChar( + @NotNull final ColumnDefinition columnDefinition) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnRegionByte makeColumnRegionByte( + @NotNull final ColumnDefinition columnDefinition) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnRegionShort makeColumnRegionShort( + @NotNull final ColumnDefinition columnDefinition) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnRegionInt makeColumnRegionInt( + @NotNull final ColumnDefinition columnDefinition) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnRegionLong makeColumnRegionLong( + @NotNull final ColumnDefinition columnDefinition) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnRegionFloat makeColumnRegionFloat( + @NotNull final ColumnDefinition columnDefinition) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnRegionDouble makeColumnRegionDouble( + @NotNull final ColumnDefinition columnDefinition) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnRegionObject makeColumnRegionObject( + @NotNull final ColumnDefinition columnDefinition) { + throw new UnsupportedOperationException(); + } + + }; + } + + @Override + @Nullable + public BasicDataIndex loadDataIndex(@NotNull final String... columns) { + throw new UnsupportedOperationException(); + } +} diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java index 65b27a6b380..0659f11a139 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java @@ -5,12 +5,23 @@ import gnu.trove.list.TLongList; import gnu.trove.list.array.TLongArrayList; +import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.RowSetFactory; +import io.deephaven.engine.rowset.RowSetShiftData; import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.ModifiedColumnSet; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.TableUpdate; import io.deephaven.engine.table.impl.select.WhereFilter; import io.deephaven.engine.table.impl.select.WhereFilterImpl; +import io.deephaven.engine.table.impl.sources.RowKeyColumnSource; +import io.deephaven.engine.table.impl.sources.RowPositionColumnSource; +import io.deephaven.engine.testutil.ControlledUpdateGraph; +import io.deephaven.engine.testutil.EvalNugget; +import io.deephaven.engine.testutil.EvalNuggetInterface; +import io.deephaven.engine.testutil.TstUtils; import io.deephaven.engine.util.TableTools; import io.deephaven.test.types.OutOfBandTest; import org.jetbrains.annotations.NotNull; @@ -21,6 +32,7 @@ import java.util.Collections; import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; @@ -119,4 +131,40 @@ public WhereFilter copy() { QueryTable.PARALLEL_WHERE_ROWS_PER_SEGMENT = oldSize; } } + + @Test + public void testParallelExecutionViaTableUpdate() { + final ControlledUpdateGraph updateGraph = ExecutionContext.getContext().getUpdateGraph().cast(); + + final long oldSize = QueryTable.PARALLEL_WHERE_ROWS_PER_SEGMENT; + try { + QueryTable.PARALLEL_WHERE_ROWS_PER_SEGMENT = 1_000; + final QueryTable table = TstUtils.testRefreshingTable(RowSetFactory.flat(1500).toTracking()) + .withAdditionalColumns(Map.of("K", new RowKeyColumnSource())); + table.setRefreshing(true); + table.setAttribute(BaseTable.TEST_SOURCE_TABLE_ATTRIBUTE, true); + final Table source = table.updateView("J = ii % 2 == 0 ? K : 0"); + + final EvalNuggetInterface[] en = new EvalNuggetInterface[] { + EvalNugget.from(() -> source.where("K == J")), + }; + + updateGraph.runWithinUnitTestCycle(() -> { + final RowSet added = RowSetFactory.fromRange(1500, 2999); + final RowSet modified = RowSetFactory.fromRange(0, 1499); + table.getRowSet().writableCast().insert(added); + + final TableUpdate upstream = new TableUpdateImpl( + added, RowSetFactory.empty(), modified, RowSetShiftData.EMPTY, ModifiedColumnSet.ALL); + + table.notifyListeners(upstream); + }); + + // Ensure the table is as expected. + TstUtils.validate(en); + + } finally { + QueryTable.PARALLEL_WHERE_ROWS_PER_SEGMENT = oldSize; + } + } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/TestPartitioningColumns.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/TestPartitioningColumns.java index adb04b08fa5..63cc10f6a7d 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/TestPartitioningColumns.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/TestPartitioningColumns.java @@ -3,10 +3,8 @@ // package io.deephaven.engine.table.impl; -import io.deephaven.api.SortColumn; import io.deephaven.api.filter.Filter; import io.deephaven.base.verify.Assert; -import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.primitive.iterator.CloseableIterator; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.table.*; @@ -17,17 +15,11 @@ import io.deephaven.engine.testutil.TstUtils; import io.deephaven.engine.testutil.junit4.EngineCleanup; import io.deephaven.time.DateTimeUtils; -import io.deephaven.engine.table.impl.locations.ColumnLocation; -import io.deephaven.engine.table.impl.locations.TableKey; -import io.deephaven.engine.table.impl.locations.TableLocation; -import io.deephaven.engine.table.impl.locations.TableLocationKey; import io.deephaven.engine.table.impl.locations.impl.*; import io.deephaven.engine.table.impl.select.MatchFilter; import io.deephaven.engine.table.impl.select.WhereFilter; import io.deephaven.engine.table.impl.sources.regioned.*; import io.deephaven.engine.rowset.RowSetFactory; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.junit.Rule; import org.junit.Test; @@ -135,109 +127,4 @@ public void testEverything() { TstUtils.assertTableEquals(expected.selectDistinct(), result.selectDistinct()); } - - private static final class DummyTableLocation extends AbstractTableLocation { - - private DummyTableLocation(@NotNull final TableKey tableKey, - @NotNull final TableLocationKey tableLocationKey) { - super(tableKey, tableLocationKey, false); - } - - @Override - public void refresh() {} - - @Override - public @NotNull List getSortedColumns() { - return List.of(); - } - - @Override - @NotNull - public List getDataIndexColumns() { - return List.of(); - } - - @Override - public boolean hasDataIndex(@NotNull final String... columns) { - return false; - } - - @NotNull - @Override - protected ColumnLocation makeColumnLocation(@NotNull final String name) { - return new ColumnLocation() { - @NotNull - @Override - public TableLocation getTableLocation() { - return DummyTableLocation.this; - } - - @NotNull - @Override - public String getName() { - return name; - } - - @Override - public boolean exists() { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnRegionChar makeColumnRegionChar( - @NotNull final ColumnDefinition columnDefinition) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnRegionByte makeColumnRegionByte( - @NotNull final ColumnDefinition columnDefinition) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnRegionShort makeColumnRegionShort( - @NotNull final ColumnDefinition columnDefinition) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnRegionInt makeColumnRegionInt( - @NotNull final ColumnDefinition columnDefinition) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnRegionLong makeColumnRegionLong( - @NotNull final ColumnDefinition columnDefinition) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnRegionFloat makeColumnRegionFloat( - @NotNull final ColumnDefinition columnDefinition) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnRegionDouble makeColumnRegionDouble( - @NotNull final ColumnDefinition columnDefinition) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnRegionObject makeColumnRegionObject( - @NotNull final ColumnDefinition columnDefinition) { - throw new UnsupportedOperationException(); - } - - }; - } - - @Override - @Nullable - public BasicDataIndex loadDataIndex(@NotNull final String... columns) { - throw new UnsupportedOperationException(); - } - } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java index 32523cbf766..4f1558f0c1d 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java @@ -3178,7 +3178,7 @@ private void check(String expression, String resultExpression, Class resultTy final Map possibleParams; final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); if (!(queryScope instanceof PoisonedQueryScope)) { - possibleParams = QueryCompilerRequestProcessor.newQueryScopeVariableSupplier().get(); + possibleParams = QueryCompilerRequestProcessor.newFormulaImportsSupplier().get().getQueryScopeVariables(); } else { possibleParams = null; } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/locations/impl/TestTableDataService.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/locations/impl/TestTableDataService.java new file mode 100644 index 00000000000..1687a46d5ed --- /dev/null +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/locations/impl/TestTableDataService.java @@ -0,0 +1,96 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.table.impl.locations.impl; + +import io.deephaven.engine.table.impl.DummyTableLocation; +import io.deephaven.engine.table.impl.locations.TableDataException; +import io.deephaven.engine.table.impl.locations.TableDataService; +import io.deephaven.engine.table.impl.locations.TableKey; +import io.deephaven.engine.table.impl.locations.TableLocation; +import io.deephaven.engine.table.impl.locations.TableLocationKey; +import io.deephaven.engine.table.impl.locations.TableLocationProvider; +import org.jetbrains.annotations.NotNull; +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class TestTableDataService { + @Test + public void testGetRawTableDataService() { + final Map> partitions1 = new HashMap<>(); + partitions1.put("Part", "A"); + final Map> partitions2 = new HashMap<>(); + partitions2.put("Part", "B"); + final Map> partitions3 = new HashMap<>(); + partitions3.put("Part", "C"); + + TableLocationKey tlk1 = new SimpleTableLocationKey(partitions1); + TableLocationKey tlk2 = new SimpleTableLocationKey(partitions2); + TableLocationKey tlk3 = new SimpleTableLocationKey(partitions3); + + // No table location overlap + final CompositeTableDataService ctds1 = + new CompositeTableDataService("ctds1", new DummyServiceSelector(tlk1, tlk2)); + Assert.assertNotNull(ctds1.getRawTableLocationProvider(StandaloneTableKey.getInstance(), tlk1)); + Assert.assertNull(ctds1.getRawTableLocationProvider(StandaloneTableKey.getInstance(), tlk3)); + + // Table location overlap + final CompositeTableDataService ctds2 = + new CompositeTableDataService("ctds2", new DummyServiceSelector(tlk1, tlk1)); + Assert.assertThrows(TableDataException.class, + () -> ctds2.getRawTableLocationProvider(StandaloneTableKey.getInstance(), tlk1)); + Assert.assertNull(ctds2.getRawTableLocationProvider(StandaloneTableKey.getInstance(), tlk3)); + } + + private static class DummyTableDataService extends AbstractTableDataService { + final TableLocation tableLocation; + + private DummyTableDataService(@NotNull final String name, @NotNull final TableLocation tableLocation) { + super(name); + this.tableLocation = tableLocation; + } + + @Override + @NotNull + protected TableLocationProvider makeTableLocationProvider(@NotNull TableKey tableKey) { + return new SingleTableLocationProvider(tableLocation); + } + } + + private static class DummyServiceSelector implements CompositeTableDataService.ServiceSelector { + final TableDataService[] tableDataServices; + + private DummyServiceSelector(final TableLocationKey tlk1, final TableLocationKey tlk2) { + final FilteredTableDataService.LocationKeyFilter dummyFilter = (tlk) -> true; + tableDataServices = new TableDataService[] { + new FilteredTableDataService(new DummyTableDataService("dummyTds1", + new DummyTableLocation(StandaloneTableKey.getInstance(), tlk1)), dummyFilter), + new FilteredTableDataService(new DummyTableDataService("dummyTds2", + new DummyTableLocation(StandaloneTableKey.getInstance(), tlk2)), dummyFilter) + }; + // Init table locations + tableDataServices[0].getTableLocationProvider(StandaloneTableKey.getInstance()) + .getTableLocationIfPresent(tlk1); + tableDataServices[1].getTableLocationProvider(StandaloneTableKey.getInstance()) + .getTableLocationIfPresent(tlk2); + } + + @Override + public TableDataService[] call(@NotNull TableKey tableKey) { + return tableDataServices; + } + + @Override + public void resetServices() { + throw new UnsupportedOperationException(); + } + + @Override + public void resetServices(@NotNull TableKey key) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/engine/time/src/main/java/io/deephaven/time/calendar/BusinessCalendarXMLParser.java b/engine/time/src/main/java/io/deephaven/time/calendar/BusinessCalendarXMLParser.java index 6f2c0432b9b..7c430b43cc6 100644 --- a/engine/time/src/main/java/io/deephaven/time/calendar/BusinessCalendarXMLParser.java +++ b/engine/time/src/main/java/io/deephaven/time/calendar/BusinessCalendarXMLParser.java @@ -35,7 +35,12 @@ * New York Stock Exchange Calendar * America/New_York * - * 09:3016:00 + * + * 09:30 + * 16:00 + * + * true + * * Saturday * Sunday * @@ -51,6 +56,8 @@ * * 09:30 * 13:00 + * + * true * * * @@ -254,6 +261,7 @@ private static TimeRange[] parseBusinessRanges(final List bu for (int i = 0; i < businessRanges.size(); i++) { final String openTxt = getText(getRequiredChild(businessRanges.get(i), "open")); final String closeTxt = getText(getRequiredChild(businessRanges.get(i), "close")); + final String includeCloseTxt = getText(businessRanges.get(i).getChild("includeClose")); if (closeTxt.startsWith("24:00")) { throw new RuntimeException("Close time (" + closeTxt @@ -262,7 +270,8 @@ private static TimeRange[] parseBusinessRanges(final List bu final LocalTime open = DateTimeUtils.parseLocalTime(openTxt); final LocalTime close = DateTimeUtils.parseLocalTime(closeTxt); - rst[i] = new TimeRange<>(open, close, true); + final boolean inclusiveEnd = Boolean.parseBoolean(includeCloseTxt); // defaults to false + rst[i] = new TimeRange<>(open, close, inclusiveEnd); } return rst; @@ -282,7 +291,7 @@ private static TimeRange[] parseBusinessRangesLegacy(final List(open, close, true); + rst[i] = new TimeRange<>(open, close, false); } else { throw new IllegalArgumentException("Can not parse business periods; open/close = " + br.getText()); } diff --git a/engine/time/src/main/java/io/deephaven/time/calendar/TimeRange.java b/engine/time/src/main/java/io/deephaven/time/calendar/TimeRange.java index e412673eb9d..837f8a23fca 100644 --- a/engine/time/src/main/java/io/deephaven/time/calendar/TimeRange.java +++ b/engine/time/src/main/java/io/deephaven/time/calendar/TimeRange.java @@ -81,7 +81,7 @@ public boolean isInclusiveEnd() { * @return length of the range in nanoseconds */ public long nanos() { - return start.until(end, ChronoUnit.NANOS) - (inclusiveEnd ? 0 : 1); + return start.until(end, ChronoUnit.NANOS) + (inclusiveEnd ? 1 : 0); } /** diff --git a/engine/time/src/test/java/io/deephaven/time/calendar/TestBusinessCalendarXMLParser.java b/engine/time/src/test/java/io/deephaven/time/calendar/TestBusinessCalendarXMLParser.java index a8a4a75ee0e..43635475aa7 100644 --- a/engine/time/src/test/java/io/deephaven/time/calendar/TestBusinessCalendarXMLParser.java +++ b/engine/time/src/test/java/io/deephaven/time/calendar/TestBusinessCalendarXMLParser.java @@ -36,6 +36,9 @@ public static void assertParserTestCal(final BusinessCalendar cal) { cal.calendarDay("2015-04-06").businessStart()); assertEquals(DateTimeUtils.parseInstant("2015-04-06T16:46 Asia/Tokyo"), cal.calendarDay("2015-04-06").businessEnd()); + + assertTrue(cal.calendarDay("2015-04-06").isInclusiveEnd()); + assertFalse(cal.calendarDay("2015-04-07").isInclusiveEnd()); } public void testLoad() throws URISyntaxException { diff --git a/engine/time/src/test/java/io/deephaven/time/calendar/TestCalendarDay.java b/engine/time/src/test/java/io/deephaven/time/calendar/TestCalendarDay.java index 367ebbdb7a0..93f2dd5520e 100644 --- a/engine/time/src/test/java/io/deephaven/time/calendar/TestCalendarDay.java +++ b/engine/time/src/test/java/io/deephaven/time/calendar/TestCalendarDay.java @@ -61,10 +61,10 @@ public void testSinglePeriod() { assertEquals(close1, single.businessEnd()); assertEquals(close1, single.businessEnd()); assertTrue(single.isInclusiveEnd()); - assertEquals(DateTimeUtils.HOUR, single.businessNanos()); - assertEquals(Duration.ofNanos(DateTimeUtils.HOUR), single.businessDuration()); - assertEquals(DateTimeUtils.HOUR, single.businessNanos()); - assertEquals(Duration.ofNanos(DateTimeUtils.HOUR), single.businessDuration()); + assertEquals(DateTimeUtils.HOUR + 1, single.businessNanos()); + assertEquals(Duration.ofNanos(DateTimeUtils.HOUR + 1), single.businessDuration()); + assertEquals(DateTimeUtils.HOUR + 1, single.businessNanos()); + assertEquals(Duration.ofNanos(DateTimeUtils.HOUR + 1), single.businessDuration()); assertTrue(single.isBusinessDay()); assertTrue(single.isBusinessTime(DateTimeUtils.parseInstant("2017-03-11T10:00:00.000000000 NY"))); assertTrue(single.isBusinessTime(DateTimeUtils.parseInstant("2017-03-11T10:15:00.000000000 NY"))); @@ -77,13 +77,13 @@ public void testSinglePeriod() { single.businessNanosElapsed(DateTimeUtils.parseInstant("2017-03-11T10:30:00.000000000 NY"))); assertEquals(Duration.ofNanos(DateTimeUtils.MINUTE * 30), single.businessDurationElapsed(DateTimeUtils.parseInstant("2017-03-11T10:30:00.000000000 NY"))); - assertEquals(DateTimeUtils.HOUR, + assertEquals(DateTimeUtils.HOUR + 1, single.businessNanosElapsed(DateTimeUtils.parseInstant("2017-03-11T13:00:00.000000000 NY"))); - assertEquals(Duration.ofNanos(DateTimeUtils.HOUR), + assertEquals(Duration.ofNanos(DateTimeUtils.HOUR + 1), single.businessDurationElapsed(DateTimeUtils.parseInstant("2017-03-11T13:00:00.000000000 NY"))); - assertEquals(DateTimeUtils.MINUTE * 30, + assertEquals(DateTimeUtils.MINUTE * 30 + 1, single.businessNanosRemaining(DateTimeUtils.parseInstant("2017-03-11T10:30:00.000000000 NY"))); - assertEquals(Duration.ofNanos(DateTimeUtils.MINUTE * 30), + assertEquals(Duration.ofNanos(DateTimeUtils.MINUTE * 30 + 1), single.businessDurationRemaining(DateTimeUtils.parseInstant("2017-03-11T10:30:00.000000000 NY"))); assertEquals(0L, single.businessNanosRemaining(DateTimeUtils.parseInstant("2017-03-11T13:00:00.000000000 NY"))); @@ -105,10 +105,10 @@ public void testMultiPeriod() { assertEquals(close2, multi.businessEnd()); assertEquals(close2, multi.businessEnd()); assertTrue(multi.isInclusiveEnd()); - assertEquals(DateTimeUtils.HOUR * 6, multi.businessNanos()); - assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 6), multi.businessDuration()); - assertEquals(DateTimeUtils.HOUR * 6, multi.businessNanos()); - assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 6), multi.businessDuration()); + assertEquals(DateTimeUtils.HOUR * 6 + 2, multi.businessNanos()); + assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 6 + 2), multi.businessDuration()); + assertEquals(DateTimeUtils.HOUR * 6 + 2, multi.businessNanos()); + assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 6 + 2), multi.businessDuration()); assertTrue(multi.isBusinessDay()); assertTrue(multi.isBusinessTime(DateTimeUtils.parseInstant("2017-03-11T10:00:00.000000000 NY"))); assertTrue(multi.isBusinessTime(DateTimeUtils.parseInstant("2017-03-11T10:15:00.000000000 NY"))); @@ -122,25 +122,25 @@ public void testMultiPeriod() { multi.businessNanosElapsed(DateTimeUtils.parseInstant("2017-03-11T10:30:00.000000000 NY"))); assertEquals(Duration.ofNanos(DateTimeUtils.MINUTE * 30), multi.businessDurationElapsed(DateTimeUtils.parseInstant("2017-03-11T10:30:00.000000000 NY"))); - assertEquals(DateTimeUtils.HOUR * 2, + assertEquals(DateTimeUtils.HOUR * 2 + 1, multi.businessNanosElapsed(DateTimeUtils.parseInstant("2017-03-11T13:00:00.000000000 NY"))); - assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 2), + assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 2 + 1), multi.businessDurationElapsed(DateTimeUtils.parseInstant("2017-03-11T13:00:00.000000000 NY"))); - assertEquals(DateTimeUtils.HOUR * 2, + assertEquals(DateTimeUtils.HOUR * 2 + 1, multi.businessNanosElapsed(DateTimeUtils.parseInstant("2017-03-11T13:00:00.000000000 NY"))); - assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 2), + assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 2 + 1), multi.businessDurationElapsed(DateTimeUtils.parseInstant("2017-03-11T13:00:00.000000000 NY"))); - assertEquals(DateTimeUtils.HOUR * 6, + assertEquals(DateTimeUtils.HOUR * 6 + 2, multi.businessNanosRemaining(DateTimeUtils.parseInstant("2017-03-11T01:00:00.000000000 NY"))); - assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 6), + assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 6 + 2), multi.businessDurationRemaining(DateTimeUtils.parseInstant("2017-03-11T01:00:00.000000000 NY"))); - assertEquals(DateTimeUtils.HOUR * 5 + DateTimeUtils.MINUTE * 30, + assertEquals(DateTimeUtils.HOUR * 5 + DateTimeUtils.MINUTE * 30 + 2, multi.businessNanosRemaining(DateTimeUtils.parseInstant("2017-03-11T10:30:00.000000000 NY"))); - assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 5 + DateTimeUtils.MINUTE * 30), + assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 5 + DateTimeUtils.MINUTE * 30 + 2), multi.businessDurationRemaining(DateTimeUtils.parseInstant("2017-03-11T10:30:00.000000000 NY"))); - assertEquals(DateTimeUtils.HOUR * 4, + assertEquals(DateTimeUtils.HOUR * 4 + 1, multi.businessNanosRemaining(DateTimeUtils.parseInstant("2017-03-11T13:00:00.000000000 NY"))); - assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 4), + assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 4 + 1), multi.businessDurationRemaining(DateTimeUtils.parseInstant("2017-03-11T13:00:00.000000000 NY"))); assertEquals(NULL_LONG, multi.businessNanosElapsed(null)); @@ -156,10 +156,10 @@ public void testMultiPeriod() { assertEquals(close2, multi2.businessEnd()); assertEquals(close2, multi2.businessEnd()); assertTrue(multi2.isInclusiveEnd()); - assertEquals(DateTimeUtils.HOUR * 6, multi2.businessNanos()); - assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 6), multi2.businessDuration()); - assertEquals(DateTimeUtils.HOUR * 6, multi2.businessNanos()); - assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 6), multi2.businessDuration()); + assertEquals(DateTimeUtils.HOUR * 6 + 2, multi2.businessNanos()); + assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 6 + 2), multi2.businessDuration()); + assertEquals(DateTimeUtils.HOUR * 6 + 2, multi2.businessNanos()); + assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 6 + 2), multi2.businessDuration()); assertTrue(multi2.isBusinessDay()); assertTrue(multi2.isBusinessTime(DateTimeUtils.parseInstant("2017-03-11T10:00:00.000000000 NY"))); assertTrue(multi2.isBusinessTime(DateTimeUtils.parseInstant("2017-03-11T10:15:00.000000000 NY"))); @@ -173,13 +173,13 @@ public void testMultiPeriod() { multi2.businessNanosElapsed(DateTimeUtils.parseInstant("2017-03-11T10:30:00.000000000 NY"))); assertEquals(Duration.ofNanos(DateTimeUtils.MINUTE * 30), multi2.businessDurationElapsed(DateTimeUtils.parseInstant("2017-03-11T10:30:00.000000000 NY"))); - assertEquals(DateTimeUtils.HOUR * 2, + assertEquals(DateTimeUtils.HOUR * 2 + 1, multi2.businessNanosElapsed(DateTimeUtils.parseInstant("2017-03-11T13:00:00.000000000 NY"))); - assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 2), + assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 2 + 1), multi2.businessDurationElapsed(DateTimeUtils.parseInstant("2017-03-11T13:00:00.000000000 NY"))); - assertEquals(DateTimeUtils.HOUR * 2, + assertEquals(DateTimeUtils.HOUR * 2 + 1, multi2.businessNanosElapsed(DateTimeUtils.parseInstant("2017-03-11T13:00:00.000000000 NY"))); - assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 2), + assertEquals(Duration.ofNanos(DateTimeUtils.HOUR * 2 + 1), multi2.businessDurationElapsed(DateTimeUtils.parseInstant("2017-03-11T13:00:00.000000000 NY"))); assertEquals(NULL_LONG, multi2.businessNanosElapsed(null)); diff --git a/engine/time/src/test/java/io/deephaven/time/calendar/TestCalendars.java b/engine/time/src/test/java/io/deephaven/time/calendar/TestCalendars.java index ef10b20e9d4..6c86dc632de 100644 --- a/engine/time/src/test/java/io/deephaven/time/calendar/TestCalendars.java +++ b/engine/time/src/test/java/io/deephaven/time/calendar/TestCalendars.java @@ -5,6 +5,7 @@ import io.deephaven.base.testing.BaseArrayTestCase; import io.deephaven.configuration.Configuration; +import io.deephaven.time.DateTimeUtils; import java.net.URISyntaxException; import java.nio.file.Paths; @@ -82,6 +83,15 @@ public void testAdd() throws URISyntaxException { } catch (Exception e) { // pass } + } + + public void testUTCDayLength() { + final BusinessCalendar cal = Calendars.calendar("UTC"); + assertEquals(DateTimeUtils.DAY, cal.standardBusinessDay().businessNanos()); + } + public void testNYSEDayLength() { + final BusinessCalendar cal = Calendars.calendar("USNYSE_EXAMPLE"); + assertEquals(6 * DateTimeUtils.HOUR + 30 * DateTimeUtils.MINUTE, cal.standardBusinessDay().businessNanos()); } } diff --git a/engine/time/src/test/java/io/deephaven/time/calendar/TestTimeRange.java b/engine/time/src/test/java/io/deephaven/time/calendar/TestTimeRange.java index 5a0a11dbd87..476d7ee243f 100644 --- a/engine/time/src/test/java/io/deephaven/time/calendar/TestTimeRange.java +++ b/engine/time/src/test/java/io/deephaven/time/calendar/TestTimeRange.java @@ -50,8 +50,8 @@ public void testTimeRangeInclusive() { assertEquals(open1, period.start()); assertEquals(close1, period.end()); assertTrue(period.isInclusiveEnd()); - assertEquals(DateTimeUtils.HOUR, period.nanos()); - assertEquals(Duration.ofNanos(DateTimeUtils.HOUR), period.duration()); + assertEquals(DateTimeUtils.HOUR + 1, period.nanos()); + assertEquals(Duration.ofNanos(DateTimeUtils.HOUR + 1), period.duration()); assertTrue(period.contains(open1)); assertTrue(period @@ -101,8 +101,8 @@ public void testTimeRangeExclusive() { assertEquals(open1, period.start()); assertEquals(close1, period.end()); assertFalse(period.isInclusiveEnd()); - assertEquals(DateTimeUtils.HOUR - 1, period.nanos()); - assertEquals(Duration.ofNanos(DateTimeUtils.HOUR - 1), period.duration()); + assertEquals(DateTimeUtils.HOUR, period.nanos()); + assertEquals(Duration.ofNanos(DateTimeUtils.HOUR), period.duration()); assertTrue(period.contains(open1)); assertTrue(period @@ -192,4 +192,19 @@ public void testToStringExclusive() { final TimeRange p1 = new TimeRange<>(start, end, false); assertEquals("TimeRange{start=01:02:03, end=07:08:09, inclusiveEnd=false}", p1.toString()); } + + public void testNanos() { + final Instant t1 = DateTimeUtils.epochMillisToInstant(0); + final Instant t2 = DateTimeUtils.epochMillisToInstant(1); + + final TimeRange pInclusive = new TimeRange<>(t1, t2, true); + final TimeRange pExclusive = new TimeRange<>(t1, t2, false); + + final long nInclusive = pInclusive.nanos(); + final long nExclusive = pExclusive.nanos(); + + assertNotEquals(nInclusive, nExclusive); + assertEquals(DateTimeUtils.MILLI, nExclusive); + assertEquals(DateTimeUtils.MILLI + 1, nInclusive); + } } diff --git a/engine/time/src/test/resources/PARSER-TEST.calendar b/engine/time/src/test/resources/PARSER-TEST.calendar index 0fff2688674..3f4d43e1344 100644 --- a/engine/time/src/test/resources/PARSER-TEST.calendar +++ b/engine/time/src/test/resources/PARSER-TEST.calendar @@ -18,6 +18,6 @@ 2015-04-06 - 14:1516:46 + 14:1516:46true diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarListChunkInputStreamGenerator.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarListChunkInputStreamGenerator.java index 8142c8d7d22..d85c2d6c3d4 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarListChunkInputStreamGenerator.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarListChunkInputStreamGenerator.java @@ -46,11 +46,17 @@ private synchronized void computePayload() { final Class myType = type.getComponentType(); final Class myComponentType = myType != null ? myType.getComponentType() : null; - ChunkType chunkType = ChunkType.fromElementType(myType); - if (chunkType == ChunkType.Boolean) { - // the internal payload is in bytes (to handle nulls), but the wire format is packed bits + + final ChunkType chunkType; + if (myType == boolean.class || myType == Boolean.class) { + // Note: Internally booleans are passed around as bytes, but the wire format is packed bits. chunkType = ChunkType.Byte; + } else if (myType != null && !myType.isPrimitive()) { + chunkType = ChunkType.Object; + } else { + chunkType = ChunkType.fromElementType(myType); } + final ArrayExpansionKernel kernel = ArrayExpansionKernel.makeExpansionKernel(chunkType, myType); offsets = WritableIntChunk.makeWritableChunk(chunk.size() + 1); diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarListChunkReader.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarListChunkReader.java index dddc6414b1e..4e5b8cb0bd7 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarListChunkReader.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarListChunkReader.java @@ -36,6 +36,8 @@ public VarListChunkReader(final StreamReaderOptions options, final TypeInfo type if (componentType == boolean.class || componentType == Boolean.class) { // Note: Internally booleans are passed around as bytes, but the wire format is packed bits. chunkType = ChunkType.Byte; + } else if (componentType != null && !componentType.isPrimitive()) { + chunkType = ChunkType.Object; } else { chunkType = ChunkType.fromElementType(componentType); } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/array/BoxedBooleanArrayExpansionKernel.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/array/BoxedBooleanArrayExpansionKernel.java index c6b901a7dd3..0a02ddb31f9 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/array/BoxedBooleanArrayExpansionKernel.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/array/BoxedBooleanArrayExpansionKernel.java @@ -47,7 +47,7 @@ public WritableChunk expand(final ObjectChunk source continue; } for (int j = 0; j < row.length; ++j) { - final byte value = row[j] ? BooleanUtils.TRUE_BOOLEAN_AS_BYTE : BooleanUtils.FALSE_BOOLEAN_AS_BYTE; + final byte value = BooleanUtils.booleanAsByte(row[j]); result.set(lenWritten + j, value); } lenWritten += row.length; diff --git a/extensions/barrage/src/main/resources/io/deephaven/extensions/barrage/Barrage.gwt.xml b/extensions/barrage/src/main/resources/io/deephaven/extensions/barrage/Barrage.gwt.xml new file mode 100644 index 00000000000..a29af5b6ca8 --- /dev/null +++ b/extensions/barrage/src/main/resources/io/deephaven/extensions/barrage/Barrage.gwt.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/extensions/iceberg/s3/build.gradle b/extensions/iceberg/s3/build.gradle index bde1e84bc7f..dfb53c52388 100644 --- a/extensions/iceberg/s3/build.gradle +++ b/extensions/iceberg/s3/build.gradle @@ -26,6 +26,12 @@ dependencies { runtimeOnly libs.awssdk.sts runtimeOnly libs.awssdk.glue + compileOnly libs.autoservice + annotationProcessor libs.autoservice.compiler + + testImplementation libs.junit4 + testImplementation project(':engine-test-utils') + testImplementation libs.testcontainers testImplementation libs.testcontainers.junit.jupiter testImplementation libs.testcontainers.localstack @@ -39,20 +45,10 @@ dependencies { testRuntimeOnly libs.slf4j.simple } -test { - useJUnitPlatform { - excludeTags("testcontainers") - } -} +TestTools.addEngineOutOfBandTest(project) -tasks.register('testOutOfBand', Test) { - useJUnitPlatform { - includeTags("testcontainers") - } +testOutOfBand.dependsOn Docker.registryTask(project, 'localstack') +testOutOfBand.systemProperty 'testcontainers.localstack.image', Docker.localImageName('localstack') - dependsOn Docker.registryTask(project, 'localstack') - systemProperty 'testcontainers.localstack.image', Docker.localImageName('localstack') - - dependsOn Docker.registryTask(project, 'minio') - systemProperty 'testcontainers.minio.image', Docker.localImageName('minio') -} +testOutOfBand.dependsOn Docker.registryTask(project, 'minio') +testOutOfBand.systemProperty 'testcontainers.minio.image', Docker.localImageName('minio') \ No newline at end of file diff --git a/extensions/iceberg/s3/src/main/java/io/deephaven/iceberg/util/IcebergToolsS3.java b/extensions/iceberg/s3/src/main/java/io/deephaven/iceberg/util/IcebergToolsS3.java index 166b47e5d28..c545ab68540 100644 --- a/extensions/iceberg/s3/src/main/java/io/deephaven/iceberg/util/IcebergToolsS3.java +++ b/extensions/iceberg/s3/src/main/java/io/deephaven/iceberg/util/IcebergToolsS3.java @@ -5,11 +5,9 @@ import com.google.common.base.Strings; import org.apache.iceberg.CatalogProperties; -import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.aws.AwsClientProperties; import org.apache.iceberg.aws.glue.GlueCatalog; import org.apache.iceberg.aws.s3.S3FileIOProperties; -import org.apache.iceberg.io.FileIO; import org.apache.iceberg.rest.RESTCatalog; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -68,12 +66,10 @@ public static IcebergCatalogAdapter createS3Rest( properties.put(S3FileIOProperties.ENDPOINT, endpointOverride); } - final FileIO fileIO = CatalogUtil.loadFileIO(S3_FILE_IO_CLASS, properties, null); - final String catalogName = name != null ? name : "IcebergCatalog-" + catalogURI; catalog.initialize(catalogName, properties); - return new IcebergCatalogAdapter(catalog, fileIO); + return new IcebergCatalogAdapter(catalog, properties); } /** @@ -101,11 +97,9 @@ public static IcebergCatalogAdapter createGlue( properties.put(CatalogProperties.URI, catalogURI); properties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouseLocation); - final FileIO fileIO = CatalogUtil.loadFileIO(S3_FILE_IO_CLASS, properties, null); - final String catalogName = name != null ? name : "IcebergCatalog-" + catalogURI; catalog.initialize(catalogName, properties); - return new IcebergCatalogAdapter(catalog, fileIO); + return new IcebergCatalogAdapter(catalog, properties); } } diff --git a/extensions/iceberg/s3/src/main/java/io/deephaven/iceberg/util/S3InstructionsProviderPlugin.java b/extensions/iceberg/s3/src/main/java/io/deephaven/iceberg/util/S3InstructionsProviderPlugin.java new file mode 100644 index 00000000000..6302ede7ff9 --- /dev/null +++ b/extensions/iceberg/s3/src/main/java/io/deephaven/iceberg/util/S3InstructionsProviderPlugin.java @@ -0,0 +1,54 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.iceberg.util; + +import com.google.auto.service.AutoService; +import io.deephaven.extensions.s3.Credentials; +import io.deephaven.extensions.s3.S3Instructions; +import io.deephaven.iceberg.internal.DataInstructionsProviderPlugin; +import org.apache.iceberg.aws.AwsClientProperties; +import org.apache.iceberg.aws.s3.S3FileIOProperties; +import org.jetbrains.annotations.NotNull; + +import java.net.URI; +import java.util.Map; + +/** + * {@link io.deephaven.iceberg.internal.DataInstructionsProviderPlugin} implementation used for reading files from S3. + */ +@AutoService(io.deephaven.iceberg.internal.DataInstructionsProviderPlugin.class) +@SuppressWarnings("unused") +public final class S3InstructionsProviderPlugin implements DataInstructionsProviderPlugin { + @Override + public Object createInstructions(@NotNull final URI uri, @NotNull final Map properties) { + // If the URI scheme is "s3","s3a","s3n" or if the properties contain one of these specific keys, we can + // create a useful S3Instructions object. + if (uri.getScheme().equals("s3") + || uri.getScheme().equals("s3a") + || uri.getScheme().equals("s3n") + || properties.containsKey(AwsClientProperties.CLIENT_REGION) + || properties.containsKey(S3FileIOProperties.ACCESS_KEY_ID) + || properties.containsKey(S3FileIOProperties.SECRET_ACCESS_KEY) + || properties.containsKey(S3FileIOProperties.ENDPOINT)) { + + final S3Instructions.Builder builder = S3Instructions.builder(); + if (properties.containsKey(AwsClientProperties.CLIENT_REGION)) { + builder.regionName(properties.get(AwsClientProperties.CLIENT_REGION)); + } + if (properties.containsKey(S3FileIOProperties.ENDPOINT)) { + builder.endpointOverride(properties.get(S3FileIOProperties.ENDPOINT)); + } + if (properties.containsKey(S3FileIOProperties.ACCESS_KEY_ID) + && properties.containsKey(S3FileIOProperties.SECRET_ACCESS_KEY)) { + builder.credentials( + Credentials.basic(properties.get(S3FileIOProperties.ACCESS_KEY_ID), + properties.get(S3FileIOProperties.SECRET_ACCESS_KEY))); + } + return builder.build(); + } + + // We have no useful properties for creating an S3Instructions object. + return null; + } +} diff --git a/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergLocalStackTest.java b/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergLocalStackTest.java index 578e358985e..6683bd42db1 100644 --- a/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergLocalStackTest.java +++ b/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergLocalStackTest.java @@ -3,24 +3,23 @@ // package io.deephaven.iceberg.util; - import io.deephaven.extensions.s3.S3Instructions.Builder; import io.deephaven.extensions.s3.testlib.SingletonContainers; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; +import org.junit.BeforeClass; import software.amazon.awssdk.services.s3.S3AsyncClient; -@Tag("testcontainers") +import java.util.Map; + public class IcebergLocalStackTest extends IcebergToolsTest { - @BeforeAll - static void initContainer() { + @BeforeClass + public static void initContainer() { // ensure container is started so container startup time isn't associated with a specific test SingletonContainers.LocalStack.init(); } @Override - public Builder s3Instructions(Builder builder) { + public Builder s3Instructions(final Builder builder) { return SingletonContainers.LocalStack.s3Instructions(builder); } @@ -28,4 +27,9 @@ public Builder s3Instructions(Builder builder) { public S3AsyncClient s3AsyncClient() { return SingletonContainers.LocalStack.s3AsyncClient(); } + + @Override + public Map s3Properties() { + return SingletonContainers.LocalStack.s3Properties(); + } } diff --git a/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergMinIOTest.java b/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergMinIOTest.java index 804d2d01746..946f3eca90d 100644 --- a/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergMinIOTest.java +++ b/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergMinIOTest.java @@ -3,20 +3,19 @@ // package io.deephaven.iceberg.util; - import io.deephaven.extensions.s3.S3Instructions.Builder; import io.deephaven.extensions.s3.testlib.SingletonContainers; import io.deephaven.stats.util.OSUtil; import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; +import org.junit.BeforeClass; import software.amazon.awssdk.services.s3.S3AsyncClient; -@Tag("testcontainers") +import java.util.Map; + public class IcebergMinIOTest extends IcebergToolsTest { - @BeforeAll - static void initContainer() { + @BeforeClass + public static void initContainer() { // TODO(deephaven-core#5116): MinIO testcontainers does not work on OS X Assumptions.assumeFalse(OSUtil.runningMacOS(), "OSUtil.runningMacOS()"); // ensure container is started so container startup time isn't associated with a specific test @@ -24,7 +23,7 @@ static void initContainer() { } @Override - public Builder s3Instructions(Builder builder) { + public Builder s3Instructions(final Builder builder) { return SingletonContainers.MinIO.s3Instructions(builder); } @@ -32,4 +31,10 @@ public Builder s3Instructions(Builder builder) { public S3AsyncClient s3AsyncClient() { return SingletonContainers.MinIO.s3AsyncClient(); } + + @Override + public Map s3Properties() { + return SingletonContainers.MinIO.s3Properties(); + } + } diff --git a/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergToolsTest.java b/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergToolsTest.java index 44a17942cdf..eb1640f07c2 100644 --- a/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergToolsTest.java +++ b/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergToolsTest.java @@ -9,17 +9,19 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.locations.TableDataException; +import io.deephaven.engine.testutil.junit4.EngineCleanup; import io.deephaven.extensions.s3.S3Instructions; import io.deephaven.iceberg.TestCatalog.IcebergTestCatalog; -import io.deephaven.iceberg.TestCatalog.IcebergTestFileIO; +import io.deephaven.test.types.OutOfBandTest; import org.apache.iceberg.Snapshot; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.io.FileIO; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.experimental.categories.Category; +import org.junit.Test; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.CreateBucketRequest; @@ -33,10 +35,7 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -47,6 +46,7 @@ import static io.deephaven.iceberg.util.IcebergCatalogAdapter.SNAPSHOT_DEFINITION; import static io.deephaven.iceberg.util.IcebergCatalogAdapter.TABLES_DEFINITION; +@Category(OutOfBandTest.class) public abstract class IcebergToolsTest { private static final TableDefinition SALES_SINGLE_DEFINITION = TableDefinition.of( @@ -101,6 +101,8 @@ public abstract class IcebergToolsTest { public abstract S3Instructions.Builder s3Instructions(S3Instructions.Builder builder); + public abstract Map s3Properties(); + private S3AsyncClient asyncClient; private String bucket; @@ -108,19 +110,20 @@ public abstract class IcebergToolsTest { private String warehousePath; private Catalog resourceCatalog; - private FileIO resourceFileIO; - @BeforeEach - void setUp() throws ExecutionException, InterruptedException { + @Rule + public final EngineCleanup framework = new EngineCleanup(); + + @Before + public void setUp() throws ExecutionException, InterruptedException { bucket = "warehouse"; asyncClient = s3AsyncClient(); asyncClient.createBucket(CreateBucketRequest.builder().bucket(bucket).build()).get(); warehousePath = IcebergToolsTest.class.getResource("/warehouse").getPath(); - resourceFileIO = new IcebergTestFileIO("s3://warehouse", warehousePath); // Create the test catalog for the tests - resourceCatalog = IcebergTestCatalog.create(warehousePath, resourceFileIO); + resourceCatalog = IcebergTestCatalog.create(warehousePath, s3Properties()); final S3Instructions s3Instructions = s3Instructions(S3Instructions.builder()).build(); @@ -129,12 +132,22 @@ void setUp() throws ExecutionException, InterruptedException { .build(); } - private void uploadParquetFiles(final File root, final String prefixToRemove) + @After + public void tearDown() throws ExecutionException, InterruptedException { + for (String key : keys) { + asyncClient.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(key).build()).get(); + } + keys.clear(); + asyncClient.deleteBucket(DeleteBucketRequest.builder().bucket(bucket).build()).get(); + asyncClient.close(); + } + + private void uploadFiles(final File root, final String prefixToRemove) throws ExecutionException, InterruptedException, TimeoutException { for (final File file : root.listFiles()) { if (file.isDirectory()) { - uploadParquetFiles(file, prefixToRemove); - } else if (file.getName().endsWith(".parquet")) { + uploadFiles(file, prefixToRemove); + } else { final String key = file.getPath().substring(prefixToRemove.length() + 1); keys.add(key); @@ -151,43 +164,33 @@ private void uploadParquetFiles(final File root, final String prefixToRemove) } private void uploadSalesPartitioned() throws ExecutionException, InterruptedException, TimeoutException { - uploadParquetFiles(new File(IcebergToolsTest.class.getResource("/warehouse/sales/sales_partitioned").getPath()), + uploadFiles(new File(IcebergToolsTest.class.getResource("/warehouse/sales/sales_partitioned").getPath()), warehousePath); } private void uploadAllTypes() throws ExecutionException, InterruptedException, TimeoutException { - uploadParquetFiles(new File(IcebergToolsTest.class.getResource("/warehouse/sample/all_types").getPath()), + uploadFiles(new File(IcebergToolsTest.class.getResource("/warehouse/sample/all_types").getPath()), warehousePath); } private void uploadSalesSingle() throws ExecutionException, InterruptedException, TimeoutException { - uploadParquetFiles(new File(IcebergToolsTest.class.getResource("/warehouse/sales/sales_single").getPath()), + uploadFiles(new File(IcebergToolsTest.class.getResource("/warehouse/sales/sales_single").getPath()), warehousePath); } private void uploadSalesMulti() throws ExecutionException, InterruptedException, TimeoutException { - uploadParquetFiles(new File(IcebergToolsTest.class.getResource("/warehouse/sales/sales_multi").getPath()), + uploadFiles(new File(IcebergToolsTest.class.getResource("/warehouse/sales/sales_multi").getPath()), warehousePath); } private void uploadSalesRenamed() throws ExecutionException, InterruptedException, TimeoutException { - uploadParquetFiles(new File(IcebergToolsTest.class.getResource("/warehouse/sales/sales_renamed").getPath()), + uploadFiles(new File(IcebergToolsTest.class.getResource("/warehouse/sales/sales_renamed").getPath()), warehousePath); } - @AfterEach - public void tearDown() throws ExecutionException, InterruptedException { - for (String key : keys) { - asyncClient.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(key).build()).get(); - } - keys.clear(); - asyncClient.deleteBucket(DeleteBucketRequest.builder().bucket(bucket).build()).get(); - asyncClient.close(); - } - @Test public void testListNamespaces() { - final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Collection namespaces = adapter.listNamespaces(); final Collection namespaceNames = @@ -204,7 +207,7 @@ public void testListNamespaces() { @Test public void testListTables() { - final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); @@ -228,7 +231,7 @@ public void testListTables() { @Test public void testListSnapshots() { - final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final TLongArrayList snapshotIds = new TLongArrayList(); final TableIdentifier tableIdentifier = TableIdentifier.of("sales", "sales_multi"); @@ -256,8 +259,7 @@ public void testListSnapshots() { public void testOpenTableA() throws ExecutionException, InterruptedException, TimeoutException { uploadSalesPartitioned(); - final IcebergCatalogAdapter adapter = - IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_partitioned"); @@ -279,8 +281,7 @@ public void testOpenTableA() throws ExecutionException, InterruptedException, Ti public void testOpenTableB() throws ExecutionException, InterruptedException, TimeoutException { uploadSalesMulti(); - final IcebergCatalogAdapter adapter = - IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_multi"); @@ -301,8 +302,7 @@ public void testOpenTableB() throws ExecutionException, InterruptedException, Ti public void testOpenTableC() throws ExecutionException, InterruptedException, TimeoutException { uploadSalesSingle(); - final IcebergCatalogAdapter adapter = - IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_single"); @@ -324,7 +324,7 @@ public void testOpenTableC() throws ExecutionException, InterruptedException, Ti public void testOpenTableS3Only() throws ExecutionException, InterruptedException, TimeoutException { uploadSalesPartitioned(); - final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_partitioned"); @@ -344,8 +344,7 @@ public void testOpenTableDefinition() throws ExecutionException, InterruptedExce .dataInstructions(instructions.dataInstructions().get()) .build(); - final IcebergCatalogAdapter adapter = - IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_partitioned"); @@ -372,8 +371,7 @@ public void testOpenTablePartitionTypeException() { .dataInstructions(instructions.dataInstructions().get()) .build(); - final IcebergCatalogAdapter adapter = - IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_partitioned"); @@ -416,8 +414,7 @@ public void testOpenTableDefinitionRename() throws ExecutionException, Interrupt .putColumnRenames("month", "__month") .build(); - final IcebergCatalogAdapter adapter = - IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_partitioned"); @@ -446,8 +443,7 @@ public void testSkippedPartitioningColumn() throws ExecutionException, Interrupt .dataInstructions(instructions.dataInstructions().get()) .build(); - final IcebergCatalogAdapter adapter = - IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_partitioned"); @@ -476,8 +472,7 @@ public void testReorderedPartitioningColumn() throws ExecutionException, Interru .dataInstructions(instructions.dataInstructions().get()) .build(); - final IcebergCatalogAdapter adapter = - IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_partitioned"); @@ -497,8 +492,7 @@ public void testZeroPartitioningColumns() throws ExecutionException, Interrupted .dataInstructions(instructions.dataInstructions().get()) .build(); - final IcebergCatalogAdapter adapter = - IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_partitioned"); @@ -525,8 +519,7 @@ public void testIncorrectPartitioningColumns() throws ExecutionException, Interr .dataInstructions(instructions.dataInstructions().get()) .build(); - final IcebergCatalogAdapter adapter = - IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_partitioned"); @@ -561,8 +554,7 @@ public void testMissingPartitioningColumns() { .dataInstructions(instructions.dataInstructions().get()) .build(); - final IcebergCatalogAdapter adapter = - IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_partitioned"); @@ -590,8 +582,7 @@ public void testOpenTableColumnRename() throws ExecutionException, InterruptedEx .putColumnRenames("Item_Type", "ItemType") .build(); - final IcebergCatalogAdapter adapter = - IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_partitioned"); @@ -609,8 +600,7 @@ public void testOpenTableColumnLegalization() throws ExecutionException, Interru .dataInstructions(instructions.dataInstructions().get()) .build(); - final IcebergCatalogAdapter adapter = - IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_renamed"); @@ -632,8 +622,7 @@ public void testOpenTableColumnLegalizationRename() .putColumnRenames("Units/Sold", "Units_Sold") .build(); - final IcebergCatalogAdapter adapter = - IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_renamed"); @@ -664,8 +653,7 @@ public void testOpenTableColumnLegalizationPartitionException() { .dataInstructions(instructions.dataInstructions().get()) .build(); - final IcebergCatalogAdapter adapter = - IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_partitioned"); @@ -695,8 +683,7 @@ public void testOpenTableColumnRenamePartitioningColumns() .putColumnRenames("year", "__year") .build(); - final IcebergCatalogAdapter adapter = - IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_partitioned"); @@ -720,7 +707,7 @@ public void testOpenTableColumnRenamePartitioningColumns() public void testOpenTableSnapshot() throws ExecutionException, InterruptedException, TimeoutException { uploadSalesMulti(); - final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_multi"); @@ -752,7 +739,7 @@ public void testOpenTableSnapshot() throws ExecutionException, InterruptedExcept public void testOpenTableSnapshotByID() throws ExecutionException, InterruptedException, TimeoutException { uploadSalesMulti(); - final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_multi"); @@ -799,20 +786,20 @@ public void testOpenTableSnapshotByID() throws ExecutionException, InterruptedEx public void testOpenAllTypesTable() throws ExecutionException, InterruptedException, TimeoutException { uploadAllTypes(); - final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sample"); final TableIdentifier tableId = TableIdentifier.of(ns, "all_types"); // Verify we retrieved all the rows. - final io.deephaven.engine.table.Table table = adapter.readTable(tableId, instructions); + final io.deephaven.engine.table.Table table = adapter.readTable(tableId, instructions).select(); Assert.eq(table.size(), "table.size()", 10, "10 rows in the table"); Assert.equals(table.getDefinition(), "table.getDefinition()", ALL_TYPES_DEF); } @Test public void testTableDefinition() { - final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_multi"); @@ -837,7 +824,7 @@ public void testTableDefinition() { @Test public void testTableDefinitionTable() { - final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); final Namespace ns = Namespace.of("sales"); final TableIdentifier tableId = TableIdentifier.of(ns, "sales_multi"); @@ -870,7 +857,7 @@ public void testTableDefinitionTable() { @Test public void testTableDefinitionWithInstructions() { - final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog, resourceFileIO); + final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog); IcebergInstructions localInstructions = IcebergInstructions.builder() .dataInstructions(instructions.dataInstructions().get()) diff --git a/extensions/iceberg/src/main/java/io/deephaven/iceberg/internal/DataInstructionsProviderLoader.java b/extensions/iceberg/src/main/java/io/deephaven/iceberg/internal/DataInstructionsProviderLoader.java new file mode 100644 index 00000000000..4ae28e0e044 --- /dev/null +++ b/extensions/iceberg/src/main/java/io/deephaven/iceberg/internal/DataInstructionsProviderLoader.java @@ -0,0 +1,88 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.iceberg.internal; + +import org.jetbrains.annotations.NotNull; + +import java.net.URI; +import java.util.*; + +/** + * A service loader class for loading {@link DataInstructionsProviderPlugin} implementations at runtime which provide + * {@link DataInstructionsProviderLoader} implementations for different URI paths. + */ +public final class DataInstructionsProviderLoader { + /** + * The list of plugins loaded by the {@link ServiceLoader}. + */ + private static volatile List cachedProviders; + + /** + * Ensure that the {@link DataInstructionsProviderPlugin plugins} are loaded exactly once. + */ + private static void ensureProviders() { + if (cachedProviders == null) { + synchronized (DataInstructionsProviderLoader.class) { + if (cachedProviders == null) { + cachedProviders = new ArrayList<>(); + // Load the plugins + for (final DataInstructionsProviderPlugin plugin : ServiceLoader + .load(DataInstructionsProviderPlugin.class)) { + cachedProviders.add(plugin); + } + } + } + } + } + + /** + * Get a {@link DataInstructionsProviderLoader} instance for the given property collection. + * + * @param properties The property collection. + * @return A {@link DataInstructionsProviderLoader} instance. + */ + public static DataInstructionsProviderLoader create(final Map properties) { + ensureProviders(); + return new DataInstructionsProviderLoader(properties); + } + + /** + * The properties collection for this instance. + */ + private final Map properties; + + /** + * The local list of plugins loaded by the {@link ServiceLoader}. + */ + private final List providers; + + /** + * Create a new {@link DataInstructionsProviderLoader} instance for the given property collection. + * + * @param properties The property collection. + */ + private DataInstructionsProviderLoader(final Map properties) { + this.properties = properties; + providers = cachedProviders; + } + + /** + * Create a new data instructions object compatible with reading from and writing to the given URI, using the + * plugins loaded by the {@link ServiceLoader}. For example, for a "S3" URI, we will create an + * {@code S3Instructions} object which can read files from S3. + * + * @param uri The URI + * @return A data instructions object for the given URI or null if one cannot be found + */ + public Object fromServiceLoader(@NotNull final URI uri) { + for (final DataInstructionsProviderPlugin plugin : providers) { + final Object pluginInstructions = plugin.createInstructions(uri, properties); + if (pluginInstructions != null) { + return pluginInstructions; + } + } + // No plugin found for this URI and property collection. + return null; + } +} diff --git a/extensions/iceberg/src/main/java/io/deephaven/iceberg/internal/DataInstructionsProviderPlugin.java b/extensions/iceberg/src/main/java/io/deephaven/iceberg/internal/DataInstructionsProviderPlugin.java new file mode 100644 index 00000000000..b0e48cc9166 --- /dev/null +++ b/extensions/iceberg/src/main/java/io/deephaven/iceberg/internal/DataInstructionsProviderPlugin.java @@ -0,0 +1,20 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.iceberg.internal; + +import org.jetbrains.annotations.NotNull; + +import java.net.URI; +import java.util.Map; + +/** + * A plugin interface for providing {@link DataInstructionsProviderPlugin} implementations for different property + * collections and URI values. Check out {@link DataInstructionsProviderLoader} for more details. + */ +public interface DataInstructionsProviderPlugin { + /** + * Create a data instructions object for the given URI. + */ + Object createInstructions(@NotNull URI uri, @NotNull final Map properties); +} diff --git a/extensions/iceberg/src/main/java/io/deephaven/iceberg/layout/IcebergBaseLayout.java b/extensions/iceberg/src/main/java/io/deephaven/iceberg/layout/IcebergBaseLayout.java index f5334cf866c..7bf0f5222a2 100644 --- a/extensions/iceberg/src/main/java/io/deephaven/iceberg/layout/IcebergBaseLayout.java +++ b/extensions/iceberg/src/main/java/io/deephaven/iceberg/layout/IcebergBaseLayout.java @@ -11,6 +11,7 @@ import io.deephaven.iceberg.location.IcebergTableParquetLocationKey; import io.deephaven.iceberg.util.IcebergInstructions; import io.deephaven.parquet.table.ParquetInstructions; +import io.deephaven.iceberg.internal.DataInstructionsProviderLoader; import org.apache.iceberg.*; import org.apache.iceberg.io.FileIO; import org.jetbrains.annotations.NotNull; @@ -53,6 +54,11 @@ public abstract class IcebergBaseLayout implements TableLocationKeyFinder cache; + /** + * The data instructions provider for creating instructions from URI and user-supplied properties. + */ + final DataInstructionsProviderLoader dataInstructionsProvider; + /** * The {@link ParquetInstructions} object that will be used to read any Parquet data files in this table. Only * accessed while synchronized on {@code this}. @@ -79,8 +85,16 @@ protected IcebergTableLocationKey locationKey( } } - // Add the data instructions. - instructions.dataInstructions().ifPresent(builder::setSpecialInstructions); + // Add the data instructions if provided as part of the IcebergInstructions. + if (instructions.dataInstructions().isPresent()) { + builder.setSpecialInstructions(instructions.dataInstructions().get()); + } else { + // Attempt to create data instructions from the properties collection and URI. + final Object dataInstructions = dataInstructionsProvider.fromServiceLoader(fileUri); + if (dataInstructions != null) { + builder.setSpecialInstructions(dataInstructions); + } + } parquetInstructions = builder.build(); } @@ -102,12 +116,14 @@ public IcebergBaseLayout( @NotNull final Table table, @NotNull final Snapshot tableSnapshot, @NotNull final FileIO fileIO, - @NotNull final IcebergInstructions instructions) { + @NotNull final IcebergInstructions instructions, + @NotNull final DataInstructionsProviderLoader dataInstructionsProvider) { this.tableDef = tableDef; this.table = table; this.snapshot = tableSnapshot; this.fileIO = fileIO; this.instructions = instructions; + this.dataInstructionsProvider = dataInstructionsProvider; this.cache = new HashMap<>(); } diff --git a/extensions/iceberg/src/main/java/io/deephaven/iceberg/layout/IcebergFlatLayout.java b/extensions/iceberg/src/main/java/io/deephaven/iceberg/layout/IcebergFlatLayout.java index ac4c19283f9..fd407d7702e 100644 --- a/extensions/iceberg/src/main/java/io/deephaven/iceberg/layout/IcebergFlatLayout.java +++ b/extensions/iceberg/src/main/java/io/deephaven/iceberg/layout/IcebergFlatLayout.java @@ -7,6 +7,7 @@ import io.deephaven.engine.table.impl.locations.impl.TableLocationKeyFinder; import io.deephaven.iceberg.location.IcebergTableLocationKey; import io.deephaven.iceberg.util.IcebergInstructions; +import io.deephaven.iceberg.internal.DataInstructionsProviderLoader; import org.apache.iceberg.*; import org.apache.iceberg.io.FileIO; import org.jetbrains.annotations.NotNull; @@ -30,8 +31,9 @@ public IcebergFlatLayout( @NotNull final Table table, @NotNull final Snapshot tableSnapshot, @NotNull final FileIO fileIO, - @NotNull final IcebergInstructions instructions) { - super(tableDef, table, tableSnapshot, fileIO, instructions); + @NotNull final IcebergInstructions instructions, + @NotNull final DataInstructionsProviderLoader dataInstructionsProvider) { + super(tableDef, table, tableSnapshot, fileIO, instructions, dataInstructionsProvider); } @Override diff --git a/extensions/iceberg/src/main/java/io/deephaven/iceberg/layout/IcebergKeyValuePartitionedLayout.java b/extensions/iceberg/src/main/java/io/deephaven/iceberg/layout/IcebergKeyValuePartitionedLayout.java index 47ec05dfd74..f1a3cc9a5ea 100644 --- a/extensions/iceberg/src/main/java/io/deephaven/iceberg/layout/IcebergKeyValuePartitionedLayout.java +++ b/extensions/iceberg/src/main/java/io/deephaven/iceberg/layout/IcebergKeyValuePartitionedLayout.java @@ -9,6 +9,7 @@ import io.deephaven.engine.table.impl.locations.impl.TableLocationKeyFinder; import io.deephaven.iceberg.location.IcebergTableLocationKey; import io.deephaven.iceberg.util.IcebergInstructions; +import io.deephaven.iceberg.internal.DataInstructionsProviderLoader; import io.deephaven.util.type.TypeUtils; import org.apache.commons.lang3.mutable.MutableInt; import org.apache.iceberg.*; @@ -24,7 +25,7 @@ * a {@link Snapshot} */ public final class IcebergKeyValuePartitionedLayout extends IcebergBaseLayout { - private class ColumnData { + private static class ColumnData { final String name; final Class type; final int index; @@ -52,8 +53,9 @@ public IcebergKeyValuePartitionedLayout( @NotNull final org.apache.iceberg.Snapshot tableSnapshot, @NotNull final FileIO fileIO, @NotNull final PartitionSpec partitionSpec, - @NotNull final IcebergInstructions instructions) { - super(tableDef, table, tableSnapshot, fileIO, instructions); + @NotNull final IcebergInstructions instructions, + @NotNull final DataInstructionsProviderLoader dataInstructionsProvider) { + super(tableDef, table, tableSnapshot, fileIO, instructions, dataInstructionsProvider); // We can assume due to upstream validation that there are no duplicate names (after renaming) that are included // in the output definition, so we can ignore duplicates. diff --git a/extensions/iceberg/src/main/java/io/deephaven/iceberg/util/IcebergCatalogAdapter.java b/extensions/iceberg/src/main/java/io/deephaven/iceberg/util/IcebergCatalogAdapter.java index 486bcf18655..b76a750602d 100644 --- a/extensions/iceberg/src/main/java/io/deephaven/iceberg/util/IcebergCatalogAdapter.java +++ b/extensions/iceberg/src/main/java/io/deephaven/iceberg/util/IcebergCatalogAdapter.java @@ -18,6 +18,7 @@ import io.deephaven.engine.table.impl.sources.regioned.RegionedTableComponentFactoryImpl; import io.deephaven.engine.updategraph.UpdateSourceRegistrar; import io.deephaven.engine.util.TableTools; +import io.deephaven.iceberg.internal.DataInstructionsProviderLoader; import io.deephaven.iceberg.layout.IcebergFlatLayout; import io.deephaven.iceberg.layout.IcebergKeyValuePartitionedLayout; import io.deephaven.iceberg.location.IcebergTableLocationFactory; @@ -32,7 +33,6 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.io.FileIO; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.jetbrains.annotations.NotNull; @@ -65,16 +65,25 @@ public class IcebergCatalogAdapter { ColumnDefinition.fromGenericType("SnapshotObject", Snapshot.class)); private final Catalog catalog; - private final FileIO fileIO; + + private final DataInstructionsProviderLoader dataInstructionsProvider; + + /** + * Construct an IcebergCatalogAdapter from a catalog. + */ + IcebergCatalogAdapter(@NotNull final Catalog catalog) { + this(catalog, Map.of()); + } /** - * Construct an IcebergCatalogAdapter from a catalog and file IO. + * Construct an IcebergCatalogAdapter from a catalog and property collection. */ IcebergCatalogAdapter( @NotNull final Catalog catalog, - @NotNull final FileIO fileIO) { + @NotNull final Map properties) { this.catalog = catalog; - this.fileIO = fileIO; + + dataInstructionsProvider = DataInstructionsProviderLoader.create(Map.copyOf(properties)); } /** @@ -648,6 +657,54 @@ public Table readTable( return readTable(TableIdentifier.parse(tableIdentifier), instructions); } + /** + * Retrieve a snapshot of an Iceberg table from the Iceberg catalog. + * + * @param tableIdentifier The table identifier to load + * @param tableSnapshotId The snapshot id to load + * @return The loaded table + * @throws IllegalArgumentException if the snapshot with the given id is not found + */ + private Snapshot getTableSnapshot(@NotNull TableIdentifier tableIdentifier, long tableSnapshotId) { + return listSnapshots(tableIdentifier).stream() + .filter(snapshot -> snapshot.snapshotId() == tableSnapshotId) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException( + "Snapshot with id " + tableSnapshotId + " for table " + tableIdentifier + " not found")); + } + + /** + * Read a static snapshot of an Iceberg table from the Iceberg catalog. + * + * @param tableIdentifier The table identifier to load + * @param tableSnapshotId The snapshot id to load + * @return The loaded table + */ + @SuppressWarnings("unused") + public Table readTable(@NotNull final TableIdentifier tableIdentifier, final long tableSnapshotId) { + // Find the snapshot with the given snapshot id + final Snapshot tableSnapshot = getTableSnapshot(tableIdentifier, tableSnapshotId); + + return readTableInternal(tableIdentifier, tableSnapshot, null); + } + + + /** + * Read a static snapshot of an Iceberg table from the Iceberg catalog. + * + * @param tableIdentifier The table identifier to load + * @param tableSnapshotId The snapshot id to load + * @return The loaded table + */ + @SuppressWarnings("unused") + public Table readTable(@NotNull final String tableIdentifier, final long tableSnapshotId) { + final TableIdentifier tableId = TableIdentifier.parse(tableIdentifier); + // Find the snapshot with the given snapshot id + final Snapshot tableSnapshot = getTableSnapshot(tableId, tableSnapshotId); + + return readTableInternal(tableId, tableSnapshot, null); + } + /** * Read a static snapshot of an Iceberg table from the Iceberg catalog. * @@ -738,11 +795,12 @@ private Table readTableInternal( if (partitionSpec.isUnpartitioned()) { // Create the flat layout location key finder - keyFinder = new IcebergFlatLayout(tableDef, table, snapshot, fileIO, userInstructions); + keyFinder = new IcebergFlatLayout(tableDef, table, snapshot, table.io(), userInstructions, + dataInstructionsProvider); } else { // Create the partitioning column location key finder - keyFinder = new IcebergKeyValuePartitionedLayout(tableDef, table, snapshot, fileIO, partitionSpec, - userInstructions); + keyFinder = new IcebergKeyValuePartitionedLayout(tableDef, table, snapshot, table.io(), partitionSpec, + userInstructions, dataInstructionsProvider); } refreshService = null; diff --git a/extensions/iceberg/src/main/java/io/deephaven/iceberg/util/IcebergTools.java b/extensions/iceberg/src/main/java/io/deephaven/iceberg/util/IcebergTools.java index bcdda326dca..5dd20f699a9 100644 --- a/extensions/iceberg/src/main/java/io/deephaven/iceberg/util/IcebergTools.java +++ b/extensions/iceberg/src/main/java/io/deephaven/iceberg/util/IcebergTools.java @@ -3,8 +3,14 @@ // package io.deephaven.iceberg.util; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.catalog.Catalog; -import org.apache.iceberg.io.FileIO; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Map; /** * Tools for accessing tables in the Iceberg table format. @@ -12,8 +18,114 @@ public abstract class IcebergTools { @SuppressWarnings("unused") public static IcebergCatalogAdapter createAdapter( - final Catalog catalog, - final FileIO fileIO) { - return new IcebergCatalogAdapter(catalog, fileIO); + final Catalog catalog) { + return new IcebergCatalogAdapter(catalog); + } + + /** + *

+ * Create an Iceberg catalog adapter for an Iceberg catalog created from configuration properties. These properties + * map to the Iceberg catalog Java API properties and are used to create the catalog and file IO implementations. + *

+ *

+ * The minimal set of properties required to create an Iceberg catalog are: + *

    + *
  • {@code "catalog-impl"} or {@code "type"} - the Java catalog implementation to use. When providing + * {@code "catalog-impl"}, the implementing Java class should be provided (e.g. + * {@code "org.apache.iceberg.rest.RESTCatalog"} or {@code "org.apache.iceberg.aws.glue.GlueCatalog")}. Choices for + * {@code "type"} include {@code "hive"}, {@code "hadoop"}, {@code "rest"}, {@code "glue"}, {@code "nessie"}, + * {@code "jdbc"}.
    • + *
  • {@code "uri"} - the URI of the catalog.
    • + *
+ *

+ * Other common properties include: + *

+ *
    + *
  • {@code "warehouse"} - the location of the data warehouse.
    • + *
  • {@code "client.region"} - the region of the AWS client.
    • + *
  • {@code "s3.access-key-id"} - the S3 access key for reading files.
    • + *
  • {@code "s3.secret-access-key"} - the S3 secret access key for reading files.
    • + *
  • {@code "s3.endpoint"} - the S3 endpoint to connect to.
    • + *
+ *

+ * Additional properties for the specific catalog should also be included, such as as S3-specific properties for + * authentication or endpoint overriding. + *

+ * + * @param name the name of the catalog; if omitted, the catalog URI will be used to generate a name + * @param properties a map containing the Iceberg catalog properties to use + * @return the Iceberg catalog adapter + */ + @SuppressWarnings("unused") + public static IcebergCatalogAdapter createAdapter( + @Nullable final String name, + @NotNull final Map properties) { + return createAdapter(name, properties, Map.of()); + } + + /** + *

+ * Create an Iceberg catalog adapter for an Iceberg catalog created from configuration properties. These properties + * map to the Iceberg catalog Java API properties and are used to create the catalog and file IO implementations. + *

+ *

+ * The minimal set of properties required to create an Iceberg catalog are: + *

    + *
  • {@code "catalog-impl"} or {@code "type"} - the Java catalog implementation to use. When providing + * {@code "catalog-impl"}, the implementing Java class should be provided (e.g. + * {@code "org.apache.iceberg.rest.RESTCatalog"} or {@code "org.apache.iceberg.aws.glue.GlueCatalog")}. Choices for + * {@code "type"} include {@code "hive"}, {@code "hadoop"}, {@code "rest"}, {@code "glue"}, {@code "nessie"}, + * {@code "jdbc"}.
    • + *
  • {@code "uri"} - the URI of the catalog.
    • + *
+ *

+ * Other common properties include: + *

+ *
    + *
  • {@code "warehouse"} - the location of the data warehouse.
    • + *
  • {@code "client.region"} - the region of the AWS client.
    • + *
  • {@code "s3.access-key-id"} - the S3 access key for reading files.
    • + *
  • {@code "s3.secret-access-key"} - the S3 secret access key for reading files.
    • + *
  • {@code "s3.endpoint"} - the S3 endpoint to connect to.
    • + *
+ *

+ * Additional properties for the specific catalog should also be included, such as as S3-specific properties for + * authentication or endpoint overriding. + *

+ * + * @param name the name of the catalog; if omitted, the catalog URI will be used to generate a name + * @param properties a map containing the Iceberg catalog properties to use + * @param hadoopConfig a map containing Hadoop configuration properties to use + * @return the Iceberg catalog adapter + */ + @SuppressWarnings("unused") + public static IcebergCatalogAdapter createAdapter( + @Nullable final String name, + @NotNull final Map properties, + @NotNull final Map hadoopConfig) { + // Validate the minimum required properties are set + if (!properties.containsKey(CatalogProperties.CATALOG_IMPL) + && !properties.containsKey(CatalogUtil.ICEBERG_CATALOG_TYPE)) { + throw new IllegalArgumentException( + String.format("Catalog type '%s' or implementation class '%s' is required", + CatalogUtil.ICEBERG_CATALOG_TYPE, CatalogProperties.CATALOG_IMPL)); + } + if (!properties.containsKey(CatalogProperties.URI)) { + throw new IllegalArgumentException(String.format("Catalog URI property '%s' is required", + CatalogProperties.URI)); + } + + final String catalogUri = properties.get(CatalogProperties.URI); + final String catalogName = name != null ? name : "IcebergCatalog-" + catalogUri; + + // Load the Hadoop configuration with the provided properties + final Configuration hadoopConf = new Configuration(); + hadoopConfig.forEach(hadoopConf::set); + + // Create the Iceberg catalog from the properties + final Catalog catalog = CatalogUtil.buildIcebergCatalog(catalogName, properties, hadoopConf); + + return new IcebergCatalogAdapter(catalog, properties); } + } diff --git a/extensions/iceberg/src/test/java/io/deephaven/iceberg/TestCatalog/IcebergTestCatalog.java b/extensions/iceberg/src/test/java/io/deephaven/iceberg/TestCatalog/IcebergTestCatalog.java index e62fbd282e0..3d95032e1f8 100644 --- a/extensions/iceberg/src/test/java/io/deephaven/iceberg/TestCatalog/IcebergTestCatalog.java +++ b/extensions/iceberg/src/test/java/io/deephaven/iceberg/TestCatalog/IcebergTestCatalog.java @@ -10,7 +10,7 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.NamespaceNotEmptyException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; -import org.apache.iceberg.io.FileIO; +import org.jetbrains.annotations.NotNull; import java.io.File; import java.util.*; @@ -19,7 +19,7 @@ public class IcebergTestCatalog implements Catalog, SupportsNamespaces { private final Map> namespaceTableMap; private final Map tableMap; - private IcebergTestCatalog(final String path, final FileIO fileIO) { + private IcebergTestCatalog(final String path, @NotNull final Map properties) { namespaceTableMap = new HashMap<>(); tableMap = new HashMap<>(); @@ -33,7 +33,7 @@ private IcebergTestCatalog(final String path, final FileIO fileIO) { if (tableFile.isDirectory()) { // Second level is table name. final TableIdentifier tableId = TableIdentifier.of(namespace, tableFile.getName()); - final Table table = IcebergTestTable.loadFromMetadata(tableFile.getAbsolutePath(), fileIO); + final Table table = IcebergTestTable.loadFromMetadata(tableFile.getAbsolutePath(), properties); // Add it to the maps. namespaceTableMap.get(namespace).put(tableId, table); @@ -44,8 +44,8 @@ private IcebergTestCatalog(final String path, final FileIO fileIO) { } } - public static IcebergTestCatalog create(final String path, final FileIO fileIO) { - return new IcebergTestCatalog(path, fileIO); + public static IcebergTestCatalog create(final String path, @NotNull final Map properties) { + return new IcebergTestCatalog(path, properties); } @Override diff --git a/extensions/iceberg/src/test/java/io/deephaven/iceberg/TestCatalog/IcebergTestFileIO.java b/extensions/iceberg/src/test/java/io/deephaven/iceberg/TestCatalog/IcebergTestFileIO.java deleted file mode 100644 index 03be6ca1b5e..00000000000 --- a/extensions/iceberg/src/test/java/io/deephaven/iceberg/TestCatalog/IcebergTestFileIO.java +++ /dev/null @@ -1,49 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.iceberg.TestCatalog; - -import org.apache.iceberg.inmemory.InMemoryFileIO; -import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.io.OutputFile; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashSet; -import java.util.Set; - -public class IcebergTestFileIO extends InMemoryFileIO { - private final Set inputFiles; - private final String matchPrefix; - private final String replacePrefix; - - public IcebergTestFileIO(final String matchPrefix, final String replacePrefix) { - this.matchPrefix = matchPrefix; - this.replacePrefix = replacePrefix; - inputFiles = new HashSet<>(); - } - - @Override - public InputFile newInputFile(String s) { - if (!inputFiles.contains(s)) { - try { - final String replaced = s.replace(matchPrefix, replacePrefix); - final byte[] data = Files.readAllBytes(Path.of(replaced)); - addFile(s, data); - inputFiles.add(s); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - return super.newInputFile(s); - } - - @Override - public OutputFile newOutputFile(String s) { - return null; - } - - @Override - public void deleteFile(String s) {} -} diff --git a/extensions/iceberg/src/test/java/io/deephaven/iceberg/TestCatalog/IcebergTestTable.java b/extensions/iceberg/src/test/java/io/deephaven/iceberg/TestCatalog/IcebergTestTable.java index d1cf5c2ee0e..bcac783def0 100644 --- a/extensions/iceberg/src/test/java/io/deephaven/iceberg/TestCatalog/IcebergTestTable.java +++ b/extensions/iceberg/src/test/java/io/deephaven/iceberg/TestCatalog/IcebergTestTable.java @@ -3,10 +3,12 @@ // package io.deephaven.iceberg.TestCatalog; +import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.*; import org.apache.iceberg.encryption.EncryptionManager; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.LocationProvider; +import org.apache.iceberg.io.ResolvingFileIO; import org.jetbrains.annotations.NotNull; import org.testcontainers.shaded.org.apache.commons.lang3.NotImplementedException; @@ -18,11 +20,14 @@ public class IcebergTestTable implements Table { private final TableMetadata metadata; - private final FileIO fileIO; + private final Map properties; + private final Configuration hadoopConf; + + private IcebergTestTable(@NotNull final String path, @NotNull final Map properties) { + this.properties = properties; + hadoopConf = new Configuration(); - private IcebergTestTable(@NotNull final String path, @NotNull final FileIO fileIO) { final File metadataRoot = new File(path, "metadata"); - this.fileIO = fileIO; final List metadataFiles = new ArrayList<>(); @@ -44,8 +49,10 @@ private IcebergTestTable(@NotNull final String path, @NotNull final FileIO fileI } } - public static IcebergTestTable loadFromMetadata(@NotNull final String path, @NotNull final FileIO fileIO) { - return new IcebergTestTable(path, fileIO); + public static IcebergTestTable loadFromMetadata( + @NotNull final String path, + @NotNull final Map properties) { + return new IcebergTestTable(path, properties); } @Override @@ -214,7 +221,10 @@ public Transaction newTransaction() { @Override public FileIO io() { - return fileIO; + final ResolvingFileIO io = new ResolvingFileIO(); + io.setConf(hadoopConf); + io.initialize(properties); + return io; } @Override diff --git a/extensions/kafka/src/main/java/io/deephaven/kafka/KafkaTools.java b/extensions/kafka/src/main/java/io/deephaven/kafka/KafkaTools.java index e51f44251d6..34cee93c488 100644 --- a/extensions/kafka/src/main/java/io/deephaven/kafka/KafkaTools.java +++ b/extensions/kafka/src/main/java/io/deephaven/kafka/KafkaTools.java @@ -1218,33 +1218,30 @@ public Function visit(@NotNull final PerPar } /** - * Consume from Kafka to {@link StreamConsumer stream consumers} supplied by {@code streamConsumerRegistrar}. - * - * @param kafkaProperties Properties to configure this table and also to be passed to create the KafkaConsumer - * @param topic Kafka topic name - * @param partitionFilter A predicate returning true for the partitions to consume. The convenience constant - * {@code ALL_PARTITIONS} is defined to facilitate requesting all partitions. - * @param partitionToInitialOffset A function specifying the desired initial offset for each partition consumed - * @param keySpec Conversion specification for Kafka record keys - * @param valueSpec Conversion specification for Kafka record values - * @param streamConsumerRegistrarProvider A provider for a function to - * {@link StreamPublisher#register(StreamConsumer) register} {@link StreamConsumer} instances. The registered - * stream consumers must accept {@link ChunkType chunk types} that correspond to - * {@link StreamChunkUtils#chunkTypeForColumnIndex(TableDefinition, int)} for the supplied - * {@link TableDefinition}. See {@link StreamConsumerRegistrarProvider#single(SingleConsumerRegistrar) - * single} and {@link StreamConsumerRegistrarProvider#perPartition(PerPartitionConsumerRegistrar) - * per-partition}. - * @param consumerLoopCallback callback to inject logic into the ingester's consumer loop + * Basic holder structure used to pass multiple objects back to a calling method. */ - public static void consume( + private static class ConsumeStruct { + final TableDefinition tableDefinition; + final KafkaStreamPublisher.Parameters publisherParameters; + final Deserializer keyDeser; + final Deserializer valueDeser; + + private ConsumeStruct( + @NotNull final TableDefinition tableDefinition, + @NotNull final KafkaStreamPublisher.Parameters publisherParameters, + @NotNull final Deserializer keyDeser, + @NotNull final Deserializer valueDeser) { + this.tableDefinition = tableDefinition; + this.publisherParameters = publisherParameters; + this.keyDeser = keyDeser; + this.valueDeser = valueDeser; + } + } + + private static ConsumeStruct getConsumeStruct( @NotNull final Properties kafkaProperties, - @NotNull final String topic, - @NotNull final IntPredicate partitionFilter, - @NotNull final InitialOffsetLookup partitionToInitialOffset, @NotNull final Consume.KeyOrValueSpec keySpec, - @NotNull final Consume.KeyOrValueSpec valueSpec, - @NotNull final StreamConsumerRegistrarProvider streamConsumerRegistrarProvider, - @Nullable final ConsumerLoopCallback consumerLoopCallback) { + @NotNull final Consume.KeyOrValueSpec valueSpec) { if (Consume.isIgnore(keySpec) && Consume.isIgnore(valueSpec)) { throw new IllegalArgumentException( "can't ignore both key and value: keySpec and valueSpec can't both be ignore specs"); @@ -1297,12 +1294,64 @@ public static void consume( .setValueToChunkObjectMapper(valueIngestData.toObjectChunkMapper); } - final KafkaStreamPublisher.Parameters publisherParameters = publisherParametersBuilder.build(); + return new ConsumeStruct(tableDefinition, publisherParametersBuilder.build(), keyDeser, valueDeser); + } + + /** + * Construct a {@link TableDefinition} based on the input Properties and {@link Consume.KeyOrValueSpec} parameters. + * Given the same input Properties and Consume.KeyOrValueSpec parameters, the returned TableDefinition is the same + * as the TableDefinition of the table produced by + * {@link #consumeToTable(Properties, String, IntPredicate, IntToLongFunction, Consume.KeyOrValueSpec, Consume.KeyOrValueSpec, TableType)} + * + * @param kafkaProperties Properties to configure this table + * @param keySpec Conversion specification for Kafka record keys + * @param valueSpec Conversion specification for Kafka record values + * @return A TableDefinition derived from the input Properties and KeyOrValueSpec instances + */ + @SuppressWarnings("unused") + public static TableDefinition getTableDefinition( + @NotNull final Properties kafkaProperties, + @NotNull final Consume.KeyOrValueSpec keySpec, + @NotNull final Consume.KeyOrValueSpec valueSpec) { + return getConsumeStruct(kafkaProperties, keySpec, valueSpec).tableDefinition; + } + + /** + * Consume from Kafka to {@link StreamConsumer stream consumers} supplied by {@code streamConsumerRegistrar}. + * + * @param kafkaProperties Properties to configure this table and also to be passed to create the KafkaConsumer + * @param topic Kafka topic name + * @param partitionFilter A predicate returning true for the partitions to consume. The convenience constant + * {@code ALL_PARTITIONS} is defined to facilitate requesting all partitions. + * @param partitionToInitialOffset A function specifying the desired initial offset for each partition consumed + * @param keySpec Conversion specification for Kafka record keys + * @param valueSpec Conversion specification for Kafka record values + * @param streamConsumerRegistrarProvider A provider for a function to + * {@link StreamPublisher#register(StreamConsumer) register} {@link StreamConsumer} instances. The registered + * stream consumers must accept {@link ChunkType chunk types} that correspond to + * {@link StreamChunkUtils#chunkTypeForColumnIndex(TableDefinition, int)} for the supplied + * {@link TableDefinition}. See {@link StreamConsumerRegistrarProvider#single(SingleConsumerRegistrar) + * single} and {@link StreamConsumerRegistrarProvider#perPartition(PerPartitionConsumerRegistrar) + * per-partition}. + * @param consumerLoopCallback callback to inject logic into the ingester's consumer loop + */ + public static void consume( + @NotNull final Properties kafkaProperties, + @NotNull final String topic, + @NotNull final IntPredicate partitionFilter, + @NotNull final InitialOffsetLookup partitionToInitialOffset, + @NotNull final Consume.KeyOrValueSpec keySpec, + @NotNull final Consume.KeyOrValueSpec valueSpec, + @NotNull final StreamConsumerRegistrarProvider streamConsumerRegistrarProvider, + @Nullable final ConsumerLoopCallback consumerLoopCallback) { + final ConsumeStruct consumeStruct = getConsumeStruct(kafkaProperties, keySpec, valueSpec); + final MutableObject kafkaIngesterHolder = new MutableObject<>(); final Function kafkaRecordConsumerFactory = streamConsumerRegistrarProvider.walk( - new KafkaRecordConsumerFactoryCreator(publisherParameters, kafkaIngesterHolder::getValue)); + new KafkaRecordConsumerFactoryCreator(consumeStruct.publisherParameters, + kafkaIngesterHolder::getValue)); final KafkaIngester ingester = new KafkaIngester( log, @@ -1311,8 +1360,8 @@ public static void consume( partitionFilter, kafkaRecordConsumerFactory, partitionToInitialOffset, - keyDeser, - valueDeser, + consumeStruct.keyDeser, + consumeStruct.valueDeser, consumerLoopCallback); kafkaIngesterHolder.setValue(ingester); ingester.start(); diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java index 042a0b5c72d..47126bc595a 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java @@ -267,9 +267,7 @@ public ColumnPageReader next(@NotNull final SeekableChannelContext channelContex final long dataOffset = ch.position(); nextHeaderOffset = dataOffset + pageHeader.getCompressed_page_size(); final PageType pageType = pageHeader.type; - if (pageType == PageType.DICTIONARY_PAGE && headerOffset == columnChunk.meta_data.getData_page_offset() - && columnChunk.meta_data.getDictionary_page_offset() == 0) { - // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding + if (pageType == PageType.DICTIONARY_PAGE) { // Skip the dictionary page and jump to the data page return next(holder.get()); } diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java index 22df95783a0..da085646a41 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java @@ -297,20 +297,13 @@ private static void buildChildren(Types.GroupBuilder builder, IteratorReference for conversions + */ + private static LogicalTypeAnnotation getLogicalTypeFromConvertedType( final ConvertedType convertedType, final SchemaElement schemaElement) throws ParquetFileReaderException { switch (convertedType) { @@ -429,17 +428,12 @@ private static LogicalTypeAnnotation getLogicalTypeAnnotation( case DATE: return LogicalTypeAnnotation.dateType(); case TIME_MILLIS: - // isAdjustedToUTC parameter is ignored while reading Parquet TIME type, so disregard it here return LogicalTypeAnnotation.timeType(true, LogicalTypeAnnotation.TimeUnit.MILLIS); case TIME_MICROS: return LogicalTypeAnnotation.timeType(true, LogicalTypeAnnotation.TimeUnit.MICROS); case TIMESTAMP_MILLIS: - // TIMESTAMP_MILLIS is always adjusted to UTC - // ref: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md return LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS); case TIMESTAMP_MICROS: - // TIMESTAMP_MICROS is always adjusted to UTC - // ref: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md return LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MICROS); case INTERVAL: return LogicalTypeAnnotation.IntervalLogicalTypeAnnotation.getInstance(); diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/BoolMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/BooleanAsByteMaterializer.java similarity index 78% rename from extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/BoolMaterializer.java rename to extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/BooleanAsByteMaterializer.java index 6e5b7f56994..e3475a5a6d8 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/BoolMaterializer.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/BooleanAsByteMaterializer.java @@ -9,17 +9,17 @@ import java.util.Arrays; -public class BoolMaterializer implements PageMaterializer { +public class BooleanAsByteMaterializer implements PageMaterializer { public static final PageMaterializerFactory FACTORY = new PageMaterializerFactory() { @Override public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { - return new BoolMaterializer(dataReader, (byte) nullValue, numValues); + return new BooleanAsByteMaterializer(dataReader, (byte) nullValue, numValues); } @Override public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { - return new BoolMaterializer(dataReader, numValues); + return new BooleanAsByteMaterializer(dataReader, numValues); } }; @@ -28,11 +28,11 @@ public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int num private final byte nullValue; private final byte[] data; - private BoolMaterializer(ValuesReader dataReader, int numValues) { + private BooleanAsByteMaterializer(ValuesReader dataReader, int numValues) { this(dataReader, (byte) 0, numValues); } - private BoolMaterializer(ValuesReader dataReader, byte nullValue, int numValues) { + private BooleanAsByteMaterializer(ValuesReader dataReader, byte nullValue, int numValues) { this.dataReader = dataReader; this.nullValue = nullValue; this.data = new byte[numValues]; diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/DoubleFromFloatMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/DoubleFromFloatMaterializer.java new file mode 100644 index 00000000000..4d97c220c7a --- /dev/null +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/DoubleFromFloatMaterializer.java @@ -0,0 +1,45 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit DoubleMaterializer and run "./gradlew replicatePageMaterializers" to regenerate +// +// @formatter:off +package io.deephaven.parquet.base.materializers; + +import io.deephaven.parquet.base.PageMaterializer; +import io.deephaven.parquet.base.PageMaterializerFactory; +import org.apache.parquet.column.values.ValuesReader; + +public class DoubleFromFloatMaterializer extends DoubleMaterializerBase implements PageMaterializer { + + public static final PageMaterializerFactory FACTORY = new PageMaterializerFactory() { + @Override + public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { + return new DoubleFromFloatMaterializer(dataReader, (double) nullValue, numValues); + } + + @Override + public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { + return new DoubleFromFloatMaterializer(dataReader, numValues); + } + }; + + private final ValuesReader dataReader; + + private DoubleFromFloatMaterializer(ValuesReader dataReader, int numValues) { + this(dataReader, 0, numValues); + } + + private DoubleFromFloatMaterializer(ValuesReader dataReader, double nullValue, int numValues) { + super(nullValue, numValues); + this.dataReader = dataReader; + } + + @Override + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = dataReader.readFloat(); + } + } +} diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/DoubleMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/DoubleMaterializer.java index f5143679f1f..c8345eff522 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/DoubleMaterializer.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/DoubleMaterializer.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit FloatMaterializer and run "./gradlew replicatePageMaterializers" to regenerate +// ****** Edit LongMaterializer and run "./gradlew replicatePageMaterializers" to regenerate // // @formatter:off package io.deephaven.parquet.base.materializers; @@ -11,9 +11,7 @@ import io.deephaven.parquet.base.PageMaterializerFactory; import org.apache.parquet.column.values.ValuesReader; -import java.util.Arrays; - -public class DoubleMaterializer implements PageMaterializer { +public class DoubleMaterializer extends DoubleMaterializerBase implements PageMaterializer { public static final PageMaterializerFactory FACTORY = new PageMaterializerFactory() { @Override @@ -29,22 +27,13 @@ public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int num private final ValuesReader dataReader; - private final double nullValue; - private final double[] data; - private DoubleMaterializer(ValuesReader dataReader, int numValues) { this(dataReader, 0, numValues); } private DoubleMaterializer(ValuesReader dataReader, double nullValue, int numValues) { + super(nullValue, numValues); this.dataReader = dataReader; - this.nullValue = nullValue; - this.data = new double[numValues]; - } - - @Override - public void fillNulls(int startIndex, int endIndex) { - Arrays.fill(data, startIndex, endIndex, nullValue); } @Override @@ -53,15 +42,4 @@ public void fillValues(int startIndex, int endIndex) { data[ii] = dataReader.readDouble(); } } - - @Override - public Object fillAll() { - fillValues(0, data.length); - return data; - } - - @Override - public Object data() { - return data; - } } diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/DoubleMaterializerBase.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/DoubleMaterializerBase.java new file mode 100644 index 00000000000..22e1c430f33 --- /dev/null +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/DoubleMaterializerBase.java @@ -0,0 +1,39 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit LongMaterializerBase and run "./gradlew replicatePageMaterializers" to regenerate +// +// @formatter:off +package io.deephaven.parquet.base.materializers; + +import io.deephaven.parquet.base.PageMaterializer; + +import java.util.Arrays; + +abstract class DoubleMaterializerBase implements PageMaterializer { + + private final double nullValue; + final double[] data; + + DoubleMaterializerBase(double nullValue, int numValues) { + this.nullValue = nullValue; + this.data = new double[numValues]; + } + + @Override + public void fillNulls(int startIndex, int endIndex) { + Arrays.fill(data, startIndex, endIndex, nullValue); + } + + @Override + public Object fillAll() { + fillValues(0, data.length); + return data; + } + + @Override + public Object data() { + return data; + } +} diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/IntFromBooleanMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/IntFromBooleanMaterializer.java new file mode 100644 index 00000000000..6255c5d6bde --- /dev/null +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/IntFromBooleanMaterializer.java @@ -0,0 +1,45 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit IntMaterializer and run "./gradlew replicatePageMaterializers" to regenerate +// +// @formatter:off +package io.deephaven.parquet.base.materializers; + +import io.deephaven.parquet.base.PageMaterializer; +import io.deephaven.parquet.base.PageMaterializerFactory; +import org.apache.parquet.column.values.ValuesReader; + +public class IntFromBooleanMaterializer extends IntMaterializerBase implements PageMaterializer { + + public static final PageMaterializerFactory FACTORY = new PageMaterializerFactory() { + @Override + public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { + return new IntFromBooleanMaterializer(dataReader, (int) nullValue, numValues); + } + + @Override + public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { + return new IntFromBooleanMaterializer(dataReader, numValues); + } + }; + + private final ValuesReader dataReader; + + private IntFromBooleanMaterializer(ValuesReader dataReader, int numValues) { + this(dataReader, 0, numValues); + } + + private IntFromBooleanMaterializer(ValuesReader dataReader, int nullValue, int numValues) { + super(nullValue, numValues); + this.dataReader = dataReader; + } + + @Override + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = dataReader.readBoolean() ? 1 : 0; + } + } +} diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/IntFromUnsignedByteMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/IntFromUnsignedByteMaterializer.java new file mode 100644 index 00000000000..30b0ae5a86a --- /dev/null +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/IntFromUnsignedByteMaterializer.java @@ -0,0 +1,45 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit LongFromUnsignedShortMaterializer and run "./gradlew replicatePageMaterializers" to regenerate +// +// @formatter:off +package io.deephaven.parquet.base.materializers; + +import io.deephaven.parquet.base.PageMaterializer; +import io.deephaven.parquet.base.PageMaterializerFactory; +import org.apache.parquet.column.values.ValuesReader; + +public class IntFromUnsignedByteMaterializer extends IntMaterializerBase implements PageMaterializer { + + public static final PageMaterializerFactory FACTORY = new PageMaterializerFactory() { + @Override + public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { + return new IntFromUnsignedByteMaterializer(dataReader, (int) nullValue, numValues); + } + + @Override + public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { + return new IntFromUnsignedByteMaterializer(dataReader, numValues); + } + }; + + private final ValuesReader dataReader; + + private IntFromUnsignedByteMaterializer(ValuesReader dataReader, int numValues) { + this(dataReader, 0, numValues); + } + + private IntFromUnsignedByteMaterializer(ValuesReader dataReader, int nullValue, int numValues) { + super(nullValue, numValues); + this.dataReader = dataReader; + } + + @Override + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = Byte.toUnsignedInt((byte) dataReader.readInteger()); + } + } +} diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/IntFromUnsignedShortMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/IntFromUnsignedShortMaterializer.java new file mode 100644 index 00000000000..369a83d56db --- /dev/null +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/IntFromUnsignedShortMaterializer.java @@ -0,0 +1,45 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit LongFromUnsignedShortMaterializer and run "./gradlew replicatePageMaterializers" to regenerate +// +// @formatter:off +package io.deephaven.parquet.base.materializers; + +import io.deephaven.parquet.base.PageMaterializer; +import io.deephaven.parquet.base.PageMaterializerFactory; +import org.apache.parquet.column.values.ValuesReader; + +public class IntFromUnsignedShortMaterializer extends IntMaterializerBase implements PageMaterializer { + + public static final PageMaterializerFactory FACTORY = new PageMaterializerFactory() { + @Override + public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { + return new IntFromUnsignedShortMaterializer(dataReader, (int) nullValue, numValues); + } + + @Override + public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { + return new IntFromUnsignedShortMaterializer(dataReader, numValues); + } + }; + + private final ValuesReader dataReader; + + private IntFromUnsignedShortMaterializer(ValuesReader dataReader, int numValues) { + this(dataReader, 0, numValues); + } + + private IntFromUnsignedShortMaterializer(ValuesReader dataReader, int nullValue, int numValues) { + super(nullValue, numValues); + this.dataReader = dataReader; + } + + @Override + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = Short.toUnsignedInt((short) dataReader.readInteger()); + } + } +} diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/IntMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/IntMaterializer.java index 97f47e23ebe..9b6f423c08a 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/IntMaterializer.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/IntMaterializer.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit FloatMaterializer and run "./gradlew replicatePageMaterializers" to regenerate +// ****** Edit LongMaterializer and run "./gradlew replicatePageMaterializers" to regenerate // // @formatter:off package io.deephaven.parquet.base.materializers; @@ -11,9 +11,7 @@ import io.deephaven.parquet.base.PageMaterializerFactory; import org.apache.parquet.column.values.ValuesReader; -import java.util.Arrays; - -public class IntMaterializer implements PageMaterializer { +public class IntMaterializer extends IntMaterializerBase implements PageMaterializer { public static final PageMaterializerFactory FACTORY = new PageMaterializerFactory() { @Override @@ -29,22 +27,13 @@ public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int num private final ValuesReader dataReader; - private final int nullValue; - private final int[] data; - private IntMaterializer(ValuesReader dataReader, int numValues) { this(dataReader, 0, numValues); } private IntMaterializer(ValuesReader dataReader, int nullValue, int numValues) { + super(nullValue, numValues); this.dataReader = dataReader; - this.nullValue = nullValue; - this.data = new int[numValues]; - } - - @Override - public void fillNulls(int startIndex, int endIndex) { - Arrays.fill(data, startIndex, endIndex, nullValue); } @Override @@ -53,15 +42,4 @@ public void fillValues(int startIndex, int endIndex) { data[ii] = dataReader.readInteger(); } } - - @Override - public Object fillAll() { - fillValues(0, data.length); - return data; - } - - @Override - public Object data() { - return data; - } } diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/IntMaterializerBase.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/IntMaterializerBase.java new file mode 100644 index 00000000000..a307e0c36d7 --- /dev/null +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/IntMaterializerBase.java @@ -0,0 +1,39 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit LongMaterializerBase and run "./gradlew replicatePageMaterializers" to regenerate +// +// @formatter:off +package io.deephaven.parquet.base.materializers; + +import io.deephaven.parquet.base.PageMaterializer; + +import java.util.Arrays; + +abstract class IntMaterializerBase implements PageMaterializer { + + private final int nullValue; + final int[] data; + + IntMaterializerBase(int nullValue, int numValues) { + this.nullValue = nullValue; + this.data = new int[numValues]; + } + + @Override + public void fillNulls(int startIndex, int endIndex) { + Arrays.fill(data, startIndex, endIndex, nullValue); + } + + @Override + public Object fillAll() { + fillValues(0, data.length); + return data; + } + + @Override + public Object data() { + return data; + } +} diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromBooleanMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromBooleanMaterializer.java new file mode 100644 index 00000000000..b3ff8fea210 --- /dev/null +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromBooleanMaterializer.java @@ -0,0 +1,45 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit LongMaterializer and run "./gradlew replicatePageMaterializers" to regenerate +// +// @formatter:off +package io.deephaven.parquet.base.materializers; + +import io.deephaven.parquet.base.PageMaterializer; +import io.deephaven.parquet.base.PageMaterializerFactory; +import org.apache.parquet.column.values.ValuesReader; + +public class LongFromBooleanMaterializer extends LongMaterializerBase implements PageMaterializer { + + public static final PageMaterializerFactory FACTORY = new PageMaterializerFactory() { + @Override + public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { + return new LongFromBooleanMaterializer(dataReader, (long) nullValue, numValues); + } + + @Override + public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { + return new LongFromBooleanMaterializer(dataReader, numValues); + } + }; + + private final ValuesReader dataReader; + + private LongFromBooleanMaterializer(ValuesReader dataReader, int numValues) { + this(dataReader, 0, numValues); + } + + private LongFromBooleanMaterializer(ValuesReader dataReader, long nullValue, int numValues) { + super(nullValue, numValues); + this.dataReader = dataReader; + } + + @Override + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = dataReader.readBoolean() ? 1 : 0; + } + } +} diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromIntMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromIntMaterializer.java new file mode 100644 index 00000000000..a2359344597 --- /dev/null +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromIntMaterializer.java @@ -0,0 +1,45 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit LongMaterializer and run "./gradlew replicatePageMaterializers" to regenerate +// +// @formatter:off +package io.deephaven.parquet.base.materializers; + +import io.deephaven.parquet.base.PageMaterializer; +import io.deephaven.parquet.base.PageMaterializerFactory; +import org.apache.parquet.column.values.ValuesReader; + +public class LongFromIntMaterializer extends LongMaterializerBase implements PageMaterializer { + + public static final PageMaterializerFactory FACTORY = new PageMaterializerFactory() { + @Override + public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { + return new LongFromIntMaterializer(dataReader, (long) nullValue, numValues); + } + + @Override + public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { + return new LongFromIntMaterializer(dataReader, numValues); + } + }; + + private final ValuesReader dataReader; + + private LongFromIntMaterializer(ValuesReader dataReader, int numValues) { + this(dataReader, 0, numValues); + } + + private LongFromIntMaterializer(ValuesReader dataReader, long nullValue, int numValues) { + super(nullValue, numValues); + this.dataReader = dataReader; + } + + @Override + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = dataReader.readInteger(); + } + } +} diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromUnsignedByteMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromUnsignedByteMaterializer.java new file mode 100644 index 00000000000..014d6f2d487 --- /dev/null +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromUnsignedByteMaterializer.java @@ -0,0 +1,45 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit LongFromUnsignedShortMaterializer and run "./gradlew replicatePageMaterializers" to regenerate +// +// @formatter:off +package io.deephaven.parquet.base.materializers; + +import io.deephaven.parquet.base.PageMaterializer; +import io.deephaven.parquet.base.PageMaterializerFactory; +import org.apache.parquet.column.values.ValuesReader; + +public class LongFromUnsignedByteMaterializer extends LongMaterializerBase implements PageMaterializer { + + public static final PageMaterializerFactory FACTORY = new PageMaterializerFactory() { + @Override + public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { + return new LongFromUnsignedByteMaterializer(dataReader, (long) nullValue, numValues); + } + + @Override + public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { + return new LongFromUnsignedByteMaterializer(dataReader, numValues); + } + }; + + private final ValuesReader dataReader; + + private LongFromUnsignedByteMaterializer(ValuesReader dataReader, int numValues) { + this(dataReader, 0, numValues); + } + + private LongFromUnsignedByteMaterializer(ValuesReader dataReader, long nullValue, int numValues) { + super(nullValue, numValues); + this.dataReader = dataReader; + } + + @Override + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = Byte.toUnsignedLong((byte) dataReader.readInteger()); + } + } +} diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromUnsignedIntMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromUnsignedIntMaterializer.java index 297d7ae6cbb..cd37ad39c6a 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromUnsignedIntMaterializer.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromUnsignedIntMaterializer.java @@ -1,6 +1,10 @@ // // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit LongFromUnsignedShortMaterializer and run "./gradlew replicatePageMaterializers" to regenerate +// +// @formatter:off package io.deephaven.parquet.base.materializers; import io.deephaven.parquet.base.PageMaterializer; @@ -35,7 +39,7 @@ private LongFromUnsignedIntMaterializer(ValuesReader dataReader, long nullValue, @Override public void fillValues(int startIndex, int endIndex) { for (int ii = startIndex; ii < endIndex; ii++) { - data[ii] = Integer.toUnsignedLong(dataReader.readInteger()); + data[ii] = Integer.toUnsignedLong((int) dataReader.readInteger()); } } } diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromUnsignedShortMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromUnsignedShortMaterializer.java new file mode 100644 index 00000000000..cf6f10049b8 --- /dev/null +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromUnsignedShortMaterializer.java @@ -0,0 +1,41 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.parquet.base.materializers; + +import io.deephaven.parquet.base.PageMaterializer; +import io.deephaven.parquet.base.PageMaterializerFactory; +import org.apache.parquet.column.values.ValuesReader; + +public class LongFromUnsignedShortMaterializer extends LongMaterializerBase implements PageMaterializer { + + public static final PageMaterializerFactory FACTORY = new PageMaterializerFactory() { + @Override + public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { + return new LongFromUnsignedShortMaterializer(dataReader, (long) nullValue, numValues); + } + + @Override + public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { + return new LongFromUnsignedShortMaterializer(dataReader, numValues); + } + }; + + private final ValuesReader dataReader; + + private LongFromUnsignedShortMaterializer(ValuesReader dataReader, int numValues) { + this(dataReader, 0, numValues); + } + + private LongFromUnsignedShortMaterializer(ValuesReader dataReader, long nullValue, int numValues) { + super(nullValue, numValues); + this.dataReader = dataReader; + } + + @Override + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = Short.toUnsignedLong((short) dataReader.readInteger()); + } + } +} diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/ShortFromBooleanMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/ShortFromBooleanMaterializer.java new file mode 100644 index 00000000000..df09abb1b8f --- /dev/null +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/ShortFromBooleanMaterializer.java @@ -0,0 +1,45 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit ShortMaterializer and run "./gradlew replicatePageMaterializers" to regenerate +// +// @formatter:off +package io.deephaven.parquet.base.materializers; + +import io.deephaven.parquet.base.PageMaterializer; +import io.deephaven.parquet.base.PageMaterializerFactory; +import org.apache.parquet.column.values.ValuesReader; + +public class ShortFromBooleanMaterializer extends ShortMaterializerBase implements PageMaterializer { + + public static final PageMaterializerFactory FACTORY = new PageMaterializerFactory() { + @Override + public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { + return new ShortFromBooleanMaterializer(dataReader, (short) nullValue, numValues); + } + + @Override + public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { + return new ShortFromBooleanMaterializer(dataReader, numValues); + } + }; + + private final ValuesReader dataReader; + + private ShortFromBooleanMaterializer(ValuesReader dataReader, int numValues) { + this(dataReader, (short) 0, numValues); + } + + private ShortFromBooleanMaterializer(ValuesReader dataReader, short nullValue, int numValues) { + super(nullValue, numValues); + this.dataReader = dataReader; + } + + @Override + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = (short) (dataReader.readBoolean() ? 1 : 0); + } + } +} diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/ShortFromUnsignedByteMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/ShortFromUnsignedByteMaterializer.java new file mode 100644 index 00000000000..833724a06cc --- /dev/null +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/ShortFromUnsignedByteMaterializer.java @@ -0,0 +1,41 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.parquet.base.materializers; + +import io.deephaven.parquet.base.PageMaterializer; +import io.deephaven.parquet.base.PageMaterializerFactory; +import org.apache.parquet.column.values.ValuesReader; + +public class ShortFromUnsignedByteMaterializer extends ShortMaterializerBase implements PageMaterializer { + + public static final PageMaterializerFactory FACTORY = new PageMaterializerFactory() { + @Override + public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { + return new ShortFromUnsignedByteMaterializer(dataReader, (short) nullValue, numValues); + } + + @Override + public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { + return new ShortFromUnsignedByteMaterializer(dataReader, numValues); + } + }; + + private final ValuesReader dataReader; + + private ShortFromUnsignedByteMaterializer(ValuesReader dataReader, int numValues) { + this(dataReader, (short) 0, numValues); + } + + private ShortFromUnsignedByteMaterializer(ValuesReader dataReader, short nullValue, int numValues) { + super(nullValue, numValues); + this.dataReader = dataReader; + } + + @Override + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = (short) Byte.toUnsignedInt((byte) dataReader.readInteger()); + } + } +} diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/ShortMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/ShortMaterializer.java index b3c5eae6d9a..cfc389081f0 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/ShortMaterializer.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/ShortMaterializer.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit CharMaterializer and run "./gradlew replicatePageMaterializers" to regenerate +// ****** Edit LongMaterializer and run "./gradlew replicatePageMaterializers" to regenerate // // @formatter:off package io.deephaven.parquet.base.materializers; @@ -11,9 +11,7 @@ import io.deephaven.parquet.base.PageMaterializerFactory; import org.apache.parquet.column.values.ValuesReader; -import java.util.Arrays; - -public class ShortMaterializer implements PageMaterializer { +public class ShortMaterializer extends ShortMaterializerBase implements PageMaterializer { public static final PageMaterializerFactory FACTORY = new PageMaterializerFactory() { @Override @@ -29,22 +27,13 @@ public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int num private final ValuesReader dataReader; - private final short nullValue; - private final short[] data; - private ShortMaterializer(ValuesReader dataReader, int numValues) { this(dataReader, (short) 0, numValues); } private ShortMaterializer(ValuesReader dataReader, short nullValue, int numValues) { + super(nullValue, numValues); this.dataReader = dataReader; - this.nullValue = nullValue; - this.data = new short[numValues]; - } - - @Override - public void fillNulls(int startIndex, int endIndex) { - Arrays.fill(data, startIndex, endIndex, nullValue); } @Override @@ -53,15 +42,4 @@ public void fillValues(int startIndex, int endIndex) { data[ii] = (short) dataReader.readInteger(); } } - - @Override - public Object fillAll() { - fillValues(0, data.length); - return data; - } - - @Override - public Object data() { - return data; - } } diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/ShortMaterializerBase.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/ShortMaterializerBase.java new file mode 100644 index 00000000000..ff9265748e5 --- /dev/null +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/ShortMaterializerBase.java @@ -0,0 +1,39 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit LongMaterializerBase and run "./gradlew replicatePageMaterializers" to regenerate +// +// @formatter:off +package io.deephaven.parquet.base.materializers; + +import io.deephaven.parquet.base.PageMaterializer; + +import java.util.Arrays; + +abstract class ShortMaterializerBase implements PageMaterializer { + + private final short nullValue; + final short[] data; + + ShortMaterializerBase(short nullValue, int numValues) { + this.nullValue = nullValue; + this.data = new short[numValues]; + } + + @Override + public void fillNulls(int startIndex, int endIndex) { + Arrays.fill(data, startIndex, endIndex, nullValue); + } + + @Override + public Object fillAll() { + fillValues(0, data.length); + return data; + } + + @Override + public Object data() { + return data; + } +} diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/TypeInfos.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/TypeInfos.java index 9307d18ad33..b95dfe98412 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/TypeInfos.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/TypeInfos.java @@ -124,7 +124,7 @@ public static PrecisionAndScale getPrecisionAndScale( @NotNull final Map> computedCache, @NotNull final String columnName, @NotNull final RowSet rowSet, - @NotNull Supplier> columnSourceSupplier) { + @NotNull final Supplier> columnSourceSupplier) { return (PrecisionAndScale) computedCache .computeIfAbsent(columnName, unusedColumnName -> new HashMap<>()) .computeIfAbsent(ParquetCacheTags.DECIMAL_ARGS, @@ -152,7 +152,7 @@ static TypeInfo bigDecimalTypeInfo( final String columnName = column.getName(); // noinspection unchecked final PrecisionAndScale precisionAndScale = getPrecisionAndScale( - computedCache, columnName, rowSet, () -> (ColumnSource) columnSourceMap.get(columnName)); + computedCache, columnName, rowSet, () -> columnSourceMap.get(columnName)); final Set> clazzes = Set.of(BigDecimal.class); return new TypeInfo() { @Override @@ -175,14 +175,9 @@ static TypeInfo getTypeInfo( final RowSet rowSet, final Map> columnSourceMap, @NotNull final ParquetInstructions instructions) { - if (BigDecimal.class.equals(column.getDataType())) { + if (column.getDataType() == BigDecimal.class || column.getComponentType() == BigDecimal.class) { return bigDecimalTypeInfo(computedCache, column, rowSet, columnSourceMap); } - if (BigDecimal.class.equals(column.getComponentType())) { - throw new UnsupportedOperationException("Writing arrays/vector columns for big decimals is currently not " + - "supported"); - // TODO(deephaven-core#4612): Add support for this - } return lookupTypeInfo(column, instructions); } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetColumnLocation.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetColumnLocation.java index 4c6e53c4b7b..4ddebb33685 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetColumnLocation.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetColumnLocation.java @@ -37,6 +37,7 @@ import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; +import java.time.Instant; import java.util.Arrays; import java.util.Optional; import java.util.function.Function; @@ -370,22 +371,59 @@ private static ToPage makeToPage( final PrimitiveType.PrimitiveTypeName typeName = type.getPrimitiveTypeName(); switch (typeName) { case BOOLEAN: - toPage = ToBooleanAsBytePage.create(pageType); + if (pageType == Boolean.class) { + toPage = ToBooleanAsBytePage.create(pageType); + } else if (pageType == byte.class) { + toPage = ToBytePage.createFromBoolean(pageType); + } else if (pageType == short.class) { + toPage = ToShortPage.createFromBoolean(pageType); + } else if (pageType == int.class) { + toPage = ToIntPage.createFromBoolean(pageType); + } else if (pageType == long.class) { + toPage = ToLongPage.createFromBoolean(pageType); + } else { + throw new IllegalArgumentException( + "Cannot convert parquet BOOLEAN primitive column to " + pageType); + } break; case INT32: - toPage = ToIntPage.create(pageType); + if (pageType == int.class) { + toPage = ToIntPage.create(pageType); + } else if (pageType == long.class) { + toPage = ToLongPage.createFromInt(pageType); + } else { + throw new IllegalArgumentException("Cannot convert parquet INT32 column to " + pageType); + } break; case INT64: - toPage = ToLongPage.create(pageType); + if (pageType == long.class) { + toPage = ToLongPage.create(pageType); + } else { + throw new IllegalArgumentException("Cannot convert parquet INT64 column to " + pageType); + } break; case INT96: - toPage = ToInstantPage.createFromInt96(pageType); + if (pageType == Instant.class) { + toPage = ToInstantPage.createFromInt96(pageType); + } else { + throw new IllegalArgumentException("Cannot convert parquet INT96 column to " + pageType); + } break; case DOUBLE: - toPage = ToDoublePage.create(pageType); + if (pageType == double.class) { + toPage = ToDoublePage.create(pageType); + } else { + throw new IllegalArgumentException("Cannot convert parquet DOUBLE column to " + pageType); + } break; case FLOAT: - toPage = ToFloatPage.create(pageType); + if (pageType == float.class) { + toPage = ToFloatPage.create(pageType); + } else if (pageType == double.class) { + toPage = ToDoublePage.createFromFloat(pageType); + } else { + throw new IllegalArgumentException("Cannot convert parquet FLOAT column to " + pageType); + } break; case BINARY: case FIXED_LEN_BYTE_ARRAY: @@ -413,9 +451,9 @@ private static ToPage makeToPage( } if (toPage == null) { - throw new TableDataException( + throw new IllegalArgumentException( "Unsupported parquet column type " + type.getPrimitiveTypeName() + - " with logical type " + logicalTypeAnnotation); + " with logical type " + logicalTypeAnnotation + " and page type " + pageType); } if (specialTypeName == ColumnTypeInfo.SpecialType.StringSet) { @@ -433,7 +471,7 @@ private static ToPage makeToPage( // noinspection unchecked return (ToPage) toPage; - } catch (RuntimeException except) { + } catch (final RuntimeException except) { throw new TableDataException( "Unexpected exception accessing column " + parquetColumnName, except); } @@ -494,19 +532,59 @@ private static class LogicalTypeVisitor if (intLogicalType.isSigned()) { switch (intLogicalType.getBitWidth()) { case 8: - return Optional.of(ToBytePage.create(pageType)); + if (pageType == byte.class) { + return Optional.of(ToBytePage.create(pageType)); + } else if (pageType == short.class) { + return Optional.of(ToShortPage.create(pageType)); + } else if (pageType == int.class) { + return Optional.of(ToIntPage.create(pageType)); + } else if (pageType == long.class) { + return Optional.of(ToLongPage.createFromInt(pageType)); + } + throw new IllegalArgumentException("Cannot convert parquet byte column to " + pageType); case 16: - return Optional.of(ToShortPage.create(pageType)); + if (pageType == short.class) { + return Optional.of(ToShortPage.create(pageType)); + } else if (pageType == int.class) { + return Optional.of(ToIntPage.create(pageType)); + } else if (pageType == long.class) { + return Optional.of(ToLongPage.createFromInt(pageType)); + } + throw new IllegalArgumentException("Cannot convert parquet short column to " + pageType); case 32: - return Optional.of(ToIntPage.create(pageType)); + if (pageType == int.class) { + return Optional.of(ToIntPage.create(pageType)); + } else if (pageType == long.class) { + return Optional.of(ToLongPage.createFromInt(pageType)); + } + throw new IllegalArgumentException("Cannot convert parquet int column to " + pageType); case 64: return Optional.of(ToLongPage.create(pageType)); } } else { switch (intLogicalType.getBitWidth()) { case 8: + if (pageType == char.class) { + return Optional.of(ToCharPage.create(pageType)); + } else if (pageType == short.class) { + return Optional.of(ToShortPage.createFromUnsignedByte(pageType)); + } else if (pageType == int.class) { + return Optional.of(ToIntPage.createFromUnsignedByte(pageType)); + } else if (pageType == long.class) { + return Optional.of(ToLongPage.createFromUnsignedByte(pageType)); + } + throw new IllegalArgumentException( + "Cannot convert parquet unsigned byte column to " + pageType); case 16: - return Optional.of(ToCharPage.create(pageType)); + if (pageType == char.class) { + return Optional.of(ToCharPage.create(pageType)); + } else if (pageType == int.class) { + return Optional.of(ToIntPage.createFromUnsignedShort(pageType)); + } else if (pageType == long.class) { + return Optional.of(ToLongPage.createFromUnsignedShort(pageType)); + } + throw new IllegalArgumentException( + "Cannot convert parquet unsigned short column to " + pageType); case 32: return Optional.of(ToLongPage.createFromUnsignedInt(pageType)); } @@ -547,7 +625,7 @@ private static class LogicalTypeVisitor case FIXED_LEN_BYTE_ARRAY: // fall through case BINARY: final int encodedSizeInBytes = typeName == BINARY ? -1 : type.getTypeLength(); - if (BigDecimal.class.equals(pageType)) { + if (pageType == BigDecimal.class) { final int precision = decimalLogicalType.getPrecision(); final int scale = decimalLogicalType.getScale(); try { @@ -560,7 +638,7 @@ private static class LogicalTypeVisitor pageType, new BigDecimalParquetBytesCodec(precision, scale, encodedSizeInBytes), columnChunkReader.getDictionarySupplier())); - } else if (BigInteger.class.equals(pageType)) { + } else if (pageType == BigInteger.class) { return Optional.of(ToBigIntegerPage.create( pageType, new BigIntegerParquetBytesCodec(encodedSizeInBytes), diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBooleanAsBytePage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBooleanAsBytePage.java index 8be732df7a2..529bba27066 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBooleanAsBytePage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBooleanAsBytePage.java @@ -5,7 +5,7 @@ import io.deephaven.chunk.attributes.Any; import io.deephaven.parquet.base.PageMaterializerFactory; -import io.deephaven.parquet.base.materializers.BoolMaterializer; +import io.deephaven.parquet.base.materializers.BooleanAsByteMaterializer; import io.deephaven.vector.ObjectVector; import io.deephaven.vector.ObjectVectorDirect; import io.deephaven.util.BooleanUtils; @@ -56,7 +56,7 @@ public final Object nullValue() { @Override public final PageMaterializerFactory getPageMaterializerFactory() { - return BoolMaterializer.FACTORY; + return BooleanAsByteMaterializer.FACTORY; } @Override diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBytePage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBytePage.java index f63d9f2ebcb..1f953292d20 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBytePage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBytePage.java @@ -1,15 +1,12 @@ // // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // -// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit ToIntPage and run "./gradlew replicateToPage" to regenerate -// -// @formatter:off package io.deephaven.parquet.table.pagestore.topage; import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.attributes.Any; import io.deephaven.parquet.base.PageMaterializerFactory; +import io.deephaven.parquet.base.materializers.BooleanAsByteMaterializer; import io.deephaven.parquet.base.materializers.ByteMaterializer; import org.jetbrains.annotations.NotNull; @@ -17,19 +14,35 @@ public class ToBytePage implements ToPage { + public static ToBytePage create(Class nativeType) { + verifyNativeType(nativeType); + // noinspection unchecked + return FROM_BYTE; + } + + public static ToBytePage createFromBoolean(final Class nativeType) { + verifyNativeType(nativeType); + // noinspection unchecked + return FROM_BOOLEAN; + } + @SuppressWarnings("rawtypes") - private static final ToBytePage INSTANCE = new ToBytePage<>(); + private static final ToBytePage FROM_BYTE = new ToBytePage<>(ByteMaterializer.FACTORY); + @SuppressWarnings("rawtypes") + private static final ToBytePage FROM_BOOLEAN = new ToBytePage<>(BooleanAsByteMaterializer.FACTORY); - public static ToBytePage create(Class nativeType) { + private static void verifyNativeType(final Class nativeType) { if (nativeType == null || byte.class.equals(nativeType)) { - // noinspection unchecked - return INSTANCE; + return; } - throw new IllegalArgumentException("The native type for a Byte column is " + nativeType.getCanonicalName()); } - private ToBytePage() {} + private final PageMaterializerFactory pageMaterializerFactory; + + private ToBytePage(@NotNull final PageMaterializerFactory pageMaterializerFactory) { + this.pageMaterializerFactory = pageMaterializerFactory; + } @Override @NotNull @@ -52,6 +65,6 @@ public final Object nullValue() { @Override @NotNull public final PageMaterializerFactory getPageMaterializerFactory() { - return ByteMaterializer.FACTORY; + return pageMaterializerFactory; } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToCharPage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToCharPage.java index 3fbccc9eb95..2248cede91a 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToCharPage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToCharPage.java @@ -1,10 +1,6 @@ // // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // -// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit ToIntPage and run "./gradlew replicateToPage" to regenerate -// -// @formatter:off package io.deephaven.parquet.table.pagestore.topage; import io.deephaven.chunk.ChunkType; diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToDoublePage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToDoublePage.java index f5f450f4753..474b4d3106f 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToDoublePage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToDoublePage.java @@ -1,15 +1,12 @@ // // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // -// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit ToIntPage and run "./gradlew replicateToPage" to regenerate -// -// @formatter:off package io.deephaven.parquet.table.pagestore.topage; import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.attributes.Any; import io.deephaven.parquet.base.PageMaterializerFactory; +import io.deephaven.parquet.base.materializers.DoubleFromFloatMaterializer; import io.deephaven.parquet.base.materializers.DoubleMaterializer; import org.jetbrains.annotations.NotNull; @@ -17,19 +14,35 @@ public class ToDoublePage implements ToPage { + public static ToDoublePage create(final Class nativeType) { + verifyNativeType(nativeType); + // noinspection unchecked + return FROM_DOUBLE; + } + + public static ToDoublePage createFromFloat(final Class nativeType) { + verifyNativeType(nativeType); + // noinspection unchecked + return FROM_FLOAT; + } + @SuppressWarnings("rawtypes") - private static final ToDoublePage INSTANCE = new ToDoublePage<>(); + private static final ToDoublePage FROM_DOUBLE = new ToDoublePage<>(DoubleMaterializer.FACTORY); + @SuppressWarnings("rawtypes") + private static final ToDoublePage FROM_FLOAT = new ToDoublePage<>(DoubleFromFloatMaterializer.FACTORY); - public static ToDoublePage create(Class nativeType) { + private static void verifyNativeType(final Class nativeType) { if (nativeType == null || double.class.equals(nativeType)) { - // noinspection unchecked - return INSTANCE; + return; } - throw new IllegalArgumentException("The native type for a Double column is " + nativeType.getCanonicalName()); } - private ToDoublePage() {} + private final PageMaterializerFactory pageMaterializerFactory; + + private ToDoublePage(@NotNull final PageMaterializerFactory pageMaterializerFactory) { + this.pageMaterializerFactory = pageMaterializerFactory; + } @Override @NotNull @@ -52,6 +65,6 @@ public final Object nullValue() { @Override @NotNull public final PageMaterializerFactory getPageMaterializerFactory() { - return DoubleMaterializer.FACTORY; + return pageMaterializerFactory; } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToFloatPage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToFloatPage.java index 64ef1bd83ed..63bb095afec 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToFloatPage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToFloatPage.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit ToIntPage and run "./gradlew replicateToPage" to regenerate +// ****** Edit ToCharPage and run "./gradlew replicateToPage" to regenerate // // @formatter:off package io.deephaven.parquet.table.pagestore.topage; diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToIntPage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToIntPage.java index 3217d95c0f5..4b0ef392fd3 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToIntPage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToIntPage.java @@ -6,6 +6,9 @@ import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.attributes.Any; import io.deephaven.parquet.base.PageMaterializerFactory; +import io.deephaven.parquet.base.materializers.IntFromBooleanMaterializer; +import io.deephaven.parquet.base.materializers.IntFromUnsignedByteMaterializer; +import io.deephaven.parquet.base.materializers.IntFromUnsignedShortMaterializer; import io.deephaven.parquet.base.materializers.IntMaterializer; import org.jetbrains.annotations.NotNull; @@ -13,19 +16,51 @@ public class ToIntPage implements ToPage { + public static ToIntPage create(Class nativeType) { + verifyNativeType(nativeType); + // noinspection unchecked + return FROM_INT; + } + + public static ToIntPage createFromUnsignedShort(final Class nativeType) { + verifyNativeType(nativeType); + // noinspection unchecked + return FROM_UNSIGNED_SHORT; + } + + public static ToIntPage createFromUnsignedByte(final Class nativeType) { + verifyNativeType(nativeType); + // noinspection unchecked + return FROM_UNSIGNED_BYTE; + } + + public static ToIntPage createFromBoolean(final Class nativeType) { + verifyNativeType(nativeType); + // noinspection unchecked + return FROM_BOOLEAN; + } + + @SuppressWarnings("rawtypes") + private static final ToIntPage FROM_INT = new ToIntPage<>(IntMaterializer.FACTORY); + @SuppressWarnings("rawtypes") + private static final ToIntPage FROM_UNSIGNED_SHORT = new ToIntPage<>(IntFromUnsignedShortMaterializer.FACTORY); + @SuppressWarnings("rawtypes") + private static final ToIntPage FROM_UNSIGNED_BYTE = new ToIntPage<>(IntFromUnsignedByteMaterializer.FACTORY); @SuppressWarnings("rawtypes") - private static final ToIntPage INSTANCE = new ToIntPage<>(); + private static final ToIntPage FROM_BOOLEAN = new ToIntPage<>(IntFromBooleanMaterializer.FACTORY); - public static ToIntPage create(Class nativeType) { + private static void verifyNativeType(final Class nativeType) { if (nativeType == null || int.class.equals(nativeType)) { - // noinspection unchecked - return INSTANCE; + return; } - throw new IllegalArgumentException("The native type for a Int column is " + nativeType.getCanonicalName()); } - private ToIntPage() {} + private final PageMaterializerFactory pageMaterializerFactory; + + private ToIntPage(@NotNull final PageMaterializerFactory pageMaterializerFactory) { + this.pageMaterializerFactory = pageMaterializerFactory; + } @Override @NotNull @@ -48,6 +83,6 @@ public final Object nullValue() { @Override @NotNull public final PageMaterializerFactory getPageMaterializerFactory() { - return IntMaterializer.FACTORY; + return pageMaterializerFactory; } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToLongPage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToLongPage.java index ca86ced2540..befd67571a1 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToLongPage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToLongPage.java @@ -6,7 +6,11 @@ import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.attributes.Any; import io.deephaven.parquet.base.PageMaterializerFactory; +import io.deephaven.parquet.base.materializers.LongFromBooleanMaterializer; +import io.deephaven.parquet.base.materializers.LongFromIntMaterializer; +import io.deephaven.parquet.base.materializers.LongFromUnsignedByteMaterializer; import io.deephaven.parquet.base.materializers.LongFromUnsignedIntMaterializer; +import io.deephaven.parquet.base.materializers.LongFromUnsignedShortMaterializer; import io.deephaven.parquet.base.materializers.LongMaterializer; import org.jetbrains.annotations.NotNull; @@ -26,10 +30,42 @@ public static ToLongPage createFromUnsignedInt(final Cl return FROM_UNSIGNED_INT; } + public static ToLongPage createFromInt(final Class nativeType) { + verifyNativeType(nativeType); + // noinspection unchecked + return FROM_INT; + } + + public static ToLongPage createFromUnsignedShort(final Class nativeType) { + verifyNativeType(nativeType); + // noinspection unchecked + return FROM_UNSIGNED_SHORT; + } + + public static ToLongPage createFromUnsignedByte(final Class nativeType) { + verifyNativeType(nativeType); + // noinspection unchecked + return FROM_UNSIGNED_BYTE; + } + + public static ToLongPage createFromBoolean(final Class nativeType) { + verifyNativeType(nativeType); + // noinspection unchecked + return FROM_BOOLEAN; + } + @SuppressWarnings("rawtypes") private static final ToLongPage FROM_LONG = new ToLongPage<>(LongMaterializer.FACTORY); @SuppressWarnings("rawtypes") private static final ToLongPage FROM_UNSIGNED_INT = new ToLongPage<>(LongFromUnsignedIntMaterializer.FACTORY); + @SuppressWarnings("rawtypes") + private static final ToLongPage FROM_INT = new ToLongPage<>(LongFromIntMaterializer.FACTORY); + @SuppressWarnings("rawtypes") + private static final ToLongPage FROM_UNSIGNED_SHORT = new ToLongPage<>(LongFromUnsignedShortMaterializer.FACTORY); + @SuppressWarnings("rawtypes") + private static final ToLongPage FROM_UNSIGNED_BYTE = new ToLongPage<>(LongFromUnsignedByteMaterializer.FACTORY); + @SuppressWarnings("rawtypes") + private static final ToLongPage FROM_BOOLEAN = new ToLongPage<>(LongFromBooleanMaterializer.FACTORY); private static void verifyNativeType(final Class nativeType) { if (nativeType == null || long.class.equals(nativeType)) { diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToShortPage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToShortPage.java index ce749f217a4..706617025b9 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToShortPage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToShortPage.java @@ -1,15 +1,13 @@ // // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // -// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit ToIntPage and run "./gradlew replicateToPage" to regenerate -// -// @formatter:off package io.deephaven.parquet.table.pagestore.topage; import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.attributes.Any; import io.deephaven.parquet.base.PageMaterializerFactory; +import io.deephaven.parquet.base.materializers.ShortFromBooleanMaterializer; +import io.deephaven.parquet.base.materializers.ShortFromUnsignedByteMaterializer; import io.deephaven.parquet.base.materializers.ShortMaterializer; import org.jetbrains.annotations.NotNull; @@ -17,19 +15,43 @@ public class ToShortPage implements ToPage { + public static ToShortPage create(final Class nativeType) { + verifyNativeType(nativeType); + // noinspection unchecked + return FROM_SHORT; + } + + public static ToShortPage createFromUnsignedByte(final Class nativeType) { + verifyNativeType(nativeType); + // noinspection unchecked + return FROM_UNSIGNED_BYTE; + } + + public static ToShortPage createFromBoolean(final Class nativeType) { + verifyNativeType(nativeType); + // noinspection unchecked + return FROM_BOOLEAN; + } + + @SuppressWarnings("rawtypes") + private static final ToShortPage FROM_SHORT = new ToShortPage<>(ShortMaterializer.FACTORY); + @SuppressWarnings("rawtypes") + private static final ToShortPage FROM_UNSIGNED_BYTE = new ToShortPage<>(ShortFromUnsignedByteMaterializer.FACTORY); @SuppressWarnings("rawtypes") - private static final ToShortPage INSTANCE = new ToShortPage<>(); + private static final ToShortPage FROM_BOOLEAN = new ToShortPage<>(ShortFromBooleanMaterializer.FACTORY); - public static ToShortPage create(Class nativeType) { + private static void verifyNativeType(final Class nativeType) { if (nativeType == null || short.class.equals(nativeType)) { - // noinspection unchecked - return INSTANCE; + return; } - throw new IllegalArgumentException("The native type for a Short column is " + nativeType.getCanonicalName()); } - private ToShortPage() {} + private final PageMaterializerFactory pageMaterializerFactory; + + private ToShortPage(@NotNull final PageMaterializerFactory pageMaterializerFactory) { + this.pageMaterializerFactory = pageMaterializerFactory; + } @Override @NotNull @@ -52,6 +74,6 @@ public final Object nullValue() { @Override @NotNull public final PageMaterializerFactory getPageMaterializerFactory() { - return ShortMaterializer.FACTORY; + return pageMaterializerFactory; } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/transfer/TransferObject.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/transfer/TransferObject.java index e87eff5aa85..fac10941cf1 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/transfer/TransferObject.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/transfer/TransferObject.java @@ -39,7 +39,7 @@ static TransferObject create( @NotNull final Map> computedCache, @NotNull final String columnName, @NotNull final ColumnSource columnSource) { - Class columnType = columnSource.getType(); + final Class columnType = columnSource.getType(); if (columnType == int.class) { return IntTransfer.create(columnSource, tableRowSet, instructions.getTargetPageSize()); } @@ -84,13 +84,11 @@ static TransferObject create( return new CodecTransfer<>(columnSource, codec, tableRowSet, instructions.getTargetPageSize()); } if (columnType == BigDecimal.class) { - // noinspection unchecked - final ColumnSource bigDecimalColumnSource = (ColumnSource) columnSource; final BigDecimalUtils.PrecisionAndScale precisionAndScale = TypeInfos.getPrecisionAndScale( - computedCache, columnName, tableRowSet, () -> bigDecimalColumnSource); + computedCache, columnName, tableRowSet, () -> columnSource); final ObjectCodec codec = new BigDecimalParquetBytesCodec( precisionAndScale.precision, precisionAndScale.scale); - return new CodecTransfer<>(bigDecimalColumnSource, codec, tableRowSet, instructions.getTargetPageSize()); + return new CodecTransfer<>(columnSource, codec, tableRowSet, instructions.getTargetPageSize()); } if (columnType == BigInteger.class) { return new CodecTransfer<>(columnSource, new BigIntegerParquetBytesCodec(), tableRowSet, @@ -136,6 +134,13 @@ static TransferObject create( if (componentType == String.class) { return new StringArrayTransfer(columnSource, tableRowSet, instructions.getTargetPageSize()); } + if (componentType == BigDecimal.class) { + final BigDecimalUtils.PrecisionAndScale precisionAndScale = TypeInfos.getPrecisionAndScale( + computedCache, columnName, tableRowSet, () -> columnSource); + final ObjectCodec codec = new BigDecimalParquetBytesCodec( + precisionAndScale.precision, precisionAndScale.scale); + return new CodecArrayTransfer<>(columnSource, codec, tableRowSet, instructions.getTargetPageSize()); + } if (componentType == BigInteger.class) { return new CodecArrayTransfer<>(columnSource, new BigIntegerParquetBytesCodec(), tableRowSet, instructions.getTargetPageSize()); @@ -152,7 +157,7 @@ static TransferObject create( if (componentType == LocalDateTime.class) { return new LocalDateTimeArrayTransfer(columnSource, tableRowSet, instructions.getTargetPageSize()); } - // TODO(deephaven-core#4612): Handle arrays of BigDecimal and if explicit codec provided + // TODO(deephaven-core#4612): Handle if explicit codec provided } if (Vector.class.isAssignableFrom(columnType)) { if (componentType == int.class) { @@ -182,6 +187,13 @@ static TransferObject create( if (componentType == String.class) { return new StringVectorTransfer(columnSource, tableRowSet, instructions.getTargetPageSize()); } + if (componentType == BigDecimal.class) { + final BigDecimalUtils.PrecisionAndScale precisionAndScale = TypeInfos.getPrecisionAndScale( + computedCache, columnName, tableRowSet, () -> columnSource); + final ObjectCodec codec = new BigDecimalParquetBytesCodec( + precisionAndScale.precision, precisionAndScale.scale); + return new CodecVectorTransfer<>(columnSource, codec, tableRowSet, instructions.getTargetPageSize()); + } if (componentType == BigInteger.class) { return new CodecVectorTransfer<>(columnSource, new BigIntegerParquetBytesCodec(), tableRowSet, instructions.getTargetPageSize()); @@ -198,7 +210,7 @@ static TransferObject create( if (componentType == LocalDateTime.class) { return new LocalDateTimeVectorTransfer(columnSource, tableRowSet, instructions.getTargetPageSize()); } - // TODO(deephaven-core#4612): Handle vectors of BigDecimal and if explicit codec provided + // TODO(deephaven-core#4612): Handle if explicit codec provided } // Go with the default diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java index 8a2df002b0e..a6d683afcc0 100644 --- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java +++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java @@ -15,6 +15,13 @@ import io.deephaven.engine.primitive.function.FloatConsumer; import io.deephaven.engine.primitive.function.ShortConsumer; import io.deephaven.engine.primitive.iterator.CloseableIterator; +import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfByte; +import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfChar; +import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfDouble; +import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfFloat; +import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfInt; +import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfLong; +import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfShort; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.PartitionedTable; @@ -637,6 +644,317 @@ public void basicParquetWithMetadataTest() { assertTableEquals(table, fromDiskWithCommonMetadata); } + @Test + public void testOverrideBooleanColumnType() { + final Table table = TableTools.emptyTable(5).update("A = i % 3 == 0 ? true : i % 3 == 1 ? false : null"); + final File dest = new File(rootFile, "testOverrideBooleanColumnType.parquet"); + ParquetTools.writeTable(table, dest.getPath()); + assertTableEquals(table, ParquetTools.readTable(dest.getPath())); + + final Table arrayTable = + TableTools.emptyTable(5).update("A = new Boolean[] {i % 3 == 0 ? true : i % 3 == 1 ? false : null}"); + final File arrayTableDest = new File(rootFile, "testOverrideBooleanArrayType.parquet"); + ParquetTools.writeTable(arrayTable, arrayTableDest.getPath()); + assertTableEquals(arrayTable, ParquetTools.readTable(arrayTableDest.getPath())); + // Boolean -> byte + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofByte("A"))) + .build(); + final Table byteTable = + TableTools.emptyTable(5).update("A = i % 3 == 0 ? (byte)1 : i % 3 == 1 ? (byte)0 : null"); + assertTableEquals(byteTable, ParquetTools.readTable(dest.getPath(), readInstructions)); + + final Table byteArrayTable = + TableTools.emptyTable(5) + .update("A = new byte[] {i % 3 == 0 ? (byte)1 : i % 3 == 1 ? (byte)0 : (byte)null}"); + assertTableEquals(byteArrayTable, ParquetTools.readTable(arrayTableDest.getPath(), + readInstructions.withTableDefinition(byteArrayTable.getDefinition())).select()); + } + + // Boolean -> short + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofShort("A"))) + .build(); + final Table shortTable = + TableTools.emptyTable(5).update("A = i % 3 == 0 ? (short)1 : i % 3 == 1 ? (short)0 : null"); + assertTableEquals(shortTable, ParquetTools.readTable(dest.getPath(), readInstructions)); + + final Table shortArrayTable = + TableTools.emptyTable(5) + .update("A = new short[] {i % 3 == 0 ? (short)1 : i % 3 == (short)1 ? 0 : (short)null}"); + assertTableEquals(shortArrayTable, ParquetTools.readTable(arrayTableDest.getPath(), + readInstructions.withTableDefinition(shortArrayTable.getDefinition())).select()); + } + + // Boolean -> int + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofInt("A"))) + .build(); + final Table intTable = TableTools.emptyTable(5).update("A = i % 3 == 0 ? 1 : i % 3 == 1 ? 0 : null"); + assertTableEquals(intTable, ParquetTools.readTable(dest.getPath(), readInstructions)); + + final Table intArrayTable = + TableTools.emptyTable(5).update("A = new int[] {i % 3 == 0 ? 1 : i % 3 == 1 ? 0 : null}"); + assertTableEquals(intArrayTable, ParquetTools.readTable(arrayTableDest.getPath(), + readInstructions.withTableDefinition(intArrayTable.getDefinition())).select()); + } + // Boolean -> long + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofLong("A"))) + .build(); + final Table longTable = TableTools.emptyTable(5).update("A = i % 3 == 0 ? 1L : i % 3 == 1 ? 0L : null"); + assertTableEquals(longTable, ParquetTools.readTable(dest.getPath(), readInstructions)); + + final Table longArrayTable = + TableTools.emptyTable(5).update("A = new long[] {i % 3 == 0 ? 1L : i % 3 == 1 ? 0L : null}"); + assertTableEquals(longArrayTable, ParquetTools.readTable(arrayTableDest.getPath(), + readInstructions.withTableDefinition(longArrayTable.getDefinition())).select()); + } + } + + @Test + public void testOverrideByteColumnType() { + final Table table = TableTools.emptyTable(5).update("A=(byte)(i-2)"); + final File dest = new File(rootFile, "testOverrideByteColumnType.parquet"); + ParquetTools.writeTable(table, dest.getPath()); + assertTableEquals(table, ParquetTools.readTable(dest.getPath())); + + final Table arrayTable = TableTools.emptyTable(5).update("A = new byte[] {i == 0 ? null : (byte)(i-2)}"); + final File arrayTableDest = new File(rootFile, "testOverrideByteArrayColumnType.parquet"); + ParquetTools.writeTable(arrayTable, arrayTableDest.getPath()); + assertTableEquals(arrayTable, ParquetTools.readTable(arrayTableDest.getPath())); + + // byte -> short + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofShort("A"))) + .build(); + assertTableEquals(table.updateView("A=(short)A"), + ParquetTools.readTable(dest.getPath(), readInstructions)); + + final Table shortArrayTable = + TableTools.emptyTable(5).update("A = new short[] {i == 0 ? null : (short)(i-2)}"); + assertTableEquals(shortArrayTable, ParquetTools.readTable(arrayTableDest.getPath(), + readInstructions.withTableDefinition(shortArrayTable.getDefinition())).select()); + } + // byte -> int + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofInt("A"))) + .build(); + assertTableEquals(table.updateView("A=(int)A"), + ParquetTools.readTable(dest.getPath(), readInstructions)); + + final Table intArrayTable = TableTools.emptyTable(5).update("A = new int[] {i == 0 ? null : (int)(i-2)}"); + assertTableEquals(intArrayTable, ParquetTools.readTable(arrayTableDest.getPath(), + readInstructions.withTableDefinition(intArrayTable.getDefinition())).select()); + } + // byte -> long + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofLong("A"))) + .build(); + assertTableEquals(table.updateView("A=(long)A"), + ParquetTools.readTable(dest.getPath(), readInstructions)); + + final Table longArrayTable = + TableTools.emptyTable(5).update("A = new long[] {i == 0 ? null : (long)(i-2)}"); + assertTableEquals(longArrayTable, ParquetTools.readTable(arrayTableDest.getPath(), + readInstructions.withTableDefinition(longArrayTable.getDefinition())).select()); + } + // byte -> char + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofChar("A"))) + .build(); + try { + ParquetTools.readTable(dest.getPath(), readInstructions).select(); + fail("Expected an exception because cannot convert byte to char"); + } catch (final RuntimeException ignored) { + } + } + } + + @Test + public void testOverrideShortColumnType() { + final Table table = TableTools.emptyTable(5).update("A=(short)(i-2)"); + final File dest = new File(rootFile, "testOverrideShortColumnType.parquet"); + ParquetTools.writeTable(table, dest.getPath()); + assertTableEquals(table, ParquetTools.readTable(dest.getPath())); + + final Table arrayTable = TableTools.emptyTable(5).update("A = new short[] {i == 0 ? null : (short)(i-2)}"); + final File arrayTableDest = new File(rootFile, "testOverrideShortArrayColumnType.parquet"); + ParquetTools.writeTable(arrayTable, arrayTableDest.getPath()); + assertTableEquals(arrayTable, ParquetTools.readTable(arrayTableDest.getPath())); + + // short -> int + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofInt("A"))) + .build(); + assertTableEquals(table.updateView("A=(int)A"), + ParquetTools.readTable(dest.getPath(), readInstructions)); + + final Table intArrayTable = TableTools.emptyTable(5).update("A = new int[] {i == 0 ? null : (int)(i-2)}"); + assertTableEquals(intArrayTable, ParquetTools.readTable(arrayTableDest.getPath(), + readInstructions.withTableDefinition(intArrayTable.getDefinition())).select()); + } + // short -> long + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofLong("A"))) + .build(); + assertTableEquals(table.updateView("A=(long)A"), + ParquetTools.readTable(dest.getPath(), readInstructions)); + + final Table longArrayTable = + TableTools.emptyTable(5).update("A = new long[] {i == 0 ? null : (long)(i-2)}"); + assertTableEquals(longArrayTable, ParquetTools.readTable(arrayTableDest.getPath(), + readInstructions.withTableDefinition(longArrayTable.getDefinition())).select()); + } + // short -> byte + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofByte("A"))) + .build(); + try { + ParquetTools.readTable(dest.getPath(), readInstructions).select(); + fail("Expected an exception because cannot convert short to byte"); + } catch (final RuntimeException ignored) { + } + } + } + + @Test + public void testOverrideCharColumnType() { + final Table table = TableTools.emptyTable(5).update("A=(char)i"); + final File dest = new File(rootFile, "testOverrideCharColumnType.parquet"); + ParquetTools.writeTable(table, dest.getPath()); + assertTableEquals(table, ParquetTools.readTable(dest.getPath())); + + final Table arrayTable = TableTools.emptyTable(5).update("A = new char[] {i == 0 ? null : (char)i}"); + final File arrayTableDest = new File(rootFile, "testOverrideCharArrayColumnType.parquet"); + ParquetTools.writeTable(arrayTable, arrayTableDest.getPath()); + assertTableEquals(arrayTable, ParquetTools.readTable(arrayTableDest.getPath())); + + // char -> int + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofInt("A"))) + .build(); + assertTableEquals(table.updateView("A=(int)A"), + ParquetTools.readTable(dest.getPath(), readInstructions)); + + final Table intArrayTable = TableTools.emptyTable(5).update("A = new int[] {i == 0 ? null : (int)i}"); + assertTableEquals(intArrayTable, ParquetTools.readTable(arrayTableDest.getPath(), + readInstructions.withTableDefinition(intArrayTable.getDefinition())).select()); + } + // char -> long + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofLong("A"))) + .build(); + assertTableEquals(table.updateView("A=(long)A"), + ParquetTools.readTable(dest.getPath(), readInstructions)); + + final Table longArrayTable = TableTools.emptyTable(5).update("A = new long[] {i == 0 ? null : (long)i}"); + assertTableEquals(longArrayTable, ParquetTools.readTable(arrayTableDest.getPath(), + readInstructions.withTableDefinition(longArrayTable.getDefinition())).select()); + } + // char -> short + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofShort("A"))) + .build(); + try { + ParquetTools.readTable(dest.getPath(), readInstructions).select(); + fail("Expected an exception because cannot convert char to short"); + } catch (final RuntimeException ignored) { + } + } + } + + @Test + public void testOverrideIntColumnType() { + final Table table = TableTools.emptyTable(5).update("A=(int)(i-2)"); + final File dest = new File(rootFile, "testOverrideIntColumnType.parquet"); + ParquetTools.writeTable(table, dest.getPath()); + assertTableEquals(table, ParquetTools.readTable(dest.getPath())); + + final Table arrayTable = TableTools.emptyTable(5).update("A = new int[] {i == 0 ? null : (int)(i-2)}"); + final File arrayTableDest = new File(rootFile, "testOverrideIntArrayColumnType.parquet"); + ParquetTools.writeTable(arrayTable, arrayTableDest.getPath()); + assertTableEquals(arrayTable, ParquetTools.readTable(arrayTableDest.getPath())); + + // int -> long + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofLong("A"))) + .build(); + assertTableEquals(table.updateView("A=(long)A"), + ParquetTools.readTable(dest.getPath(), readInstructions)); + final Table longArrayTable = + TableTools.emptyTable(5).update("A = new long[] {i == 0 ? null : (long)(i-2)}"); + assertTableEquals(longArrayTable, ParquetTools.readTable(arrayTableDest.getPath(), + readInstructions.withTableDefinition(longArrayTable.getDefinition())).select()); + } + + // int -> short + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofShort("A"))) + .build(); + try { + ParquetTools.readTable(dest.getPath(), readInstructions).select(); + fail("Expected an exception because cannot convert int to short"); + } catch (final RuntimeException ignored) { + } + } + } + + @Test + public void testOverrideFloatColumnType() { + final Table table = TableTools.emptyTable(5).update("A=(float)(i-2)"); + final File dest = new File(rootFile, "testOverrideFloatColumnType.parquet"); + ParquetTools.writeTable(table, dest.getPath()); + assertTableEquals(table, ParquetTools.readTable(dest.getPath())); + + final Table arrayTable = TableTools.emptyTable(5).update("A = new float[] {i == 0 ? null : (float)(i-2)}"); + final File arrayTableDest = new File(rootFile, "testOverrideFloatArrayColumnType.parquet"); + ParquetTools.writeTable(arrayTable, arrayTableDest.getPath()); + assertTableEquals(arrayTable, ParquetTools.readTable(arrayTableDest.getPath())); + + // float -> double + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofDouble("A"))) + .build(); + assertTableEquals(table.updateView("A=(double)A"), + ParquetTools.readTable(dest.getPath(), readInstructions)); + final Table doubleArrayTable = + TableTools.emptyTable(5).update("A = new double[] {i == 0 ? null : (double)(i-2)}"); + assertTableEquals(doubleArrayTable, ParquetTools.readTable(arrayTableDest.getPath(), + readInstructions.withTableDefinition(doubleArrayTable.getDefinition())).select()); + } + + // float -> short + { + final ParquetInstructions readInstructions = ParquetInstructions.builder() + .setTableDefinition(TableDefinition.of(ColumnDefinition.ofShort("A"))) + .build(); + try { + ParquetTools.readTable(dest.getPath(), readInstructions).select(); + fail("Expected an exception because cannot convert float to short"); + } catch (final RuntimeException ignored) { + } + } + } + + @Test public void parquetIndexingBuilderTest() { final Table source = TableTools.emptyTable(1_000_000).updateView( @@ -1514,29 +1832,6 @@ private static Table arrayToVectorTable(final Table table) { return arrayToVectorFormulas.isEmpty() ? table : table.updateView(arrayToVectorFormulas); } - @Test - public void testBigDecimalArrayColumn() { - final Table bdArrayTable = TableTools.emptyTable(10000).select(Selectable.from(List.of( - "someBigDecimalArrayColumn = new java.math.BigDecimal[] {i % 10 == 0 ? null : " + - "java.math.BigDecimal.valueOf(ii).stripTrailingZeros()}"))); - final File dest = new File(rootFile + File.separator + "testBigDecimalArrayColumn.parquet"); - try { - ParquetTools.writeTable(bdArrayTable, dest.getPath()); - fail("Expected exception because writing arrays of big decimal column types is not supported"); - } catch (final RuntimeException e) { - assertTrue(e.getCause() instanceof UnsupportedOperationException); - } - - // Convert array to vector table - final Table bdVectorTable = arrayToVectorTable(bdArrayTable); - try { - ParquetTools.writeTable(bdVectorTable, dest.getPath()); - fail("Expected exception because writing vectors of big decimal column types is not supported"); - } catch (final RuntimeException e) { - assertTrue(e.getCause() instanceof UnsupportedOperationException); - } - } - @Test public void testArrayColumns() { ArrayList columns = @@ -1551,6 +1846,7 @@ public void testArrayColumns() { "someByteArrayColumn = new byte[] {i % 10 == 0 ? null : (byte)i}", "someCharArrayColumn = new char[] {i % 10 == 0 ? null : (char)i}", "someTimeArrayColumn = new Instant[] {i % 10 == 0 ? null : (Instant)DateTimeUtils.now() + i}", + "someBigDecimalArrayColumn = new java.math.BigDecimal[] {i % 10 == 0 ? null : java.math.BigDecimal.valueOf(ii).stripTrailingZeros()}", "someBiArrayColumn = new java.math.BigInteger[] {i % 10 == 0 ? null : java.math.BigInteger.valueOf(i)}", "someDateArrayColumn = new java.time.LocalDate[] {i % 10 == 0 ? null : java.time.LocalDate.ofEpochDay(i)}", "someTimeArrayColumn = new java.time.LocalTime[] {i % 10 == 0 ? null : java.time.LocalTime.of(i%24, i%60, (i+10)%60)}", @@ -3802,18 +4098,20 @@ private void assertByteVectorColumnStatistics(SerialObjectColumnIterator max.get()) { - max.set(value); + try (final CloseablePrimitiveIteratorOfByte valuesIterator = values.iterator()) { + valuesIterator.forEachRemaining((final byte value) -> { + itemCount.increment(); + if (value == NULL_BYTE) { + nullCount.increment(); + } else { + if (min.get() == NULL_BYTE || value < min.get()) { + min.set(value); + } + if (max.get() == NULL_BYTE || value > max.get()) { + max.set(value); + } } - } + }); } }); @@ -3911,18 +4209,20 @@ private void assertCharVectorColumnStatistics(SerialObjectColumnIterator max.get()) { - max.set(value); + try (final CloseablePrimitiveIteratorOfChar valuesIterator = values.iterator()) { + valuesIterator.forEachRemaining((final char value) -> { + itemCount.increment(); + if (value == NULL_CHAR) { + nullCount.increment(); + } else { + if (min.get() == NULL_CHAR || value < min.get()) { + min.set(value); + } + if (max.get() == NULL_CHAR || value > max.get()) { + max.set(value); + } } - } + }); } }); @@ -4020,18 +4320,20 @@ private void assertShortVectorColumnStatistics(SerialObjectColumnIterator { + itemCount.increment(); + if (value == NULL_SHORT) { + nullCount.increment(); + } else { + if (min.get() == NULL_SHORT || value < min.get()) { + min.set(value); + } + if (max.get() == NULL_SHORT || value > max.get()) { + max.set(value); + } } - if (max.get() == NULL_SHORT || value > max.get()) { - max.set(value); - } - } + }); } }); @@ -4129,18 +4431,20 @@ private void assertIntVectorColumnStatistics(SerialObjectColumnIterator max.get()) { - max.set(value); + try (final CloseablePrimitiveIteratorOfInt valuesIterator = values.iterator()) { + valuesIterator.forEachRemaining((final int value) -> { + itemCount.increment(); + if (value == NULL_INT) { + nullCount.increment(); + } else { + if (min.get() == NULL_INT || value < min.get()) { + min.set(value); + } + if (max.get() == NULL_INT || value > max.get()) { + max.set(value); + } } - } + }); } }); @@ -4238,18 +4542,20 @@ private void assertLongVectorColumnStatistics(SerialObjectColumnIterator max.get()) { - max.set(value); + try (final CloseablePrimitiveIteratorOfLong valuesIterator = values.iterator()) { + valuesIterator.forEachRemaining((final long value) -> { + itemCount.increment(); + if (value == NULL_LONG) { + nullCount.increment(); + } else { + if (min.get() == NULL_LONG || value < min.get()) { + min.set(value); + } + if (max.get() == NULL_LONG || value > max.get()) { + max.set(value); + } } - } + }); } }); @@ -4349,18 +4655,20 @@ private void assertFloatVectorColumnStatistics(SerialObjectColumnIterator { + itemCount.increment(); + if (value == NULL_FLOAT) { + nullCount.increment(); + } else { + if (min.floatValue() == NULL_FLOAT || value < min.floatValue()) { + min.setValue(value); + } + if (max.floatValue() == NULL_FLOAT || value > max.floatValue()) { + max.setValue(value); + } } - if (max.floatValue() == NULL_FLOAT || value > max.floatValue()) { - max.setValue(value); - } - } + }); } }); @@ -4461,18 +4769,20 @@ private void assertDoubleVectorColumnStatistics(SerialObjectColumnIterator { + itemCount.increment(); + if (value == NULL_DOUBLE) { + nullCount.increment(); + } else { + if (min.doubleValue() == NULL_DOUBLE || value < min.doubleValue()) { + min.setValue(value); + } + if (max.doubleValue() == NULL_DOUBLE || value > max.doubleValue()) { + max.setValue(value); + } } - if (max.doubleValue() == NULL_DOUBLE || value > max.doubleValue()) { - max.setValue(value); - } - } + }); } }); @@ -4572,18 +4882,20 @@ private void assertStringVectorColumnStatistics(SerialObjectColumnIterator 0) { - max.setValue(value); + try (final CloseableIterator valuesIterator = values.iterator()) { + valuesIterator.forEachRemaining((final String value) -> { + itemCount.increment(); + if (value == null) { + nullCount.increment(); + } else { + if (min.getValue() == null || value.compareTo(min.getValue()) < 0) { + min.setValue(value); + } + if (max.getValue() == null || value.compareTo(max.getValue()) > 0) { + max.setValue(value); + } } - } + }); } }); @@ -4684,19 +4996,21 @@ private void assertInstantVectorColumnStatistics(SerialObjectColumnIterator max.get()) { - max.set(DateTimeUtils.epochNanos(value)); + try (final CloseableIterator valuesIterator = values.iterator()) { + valuesIterator.forEachRemaining((final Instant value) -> { + itemCount.increment(); + if (value == null) { + nullCount.increment(); + } else { + // DateTimeUtils.epochNanos() is the correct conversion for Instant to long. + if (min.get() == NULL_LONG || DateTimeUtils.epochNanos(value) < min.get()) { + min.set(DateTimeUtils.epochNanos(value)); + } + if (max.get() == NULL_LONG || DateTimeUtils.epochNanos(value) > max.get()) { + max.set(DateTimeUtils.epochNanos(value)); + } } - } + }); } }); diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/S3ParquetRemoteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/S3ParquetRemoteTest.java index 6d6f1e2803b..6af4c2e45a1 100644 --- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/S3ParquetRemoteTest.java +++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/S3ParquetRemoteTest.java @@ -22,6 +22,7 @@ import static io.deephaven.engine.testutil.TstUtils.assertTableEquals; import static io.deephaven.parquet.table.ParquetTools.readTable; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * These tests verify the behavior of Parquet implementation when reading against remote S3 servers. @@ -86,6 +87,73 @@ public void readSampleParquetFilesFromPublicS3Part2() { .head(10).select(); } + @Test + public void readSampleParquetFilesFromPublicS3Part3() { + Assume.assumeTrue("Skipping test because s3 testing disabled.", ENABLE_REMOTE_S3_TESTING); + final S3Instructions s3Instructions = S3Instructions.builder() + .regionName("us-east-1") + .readTimeout(Duration.ofSeconds(60)) + .credentials(Credentials.anonymous()) + .build(); + final ParquetInstructions readInstructions = new ParquetInstructions.Builder() + .setSpecialInstructions(s3Instructions) + .build(); + readTable("s3://redshift-downloads/redset/serverless/full.parquet", readInstructions).head(10).select(); + } + + @Test + public void readSampleParquetFromPublicGCS() { + Assume.assumeTrue("Skipping test because s3 testing disabled.", ENABLE_REMOTE_S3_TESTING); + final Table tableWithEndpointOverride; + { + final ParquetInstructions readInstructions = new ParquetInstructions.Builder() + .setSpecialInstructions(S3Instructions.builder() + .readTimeout(Duration.ofSeconds(60)) + .credentials(Credentials.anonymous()) + .regionName("us-east-1") + .endpointOverride("https://storage.googleapis.com") + .build()) + .build(); + tableWithEndpointOverride = ParquetTools.readTable( + "s3://cloud-samples-data/bigquery/us-states/us-states.parquet", readInstructions).select(); + assertEquals(2, tableWithEndpointOverride.numColumns()); + assertEquals(50, tableWithEndpointOverride.size()); + } + + final Table tableWithoutEndpointOverride; + { + final ParquetInstructions readInstructions = new ParquetInstructions.Builder() + .setSpecialInstructions(S3Instructions.builder() + .readTimeout(Duration.ofSeconds(60)) + .regionName("us-east-1") + .credentials(Credentials.anonymous()) + .build()) + .build(); + tableWithoutEndpointOverride = ParquetTools.readTable( + "gs://cloud-samples-data/bigquery/us-states/us-states.parquet", readInstructions).select(); + assertEquals(2, tableWithoutEndpointOverride.numColumns()); + assertEquals(50, tableWithoutEndpointOverride.size()); + } + assertTableEquals(tableWithEndpointOverride, tableWithoutEndpointOverride); + } + + @Test + public void testReadFromGCSFailure() { + final ParquetInstructions readInstructions = new ParquetInstructions.Builder() + .setSpecialInstructions(S3Instructions.builder() + .readTimeout(Duration.ofSeconds(60)) + .credentials(Credentials.anonymous()) + .endpointOverride("https://storage.com") + .build()) + .build(); + try { + ParquetTools.readTable( + "gs://cloud-samples-data/bigquery/us-states/us-states.parquet", readInstructions).select(); + } catch (final IllegalArgumentException e) { + assertTrue(e.toString().contains("endpoint override")); + } + } + @Test public void readKeyValuePartitionedParquetFromPublicS3() { Assume.assumeTrue("Skipping test because s3 testing disabled.", ENABLE_REMOTE_S3_TESTING); @@ -106,6 +174,9 @@ public void readKeyValuePartitionedParquetFromPublicS3() { assertEquals(2, table.numColumns()); } + /** + * The follow test reads from Deephaven's s3 bucket, thus requires the credentials to be set up. + */ @Test public void readMetadataPartitionedParquetFromS3() { Assume.assumeTrue("Skipping test because s3 testing disabled.", ENABLE_REMOTE_S3_TESTING); diff --git a/extensions/s3/src/main/java/io/deephaven/extensions/s3/GCSSeekableChannelProvider.java b/extensions/s3/src/main/java/io/deephaven/extensions/s3/GCSSeekableChannelProvider.java new file mode 100644 index 00000000000..cbf7b410101 --- /dev/null +++ b/extensions/s3/src/main/java/io/deephaven/extensions/s3/GCSSeekableChannelProvider.java @@ -0,0 +1,72 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.extensions.s3; + +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.logger.Logger; +import io.deephaven.util.channel.CompletableOutputStream; +import io.deephaven.util.channel.SeekableChannelContext; +import org.jetbrains.annotations.NotNull; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.channels.SeekableByteChannel; +import java.util.stream.Stream; + +import static io.deephaven.extensions.s3.GCSSeekableChannelProviderPlugin.GCS_URI_SCHEME; +import static io.deephaven.extensions.s3.S3SeekableChannelProviderPlugin.S3_URI_SCHEME; + +final class GCSSeekableChannelProvider extends S3SeekableChannelProvider { + + private static final Logger log = LoggerFactory.getLogger(GCSSeekableChannelProvider.class); + + GCSSeekableChannelProvider(@NotNull final S3Instructions s3Instructions) { + super(s3Instructions); + } + + @Override + public boolean exists(@NotNull final URI uri) { + return super.exists(gcsToS3Uri(uri)); + } + + @Override + public SeekableByteChannel getReadChannel( + @NotNull final SeekableChannelContext channelContext, + @NotNull final URI uri) { + return super.getReadChannel(channelContext, gcsToS3Uri(uri)); + } + + @Override + public CompletableOutputStream getOutputStream(@NotNull final URI uri, final int bufferSizeHint) { + return super.getOutputStream(gcsToS3Uri(uri), bufferSizeHint); + } + + @Override + public Stream list(@NotNull final URI directory) { + if (log.isDebugEnabled()) { + log.debug().append("Fetching child URIs for directory: ").append(directory.toString()).endl(); + } + return createStream(gcsToS3Uri(directory), false, GCS_URI_SCHEME); + } + + @Override + public Stream walk(@NotNull final URI directory) { + if (log.isDebugEnabled()) { + log.debug().append("Performing recursive traversal from directory: ").append(directory.toString()).endl(); + } + return createStream(gcsToS3Uri(directory), true, GCS_URI_SCHEME); + } + + private static URI gcsToS3Uri(@NotNull final URI uri) { + try { + if (S3_URI_SCHEME.equals(uri.getScheme())) { + return uri; + } + return new URI(S3_URI_SCHEME, uri.getUserInfo(), uri.getHost(), uri.getPort(), uri.getPath(), + uri.getQuery(), uri.getFragment()); + } catch (final URISyntaxException e) { + throw new IllegalArgumentException("Failed to convert GCS URI " + uri + " to s3 URI", e); + } + } +} diff --git a/extensions/s3/src/main/java/io/deephaven/extensions/s3/GCSSeekableChannelProviderPlugin.java b/extensions/s3/src/main/java/io/deephaven/extensions/s3/GCSSeekableChannelProviderPlugin.java new file mode 100644 index 00000000000..6c3ca91ac8e --- /dev/null +++ b/extensions/s3/src/main/java/io/deephaven/extensions/s3/GCSSeekableChannelProviderPlugin.java @@ -0,0 +1,63 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.extensions.s3; + +import com.google.auto.service.AutoService; +import io.deephaven.util.channel.SeekableChannelsProvider; +import io.deephaven.util.channel.SeekableChannelsProviderPlugin; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.net.URI; + +/** + * {@link SeekableChannelsProviderPlugin} implementation used for reading files from Google Cloud Storage. + */ +@AutoService(SeekableChannelsProviderPlugin.class) +public final class GCSSeekableChannelProviderPlugin implements SeekableChannelsProviderPlugin { + + static final String GCS_URI_SCHEME = "gs"; + + private static final String ENDPOINT_OVERRIDE_SUFFIX = ".googleapis.com"; + private static final URI DEFAULT_ENDPOINT_OVERRIDE = URI.create("https://storage.googleapis.com"); + private static final S3Instructions DEFAULT_INSTRUCTIONS = + S3Instructions.builder().endpointOverride(DEFAULT_ENDPOINT_OVERRIDE).build(); + + @Override + public boolean isCompatible(@NotNull final URI uri, @Nullable final Object config) { + return GCS_URI_SCHEME.equals(uri.getScheme()); + } + + @Override + public SeekableChannelsProvider createProvider(@NotNull final URI uri, @Nullable final Object config) { + if (!isCompatible(uri, config)) { + throw new IllegalArgumentException("Arguments not compatible, provided uri " + uri); + } + return new GCSSeekableChannelProvider(s3Instructions(config)); + } + + /** + * Get the S3Instructions from the config object, or use the default if the config is null. + */ + private static S3Instructions s3Instructions(@Nullable final Object config) { + if (config == null) { + return DEFAULT_INSTRUCTIONS; + } + if (!(config instanceof S3Instructions)) { + throw new IllegalArgumentException("Only S3Instructions are valid when reading GCS URIs, " + + "provided config instance of class " + config.getClass().getName()); + } + final S3Instructions s3Instructions = (S3Instructions) config; + if (s3Instructions.endpointOverride().isEmpty()) { + return s3Instructions.withEndpointOverride(DEFAULT_ENDPOINT_OVERRIDE); + } + if (!(s3Instructions.endpointOverride().get()).toString().endsWith(ENDPOINT_OVERRIDE_SUFFIX)) { + throw new IllegalArgumentException("Provided endpoint override=(" + + s3Instructions.endpointOverride().get() + " not supported when reading GCS URIs, must end with " + + ENDPOINT_OVERRIDE_SUFFIX); + } + return s3Instructions; + } +} + diff --git a/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3Instructions.java b/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3Instructions.java index f6a259c26aa..9fa3e955a2b 100644 --- a/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3Instructions.java +++ b/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3Instructions.java @@ -145,6 +145,8 @@ public LogOutput append(final LogOutput logOutput) { */ public abstract Optional endpointOverride(); + public abstract S3Instructions withEndpointOverride(final URI endpointOverride); + public interface Builder { Builder regionName(String regionName); diff --git a/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3SeekableChannelProvider.java b/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3SeekableChannelProvider.java index 4bd06a1b661..84d1566a374 100644 --- a/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3SeekableChannelProvider.java +++ b/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3SeekableChannelProvider.java @@ -51,7 +51,7 @@ /** * {@link SeekableChannelsProvider} implementation that is used to fetch objects from an S3-compatible API. */ -final class S3SeekableChannelProvider implements SeekableChannelsProvider { +class S3SeekableChannelProvider implements SeekableChannelsProvider { private static final int MAX_KEYS_PER_BATCH = 1000; private static final int UNKNOWN_SIZE = -1; @@ -97,7 +97,8 @@ public boolean exists(@NotNull final URI uri) { } @Override - public SeekableByteChannel getReadChannel(@NotNull final SeekableChannelContext channelContext, + public SeekableByteChannel getReadChannel( + @NotNull final SeekableChannelContext channelContext, @NotNull final URI uri) { final S3Uri s3Uri = s3AsyncClient.utilities().parseUri(uri); // context is unused here, will be set before reading from the channel @@ -140,7 +141,7 @@ public Stream list(@NotNull final URI directory) { if (log.isDebugEnabled()) { log.debug().append("Fetching child URIs for directory: ").append(directory.toString()).endl(); } - return createStream(directory, false); + return createStream(directory, false, S3_URI_SCHEME); } @Override @@ -148,10 +149,20 @@ public Stream walk(@NotNull final URI directory) { if (log.isDebugEnabled()) { log.debug().append("Performing recursive traversal from directory: ").append(directory.toString()).endl(); } - return createStream(directory, true); + return createStream(directory, true, S3_URI_SCHEME); } - private Stream createStream(@NotNull final URI directory, final boolean isRecursive) { + /** + * Create a stream of URIs, the elements of which are the entries in the directory. + * + * @param directory The parent directory to list. + * @param isRecursive Whether to list the entries recursively. + * @param childScheme The scheme to apply to the children URIs in the returned stream. + */ + Stream createStream( + @NotNull final URI directory, + final boolean isRecursive, + @NotNull final String childScheme) { // The following iterator fetches URIs from S3 in batches and creates a stream final Iterator iterator = new Iterator<>() { private final String bucketName; @@ -222,7 +233,7 @@ private void fetchNextBatch() throws IOException { } final URI uri; try { - uri = new URI(S3_URI_SCHEME, directory.getUserInfo(), directory.getHost(), + uri = new URI(childScheme, directory.getUserInfo(), directory.getHost(), directory.getPort(), path, null, null); } catch (final URISyntaxException e) { throw new UncheckedDeephavenException("Failed to create URI for S3 object with key: " diff --git a/extensions/s3/src/test/java/io/deephaven/extensions/s3/testlib/SingletonContainers.java b/extensions/s3/src/test/java/io/deephaven/extensions/s3/testlib/SingletonContainers.java index befd758f980..5d5b550089d 100644 --- a/extensions/s3/src/test/java/io/deephaven/extensions/s3/testlib/SingletonContainers.java +++ b/extensions/s3/src/test/java/io/deephaven/extensions/s3/testlib/SingletonContainers.java @@ -16,6 +16,7 @@ import software.amazon.awssdk.services.s3.S3AsyncClient; import java.net.URI; +import java.util.Map; public final class SingletonContainers { @@ -53,6 +54,14 @@ public static S3AsyncClient s3AsyncClient() { AwsBasicCredentials.create(LOCALSTACK_S3.getAccessKey(), LOCALSTACK_S3.getSecretKey()))) .build(); } + + public static Map s3Properties() { + return Map.of( + "s3.endpoint", LOCALSTACK_S3.getEndpoint().toString(), + "client.region", LOCALSTACK_S3.getRegion(), + "s3.access-key-id", LOCALSTACK_S3.getAccessKey(), + "s3.secret-access-key", LOCALSTACK_S3.getSecretKey()); + } } public static final class MinIO { @@ -87,5 +96,13 @@ public static S3AsyncClient s3AsyncClient() { AwsBasicCredentials.create(MINIO.getUserName(), MINIO.getPassword()))) .build(); } + + public static Map s3Properties() { + return Map.of( + "s3.endpoint", MINIO.getS3URL(), + "client.region", Region.AWS_GLOBAL.toString(), + "s3.access-key-id", MINIO.getUserName(), + "s3.secret-access-key", MINIO.getPassword()); + } } } diff --git a/go/pkg/client/tokens.go b/go/pkg/client/tokens.go index e00d481c10c..8594e06cbb0 100644 --- a/go/pkg/client/tokens.go +++ b/go/pkg/client/tokens.go @@ -7,7 +7,9 @@ import ( "fmt" "github.com/apache/arrow/go/v8/arrow/flight" configpb2 "github.com/deephaven/deephaven-core/go/internal/proto/config" + sessionpb2 "github.com/deephaven/deephaven-core/go/internal/proto/session" "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/proto" "log" "strconv" "sync" @@ -36,8 +38,19 @@ func withAuthToken(ctx context.Context, token []byte) context.Context { } // requestToken requests a new token from flight. -func requestToken(handshakeClient flight.FlightService_HandshakeClient, handshakeReq *flight.HandshakeRequest) ([]byte, error) { - err := handshakeClient.Send(handshakeReq) +func requestToken(handshakeClient flight.FlightService_HandshakeClient, authType string, authToken []byte) ([]byte, error) { + + war := sessionpb2.WrappedAuthenticationRequest{ + Type: authType, + Payload: authToken, + } + payload, err := proto.Marshal(&war) + if err != nil { + return nil, err + } + handshakeReq := flight.HandshakeRequest{Payload: []byte(payload)} + + err = handshakeClient.Send(&handshakeReq) if err != nil { return nil, err @@ -122,15 +135,13 @@ func (tr *tokenManager) Close() error { // "user:password"; when auth_type is DefaultAuth, it will be ignored; when auth_type is a custom-built // authenticator, it must conform to the specific requirement of the authenticator. func newTokenManager(ctx context.Context, fs *flightStub, cfg configpb2.ConfigServiceClient, authType string, authToken string) (*tokenManager, error) { - authString := makeAuthString(authType, authToken) - - handshakeClient, err := fs.handshake(withAuth(ctx, authString)) + handshakeClient, err := fs.handshake(ctx) if err != nil { return nil, err } - tkn, err := requestToken(handshakeClient, &flight.HandshakeRequest{Payload: []byte(authString)}) + tkn, err := requestToken(handshakeClient, authType, []byte(authToken)) if err != nil { return nil, err @@ -174,10 +185,10 @@ func newTokenManager(ctx context.Context, fs *flightStub, cfg configpb2.ConfigSe var tkn []byte if err == nil { - tkn, err = requestToken(handshakeClient, &flight.HandshakeRequest{Payload: oldToken}) + tkn, err = requestToken(handshakeClient, "Bearer", oldToken) } else { log.Println("Old token has an error during token update. Attempting to acquire a fresh token. err=", err) - tkn, err = requestToken(handshakeClient, &flight.HandshakeRequest{Payload: []byte(authString)}) + tkn, err = requestToken(handshakeClient, authType, []byte(authToken)) } if err != nil { diff --git a/gradle.properties b/gradle.properties index 3db9b09aa85..e03001083f8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -9,7 +9,7 @@ # Re-builders who want to inherit the base version, but have their own qualifier can set -PdeephavenBaseQualifier="customQualifier": "X.Y.Z-customQualifier". # # Re-builders who want a fully custom version can set -PdeephavenBaseVersion="customVersion" -PdeephavenBaseQualifier="": "customVersion". -deephavenBaseVersion=0.36.0 +deephavenBaseVersion=0.37.0 deephavenBaseQualifier=SNAPSHOT #org.gradle.debug diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b4fb36cbd85..e02cdb202c5 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -76,7 +76,8 @@ tdunning = "3.2" trove = "3.0.3" undercouch = "2.15.1" univocity = "2.6.0" -vertispan-nio = "1.0-alpha-1" +vertispan-nio = "1.0-alpha-2" +vertispan-flatbuffers-gwt = "1.12.0-1" vertispan-ts-defs = "1.0.0-RC4" # test versions @@ -287,6 +288,8 @@ univocity-parsers = { module = "com.univocity:univocity-parsers", version.ref = vertispan-nio-gwt = { module = "com.vertispan.nio:gwt-nio", version.ref = "vertispan-nio" } +vertispan-flatbuffers-gwt = { module = "com.vertispan.flatbuffers:flatbuffers-gwt", version.ref = "vertispan-flatbuffers-gwt" } + vertispan-ts-defs-annotations = { module = "com.vertispan.tsdefs:jsinterop-ts-defs-annotations", version.ref = "vertispan-ts-defs" } vertispan-ts-defs-doclet = { module = "com.vertispan.tsdefs:jsinterop-ts-defs-doclet", version.ref = "vertispan-ts-defs" } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index e6441136f3d..a4b76b9530d 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 515ab9d5f18..9036682bf0f 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip +distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index b740cf13397..f5feea6d6b1 100755 --- a/gradlew +++ b/gradlew @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -84,7 +86,8 @@ done # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) -APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum diff --git a/gradlew.bat b/gradlew.bat index 7101f8e4676..9b42019c791 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -13,6 +13,8 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem @if "%DEBUG%"=="" @echo off @rem ########################################################################## diff --git a/java-client/session-examples/build.gradle b/java-client/session-examples/build.gradle index db9d4d859bf..f6467f1bdaf 100644 --- a/java-client/session-examples/build.gradle +++ b/java-client/session-examples/build.gradle @@ -49,6 +49,7 @@ application.applicationDistribution.into('bin') { from(createApplication('message-stream-send-receive', 'io.deephaven.client.examples.MessageStreamSendReceive')) from(createApplication('filter-table', 'io.deephaven.client.examples.FilterTable')) from(createApplication('create-shared-id', 'io.deephaven.client.examples.CreateSharedId')) + from(createApplication('print-configuration-constants', 'io.deephaven.client.examples.PrintConfigurationConstants')) fileMode = 0755 } diff --git a/java-client/session-examples/src/main/java/io/deephaven/client/examples/PrintConfigurationConstants.java b/java-client/session-examples/src/main/java/io/deephaven/client/examples/PrintConfigurationConstants.java new file mode 100644 index 00000000000..8a2bf16ccc4 --- /dev/null +++ b/java-client/session-examples/src/main/java/io/deephaven/client/examples/PrintConfigurationConstants.java @@ -0,0 +1,28 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.client.examples; + +import io.deephaven.client.impl.Session; +import io.deephaven.proto.backplane.grpc.ConfigValue; +import picocli.CommandLine; +import picocli.CommandLine.Command; + +import java.util.Map.Entry; + +@Command(name = "print-configuration-constants", mixinStandardHelpOptions = true, + description = "Print configuration constants", version = "0.1.0") +class PrintConfigurationConstants extends SingleSessionExampleBase { + + @Override + protected void execute(Session session) throws Exception { + for (Entry entry : session.getConfigurationConstants().get().entrySet()) { + System.out.println(entry.getKey() + "=" + entry.getValue().getStringValue()); + } + } + + public static void main(String[] args) { + int execute = new CommandLine(new PrintConfigurationConstants()).execute(args); + System.exit(execute); + } +} diff --git a/props/configs/src/main/resources/calendar/UTC.calendar b/props/configs/src/main/resources/calendar/UTC.calendar index 5b1f6c25fda..f3b1a8430d6 100644 --- a/props/configs/src/main/resources/calendar/UTC.calendar +++ b/props/configs/src/main/resources/calendar/UTC.calendar @@ -9,6 +9,6 @@ 1900-01-01 2100-12-31 - 00:0023:59:59.999999999 + 00:0023:59:59.999999999true \ No newline at end of file diff --git a/props/configs/src/main/resources/dh-defaults.prop b/props/configs/src/main/resources/dh-defaults.prop index 94e1a947bf3..d3e0c6d2ede 100644 --- a/props/configs/src/main/resources/dh-defaults.prop +++ b/props/configs/src/main/resources/dh-defaults.prop @@ -57,14 +57,17 @@ web.storage.notebook.directory=/notebooks web.webgl=true web.webgl.editable=true +# Default to not flattening web viewports, but allow it as an option +web.flattenViewports=false + # List of configuration properties to provide to unauthenticated clients, so that they can decide how best to prove their # identity to the server. authentication.client.configuration.list=AuthHandlers # List of configuration properties to provide to authenticated clients, so they can interact with the server. -client.configuration.list=java.version,deephaven.version,barrage.version,http.session.durationMs,file.separator,web.storage.layout.directory,web.storage.notebook.directory,web.webgl,web.webgl.editable +client.configuration.list=java.version,deephaven.version,barrage.version,groovy.version,python.version,http.session.durationMs,file.separator,web.storage.layout.directory,web.storage.notebook.directory,web.webgl,web.webgl.editable,web.flattenViewports # Version list to add to the configuration property list. Each `=`-delimited pair denotes a short name for a versioned # jar, and a class that is found in that jar. Any such keys will be made available to the client.configuration.list # as .version. -client.version.list=deephaven=io.deephaven.engine.table.Table,barrage=io.deephaven.barrage.flatbuf.BarrageMessageWrapper +client.version.list=deephaven=io.deephaven.engine.table.Table,barrage=io.deephaven.barrage.flatbuf.BarrageMessageWrapper,groovy=groovy.lang.GroovyShell diff --git a/proto/proto-backplane-grpc/Dockerfile b/proto/proto-backplane-grpc/Dockerfile index 1fcbd80b8f8..6127483c68d 100644 --- a/proto/proto-backplane-grpc/Dockerfile +++ b/proto/proto-backplane-grpc/Dockerfile @@ -11,6 +11,10 @@ RUN set -eux; \ mkdir -p /generated/go; \ mkdir -p /generated/python; \ mkdir -p /generated/cpp; \ + mkdir -p /generated/proto-doc/single-html; \ + mkdir -p /generated/proto-doc/single-md; \ + mkdir -p /generated/proto-doc/multi-html; \ + mkdir -p /generated/proto-doc/multi-md; \ /opt/protoc/bin/protoc \ --plugin=protoc-gen-grpc=/opt/java/bin/protoc-gen-grpc-java \ --java_out=/generated/java \ @@ -84,4 +88,154 @@ RUN set -eux; \ /includes/deephaven/proto/partitionedtable.proto \ /includes/deephaven/proto/config.proto \ /includes/deephaven/proto/hierarchicaltable.proto \ - /includes/deephaven/proto/storage.proto; + /includes/deephaven/proto/storage.proto; \ + # proto-doc-gen does not support writing multiple proto files to multiple docs, must break up command \ + # we are going to generate 4 sets of docs - single html, multi html, single markdown, multi markdown \ + # first, single html \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/single-html \ + --doc_opt=html,index.html \ + -I/includes \ + /includes/deephaven/proto/*.proto; \ + # next, single markdown \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/single-md \ + --doc_opt=markdown,index.md \ + -I/includes \ + /includes/deephaven/proto/*.proto; \ + # then, multi html \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-html \ + --doc_opt=html,ticket.html \ + -I/includes \ + /includes/deephaven/proto/ticket.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-html \ + --doc_opt=html,console.html \ + -I/includes \ + /includes/deephaven/proto/console.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-html \ + --doc_opt=html,object.html \ + -I/includes \ + /includes/deephaven/proto/object.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-html \ + --doc_opt=html,session.html \ + -I/includes \ + /includes/deephaven/proto/session.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-html \ + --doc_opt=html,table.html \ + -I/includes \ + /includes/deephaven/proto/table.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-html \ + --doc_opt=html,application.html \ + -I/includes \ + /includes/deephaven/proto/application.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-html \ + --doc_opt=html,inputtable.html \ + -I/includes \ + /includes/deephaven/proto/inputtable.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-html \ + --doc_opt=html,partitionedtable.html \ + -I/includes \ + /includes/deephaven/proto/partitionedtable.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-html \ + --doc_opt=html,config.html \ + -I/includes \ + /includes/deephaven/proto/config.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-html \ + --doc_opt=html,hierarchicaltable.html \ + -I/includes \ + /includes/deephaven/proto/hierarchicaltable.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-html \ + --doc_opt=html,storage.html \ + -I/includes \ + /includes/deephaven/proto/storage.proto; \ + # finally, multi md \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-md \ + --doc_opt=markdown,ticket.md \ + -I/includes \ + /includes/deephaven/proto/ticket.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-md \ + --doc_opt=markdown,console.md \ + -I/includes \ + /includes/deephaven/proto/console.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-md \ + --doc_opt=markdown,object.md \ + -I/includes \ + /includes/deephaven/proto/object.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-md \ + --doc_opt=markdown,session.md \ + -I/includes \ + /includes/deephaven/proto/session.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-md \ + --doc_opt=markdown,table.md \ + -I/includes \ + /includes/deephaven/proto/table.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-md \ + --doc_opt=markdown,application.md \ + -I/includes \ + /includes/deephaven/proto/application.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-md \ + --doc_opt=markdown,inputtable.md \ + -I/includes \ + /includes/deephaven/proto/inputtable.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-md \ + --doc_opt=markdown,partitionedtable.md \ + -I/includes \ + /includes/deephaven/proto/partitionedtable.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-md \ + --doc_opt=markdown,config.md \ + -I/includes \ + /includes/deephaven/proto/config.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-md \ + --doc_opt=markdown,hierarchicaltable.md \ + -I/includes \ + /includes/deephaven/proto/hierarchicaltable.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-md \ + --doc_opt=markdown,storage.md \ + -I/includes \ + /includes/deephaven/proto/storage.proto; \ diff --git a/proto/proto-backplane-grpc/build.gradle b/proto/proto-backplane-grpc/build.gradle index 7d1e68d7653..c14dd435496 100644 --- a/proto/proto-backplane-grpc/build.gradle +++ b/proto/proto-backplane-grpc/build.gradle @@ -18,6 +18,7 @@ configurations { python {} go {} cpp {} + protoDocs {} } dependencies { @@ -88,6 +89,9 @@ artifacts { cpp(layout.buildDirectory.dir('generated/source/proto/main/cpp')) { builtBy generateProtobuf } + protoDocs(layout.buildDirectory.dir('generated/source/proto/main/proto-doc')) { + builtBy generateProtobuf + } } diff --git a/proto/raw-js-openapi/flight_format/Message.fbs b/proto/raw-js-openapi/flight_format/Message.fbs deleted file mode 100644 index b93c9e991e9..00000000000 --- a/proto/raw-js-openapi/flight_format/Message.fbs +++ /dev/null @@ -1,138 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -include "Schema.fbs"; - -namespace org.apache.arrow.flatbuf; - -/// ---------------------------------------------------------------------- -/// Data structures for describing a table row batch (a collection of -/// equal-length Arrow arrays) - -/// Metadata about a field at some level of a nested type tree (but not -/// its children). -/// -/// For example, a List with values `[[1, 2, 3], null, [4], [5, 6], null]` -/// would have {length: 5, null_count: 2} for its List node, and {length: 6, -/// null_count: 0} for its Int16 node, as separate FieldNode structs -struct FieldNode { - /// The number of value slots in the Arrow array at this level of a nested - /// tree - length: long; - - /// The number of observed nulls. Fields with null_count == 0 may choose not - /// to write their physical validity bitmap out as a materialized buffer, - /// instead setting the length of the bitmap buffer to 0. - null_count: long; -} - -enum CompressionType:byte { - // LZ4 frame format, for portability, as provided by lz4frame.h or wrappers - // thereof. Not to be confused with "raw" (also called "block") format - // provided by lz4.h - LZ4_FRAME, - - // Zstandard - ZSTD -} - -/// Provided for forward compatibility in case we need to support different -/// strategies for compressing the IPC message body (like whole-body -/// compression rather than buffer-level) in the future -enum BodyCompressionMethod:byte { - /// Each constituent buffer is first compressed with the indicated - /// compressor, and then written with the uncompressed length in the first 8 - /// bytes as a 64-bit little-endian signed integer followed by the compressed - /// buffer bytes (and then padding as required by the protocol). The - /// uncompressed length may be set to -1 to indicate that the data that - /// follows is not compressed, which can be useful for cases where - /// compression does not yield appreciable savings. - BUFFER -} - -/// Optional compression for the memory buffers constituting IPC message -/// bodies. Intended for use with RecordBatch but could be used for other -/// message types -table BodyCompression { - /// Compressor library - codec: CompressionType = LZ4_FRAME; - - /// Indicates the way the record batch body was compressed - method: BodyCompressionMethod = BUFFER; -} - -/// A data header describing the shared memory layout of a "record" or "row" -/// batch. Some systems call this a "row batch" internally and others a "record -/// batch". -table RecordBatch { - /// number of records / rows. The arrays in the batch should all have this - /// length - length: long; - - /// Nodes correspond to the pre-ordered flattened logical schema - nodes: [FieldNode]; - - /// Buffers correspond to the pre-ordered flattened buffer tree - /// - /// The number of buffers appended to this list depends on the schema. For - /// example, most primitive arrays will have 2 buffers, 1 for the validity - /// bitmap and 1 for the values. For struct arrays, there will only be a - /// single buffer for the validity (nulls) bitmap - buffers: [Buffer]; - - /// Optional compression of the message body - compression: BodyCompression; -} - -/// For sending dictionary encoding information. Any Field can be -/// dictionary-encoded, but in this case none of its children may be -/// dictionary-encoded. -/// There is one vector / column per dictionary, but that vector / column -/// may be spread across multiple dictionary batches by using the isDelta -/// flag - -table DictionaryBatch { - id: long; - data: RecordBatch; - - /// If isDelta is true the values in the dictionary are to be appended to a - /// dictionary with the indicated id. If isDelta is false this dictionary - /// should replace the existing dictionary. - isDelta: bool = false; -} - -/// ---------------------------------------------------------------------- -/// The root Message type - -/// This union enables us to easily send different message types without -/// redundant storage, and in the future we can easily add new message types. -/// -/// Arrow implementations do not need to implement all of the message types, -/// which may include experimental metadata types. For maximum compatibility, -/// it is best to send data using RecordBatch -union MessageHeader { - Schema, DictionaryBatch, RecordBatch -} - -table Message { - version: org.apache.arrow.flatbuf.MetadataVersion; - header: MessageHeader; - bodyLength: long; - custom_metadata: [ KeyValue ]; -} - -root_type Message; diff --git a/proto/raw-js-openapi/flight_format/README.md b/proto/raw-js-openapi/flight_format/README.md deleted file mode 100644 index 52d45ad2b7e..00000000000 --- a/proto/raw-js-openapi/flight_format/README.md +++ /dev/null @@ -1,3 +0,0 @@ -Currently we are generating flatbuffer files manually. See [deephaven-core/#1052](https://github.com/deephaven/deephaven-core/issues/1052) to track the work to automate this. - -${FLATC} --ts --no-fb-import --no-ts-reexport -o src/arrow/flight/flatbuf/ flight_format/\*.fbs diff --git a/proto/raw-js-openapi/flight_format/Schema.fbs b/proto/raw-js-openapi/flight_format/Schema.fbs deleted file mode 100644 index 2d447d30791..00000000000 --- a/proto/raw-js-openapi/flight_format/Schema.fbs +++ /dev/null @@ -1,430 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -/// Logical types, vector layouts, and schemas - -namespace org.apache.arrow.flatbuf; - -enum MetadataVersion:short { - /// 0.1.0 (October 2016). - V1, - - /// 0.2.0 (February 2017). Non-backwards compatible with V1. - V2, - - /// 0.3.0 -> 0.7.1 (May - December 2017). Non-backwards compatible with V2. - V3, - - /// >= 0.8.0 (December 2017). Non-backwards compatible with V3. - V4, - - /// >= 1.0.0 (July 2020. Backwards compatible with V4 (V5 readers can read V4 - /// metadata and IPC messages). Implementations are recommended to provide a - /// V4 compatibility mode with V5 format changes disabled. - /// - /// Incompatible changes between V4 and V5: - /// - Union buffer layout has changed. In V5, Unions don't have a validity - /// bitmap buffer. - V5, -} - -/// Represents Arrow Features that might not have full support -/// within implementations. This is intended to be used in -/// two scenarios: -/// 1. A mechanism for readers of Arrow Streams -/// and files to understand that the stream or file makes -/// use of a feature that isn't supported or unknown to -/// the implementation (and therefore can meet the Arrow -/// forward compatibility guarantees). -/// 2. A means of negotiating between a client and server -/// what features a stream is allowed to use. The enums -/// values here are intented to represent higher level -/// features, additional details maybe negotiated -/// with key-value pairs specific to the protocol. -/// -/// Enums added to this list should be assigned power-of-two values -/// to facilitate exchanging and comparing bitmaps for supported -/// features. -enum Feature : long { - /// Needed to make flatbuffers happy. - UNUSED = 0, - /// The stream makes use of multiple full dictionaries with the - /// same ID and assumes clients implement dictionary replacement - /// correctly. - DICTIONARY_REPLACEMENT = 1, - /// The stream makes use of compressed bodies as described - /// in Message.fbs. - COMPRESSED_BODY = 2 -} - -/// These are stored in the flatbuffer in the Type union below - -table Null { -} - -/// A Struct_ in the flatbuffer metadata is the same as an Arrow Struct -/// (according to the physical memory layout). We used Struct_ here as -/// Struct is a reserved word in Flatbuffers -table Struct_ { -} - -table List { -} - -/// Same as List, but with 64-bit offsets, allowing to represent -/// extremely large data values. -table LargeList { -} - -table FixedSizeList { - /// Number of list items per value - listSize: int; -} - -/// A Map is a logical nested type that is represented as -/// -/// List> -/// -/// In this layout, the keys and values are each respectively contiguous. We do -/// not constrain the key and value types, so the application is responsible -/// for ensuring that the keys are hashable and unique. Whether the keys are sorted -/// may be set in the metadata for this field. -/// -/// In a field with Map type, the field has a child Struct field, which then -/// has two children: key type and the second the value type. The names of the -/// child fields may be respectively "entries", "key", and "value", but this is -/// not enforced. -/// -/// Map -/// ```text -/// - child[0] entries: Struct -/// - child[0] key: K -/// - child[1] value: V -/// ``` -/// Neither the "entries" field nor the "key" field may be nullable. -/// -/// The metadata is structured so that Arrow systems without special handling -/// for Map can make Map an alias for List. The "layout" attribute for the Map -/// field must have the same contents as a List. -table Map { - /// Set to true if the keys within each value are sorted - keysSorted: bool; -} - -enum UnionMode:short { Sparse, Dense } - -/// A union is a complex type with children in Field -/// By default ids in the type vector refer to the offsets in the children -/// optionally typeIds provides an indirection between the child offset and the type id -/// for each child `typeIds[offset]` is the id used in the type vector -table Union { - mode: UnionMode; - typeIds: [ int ]; // optional, describes typeid of each child. -} - -table Int { - bitWidth: int; // restricted to 8, 16, 32, and 64 in v1 - is_signed: bool; -} - -enum Precision:short {HALF, SINGLE, DOUBLE} - -table FloatingPoint { - precision: Precision; -} - -/// Unicode with UTF-8 encoding -table Utf8 { -} - -/// Opaque binary data -table Binary { -} - -/// Same as Utf8, but with 64-bit offsets, allowing to represent -/// extremely large data values. -table LargeUtf8 { -} - -/// Same as Binary, but with 64-bit offsets, allowing to represent -/// extremely large data values. -table LargeBinary { -} - -table FixedSizeBinary { - /// Number of bytes per value - byteWidth: int; -} - -table Bool { -} - -/// Exact decimal value represented as an integer value in two's -/// complement. Currently only 128-bit (16-byte) and 256-bit (32-byte) integers -/// are used. The representation uses the endianness indicated -/// in the Schema. -table Decimal { - /// Total number of decimal digits - precision: int; - - /// Number of digits after the decimal point "." - scale: int; - - /// Number of bits per value. The only accepted widths are 128 and 256. - /// We use bitWidth for consistency with Int::bitWidth. - bitWidth: int = 128; -} - -enum DateUnit: short { - DAY, - MILLISECOND -} - -/// Date is either a 32-bit or 64-bit type representing elapsed time since UNIX -/// epoch (1970-01-01), stored in either of two units: -/// -/// * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no -/// leap seconds), where the values are evenly divisible by 86400000 -/// * Days (32 bits) since the UNIX epoch -table Date { - unit: DateUnit = MILLISECOND; -} - -enum TimeUnit: short { SECOND, MILLISECOND, MICROSECOND, NANOSECOND } - -/// Time type. The physical storage type depends on the unit -/// - SECOND and MILLISECOND: 32 bits -/// - MICROSECOND and NANOSECOND: 64 bits -table Time { - unit: TimeUnit = MILLISECOND; - bitWidth: int = 32; -} - -/// Time elapsed from the Unix epoch, 00:00:00.000 on 1 January 1970, excluding -/// leap seconds, as a 64-bit integer. Note that UNIX time does not include -/// leap seconds. -/// -/// Date & time libraries often have multiple different data types for temporal -/// data. In order to ease interoperability between different implementations the -/// Arrow project has some recommendations for encoding these types into a Timestamp -/// column. -/// -/// An "instant" represents a single moment in time that has no meaningful time zone -/// or the time zone is unknown. A column of instants can also contain values from -/// multiple time zones. To encode an instant set the timezone string to "UTC". -/// -/// A "zoned date-time" represents a single moment in time that has a meaningful -/// reference time zone. To encode a zoned date-time as a Timestamp set the timezone -/// string to the name of the timezone. There is some ambiguity between an instant -/// and a zoned date-time with the UTC time zone. Both of these are stored the same. -/// Typically, this distinction does not matter. If it does, then an application should -/// use custom metadata or an extension type to distinguish between the two cases. -/// -/// An "offset date-time" represents a single moment in time combined with a meaningful -/// offset from UTC. To encode an offset date-time as a Timestamp set the timezone string -/// to the numeric time zone offset string (e.g. "+03:00"). -/// -/// A "local date-time" does not represent a single moment in time. It represents a wall -/// clock time combined with a date. Because of daylight savings time there may multiple -/// instants that correspond to a single local date-time in any given time zone. A -/// local date-time is often stored as a struct or a Date32/Time64 pair. However, it can -/// also be encoded into a Timestamp column. To do so the value should be the the time -/// elapsed from the Unix epoch so that a wall clock in UTC would display the desired time. -/// The timezone string should be set to null or the empty string. -table Timestamp { - unit: TimeUnit; - - /// The time zone is a string indicating the name of a time zone, one of: - /// - /// * As used in the Olson time zone database (the "tz database" or - /// "tzdata"), such as "America/New_York" - /// * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30 - /// - /// Whether a timezone string is present indicates different semantics about - /// the data: - /// - /// * If the time zone is null or an empty string, the data is a local date-time - /// and does not represent a single moment in time. Instead it represents a wall clock - /// time and care should be taken to avoid interpreting it semantically as an instant. - /// - /// * If the time zone is set to a valid value, values can be displayed as - /// "localized" to that time zone, even though the underlying 64-bit - /// integers are identical to the same data stored in UTC. Converting - /// between time zones is a metadata-only operation and does not change the - /// underlying values - timezone: string; -} - -enum IntervalUnit: short { YEAR_MONTH, DAY_TIME} -// A "calendar" interval which models types that don't necessarily -// have a precise duration without the context of a base timestamp (e.g. -// days can differ in length during day light savings time transitions). -// YEAR_MONTH - Indicates the number of elapsed whole months, stored as -// 4-byte integers. -// DAY_TIME - Indicates the number of elapsed days and milliseconds, -// stored as 2 contiguous 32-bit integers (8-bytes in total). Support -// of this IntervalUnit is not required for full arrow compatibility. -table Interval { - unit: IntervalUnit; -} - -// An absolute length of time unrelated to any calendar artifacts. -// -// For the purposes of Arrow Implementations, adding this value to a Timestamp -// ("t1") naively (i.e. simply summing the two number) is acceptable even -// though in some cases the resulting Timestamp (t2) would not account for -// leap-seconds during the elapsed time between "t1" and "t2". Similarly, -// representing the difference between two Unix timestamp is acceptable, but -// would yield a value that is possibly a few seconds off from the true elapsed -// time. -// -// The resolution defaults to millisecond, but can be any of the other -// supported TimeUnit values as with Timestamp and Time types. This type is -// always represented as an 8-byte integer. -table Duration { - unit: TimeUnit = MILLISECOND; -} - -/// ---------------------------------------------------------------------- -/// Top-level Type value, enabling extensible type-specific metadata. We can -/// add new logical types to Type without breaking backwards compatibility - -union Type { - Null, - Int, - FloatingPoint, - Binary, - Utf8, - Bool, - Decimal, - Date, - Time, - Timestamp, - Interval, - List, - Struct_, - Union, - FixedSizeBinary, - FixedSizeList, - Map, - Duration, - LargeBinary, - LargeUtf8, - LargeList, -} - -/// ---------------------------------------------------------------------- -/// user defined key value pairs to add custom metadata to arrow -/// key namespacing is the responsibility of the user - -table KeyValue { - key: string; - value: string; -} - -/// ---------------------------------------------------------------------- -/// Dictionary encoding metadata -/// Maintained for forwards compatibility, in the future -/// Dictionaries might be explicit maps between integers and values -/// allowing for non-contiguous index values -enum DictionaryKind : short { DenseArray } -table DictionaryEncoding { - /// The known dictionary id in the application where this data is used. In - /// the file or streaming formats, the dictionary ids are found in the - /// DictionaryBatch messages - id: long; - - /// The dictionary indices are constrained to be non-negative integers. If - /// this field is null, the indices must be signed int32. To maximize - /// cross-language compatibility and performance, implementations are - /// recommended to prefer signed integer types over unsigned integer types - /// and to avoid uint64 indices unless they are required by an application. - indexType: Int; - - /// By default, dictionaries are not ordered, or the order does not have - /// semantic meaning. In some statistical, applications, dictionary-encoding - /// is used to represent ordered categorical data, and we provide a way to - /// preserve that metadata here - isOrdered: bool; - - dictionaryKind: DictionaryKind; -} - -/// ---------------------------------------------------------------------- -/// A field represents a named column in a record / row batch or child of a -/// nested type. - -table Field { - /// Name is not required, in i.e. a List - name: string; - - /// Whether or not this field can contain nulls. Should be true in general. - nullable: bool; - - /// This is the type of the decoded value if the field is dictionary encoded. - type: Type; - - /// Present only if the field is dictionary encoded. - dictionary: DictionaryEncoding; - - /// children apply only to nested data types like Struct, List and Union. For - /// primitive types children will have length 0. - children: [ Field ]; - - /// User-defined metadata - custom_metadata: [ KeyValue ]; -} - -/// ---------------------------------------------------------------------- -/// Endianness of the platform producing the data - -enum Endianness:short { Little, Big } - -/// ---------------------------------------------------------------------- -/// A Buffer represents a single contiguous memory segment -struct Buffer { - /// The relative offset into the shared memory page where the bytes for this - /// buffer starts - offset: long; - - /// The absolute length (in bytes) of the memory buffer. The memory is found - /// from offset (inclusive) to offset + length (non-inclusive). When building - /// messages using the encapsulated IPC message, padding bytes may be written - /// after a buffer, but such padding bytes do not need to be accounted for in - /// the size here. - length: long; -} - -/// ---------------------------------------------------------------------- -/// A Schema describes the columns in a row batch - -table Schema { - - /// endianness of the buffer - /// it is Little Endian by default - /// if endianness doesn't match the underlying system then the vectors need to be converted - endianness: Endianness=Little; - - fields: [Field]; - // User-defined metadata - custom_metadata: [ KeyValue ]; - - /// Features used in the stream/file. - features : [ Feature ]; -} - -root_type Schema; diff --git a/proto/raw-js-openapi/package.json b/proto/raw-js-openapi/package.json index 455c3d67837..fdb83f4ec86 100644 --- a/proto/raw-js-openapi/package.json +++ b/proto/raw-js-openapi/package.json @@ -1,9 +1,6 @@ { "dependencies": { - "@deephaven/barrage": "0.5.0", "@improbable-eng/grpc-web": "^0.14.0", - "apache-arrow": "7.0.0", - "flatbuffers": "1.12.0", "google-protobuf": "^3.20.1" }, "devDependencies": { diff --git a/proto/raw-js-openapi/src/arrow/flight/flatbuf/Message_generated.ts b/proto/raw-js-openapi/src/arrow/flight/flatbuf/Message_generated.ts deleted file mode 100644 index 8d86b6aa04f..00000000000 --- a/proto/raw-js-openapi/src/arrow/flight/flatbuf/Message_generated.ts +++ /dev/null @@ -1,741 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -import * as NS17716817176095924048 from "./Schema_generated"; -/** - * @enum {number} - */ -export namespace org.apache.arrow.flatbuf{ -export enum CompressionType{ - LZ4_FRAME= 0, - ZSTD= 1 -}; -} - -/** - * Provided for forward compatibility in case we need to support different - * strategies for compressing the IPC message body (like whole-body - * compression rather than buffer-level) in the future - * - * @enum {number} - */ -export namespace org.apache.arrow.flatbuf{ -export enum BodyCompressionMethod{ - /** - * Each constituent buffer is first compressed with the indicated - * compressor, and then written with the uncompressed length in the first 8 - * bytes as a 64-bit little-endian signed integer followed by the compressed - * buffer bytes (and then padding as required by the protocol). The - * uncompressed length may be set to -1 to indicate that the data that - * follows is not compressed, which can be useful for cases where - * compression does not yield appreciable savings. - */ - BUFFER= 0 -}; -} - -/** - * ---------------------------------------------------------------------- - * The root Message type - * This union enables us to easily send different message types without - * redundant storage, and in the future we can easily add new message types. - * - * Arrow implementations do not need to implement all of the message types, - * which may include experimental metadata types. For maximum compatibility, - * it is best to send data using RecordBatch - * - * @enum {number} - */ -export namespace org.apache.arrow.flatbuf{ -export enum MessageHeader{ - NONE= 0, - Schema= 1, - DictionaryBatch= 2, - RecordBatch= 3 -}; - -export function unionToMessageHeader( - type: MessageHeader, - accessor: (obj:NS17716817176095924048.org.apache.arrow.flatbuf.Schema|org.apache.arrow.flatbuf.DictionaryBatch|org.apache.arrow.flatbuf.RecordBatch) => NS17716817176095924048.org.apache.arrow.flatbuf.Schema|org.apache.arrow.flatbuf.DictionaryBatch|org.apache.arrow.flatbuf.RecordBatch|null -): NS17716817176095924048.org.apache.arrow.flatbuf.Schema|org.apache.arrow.flatbuf.DictionaryBatch|org.apache.arrow.flatbuf.RecordBatch|null { - switch(org.apache.arrow.flatbuf.MessageHeader[type]) { - case 'NONE': return null; - case 'Schema': return accessor(new NS17716817176095924048.org.apache.arrow.flatbuf.Schema())! as NS17716817176095924048.org.apache.arrow.flatbuf.Schema; - case 'DictionaryBatch': return accessor(new org.apache.arrow.flatbuf.DictionaryBatch())! as org.apache.arrow.flatbuf.DictionaryBatch; - case 'RecordBatch': return accessor(new org.apache.arrow.flatbuf.RecordBatch())! as org.apache.arrow.flatbuf.RecordBatch; - default: return null; - } -} - -export function unionListToMessageHeader( - type: MessageHeader, - accessor: (index: number, obj:NS17716817176095924048.org.apache.arrow.flatbuf.Schema|org.apache.arrow.flatbuf.DictionaryBatch|org.apache.arrow.flatbuf.RecordBatch) => NS17716817176095924048.org.apache.arrow.flatbuf.Schema|org.apache.arrow.flatbuf.DictionaryBatch|org.apache.arrow.flatbuf.RecordBatch|null, - index: number -): NS17716817176095924048.org.apache.arrow.flatbuf.Schema|org.apache.arrow.flatbuf.DictionaryBatch|org.apache.arrow.flatbuf.RecordBatch|null { - switch(org.apache.arrow.flatbuf.MessageHeader[type]) { - case 'NONE': return null; - case 'Schema': return accessor(index, new NS17716817176095924048.org.apache.arrow.flatbuf.Schema())! as NS17716817176095924048.org.apache.arrow.flatbuf.Schema; - case 'DictionaryBatch': return accessor(index, new org.apache.arrow.flatbuf.DictionaryBatch())! as org.apache.arrow.flatbuf.DictionaryBatch; - case 'RecordBatch': return accessor(index, new org.apache.arrow.flatbuf.RecordBatch())! as org.apache.arrow.flatbuf.RecordBatch; - default: return null; - } -} -} - -/** - * ---------------------------------------------------------------------- - * Data structures for describing a table row batch (a collection of - * equal-length Arrow arrays) - * Metadata about a field at some level of a nested type tree (but not - * its children). - * - * For example, a List with values `[[1, 2, 3], null, [4], [5, 6], null]` - * would have {length: 5, null_count: 2} for its List node, and {length: 6, - * null_count: 0} for its Int16 node, as separate FieldNode structs - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class FieldNode { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns FieldNode - */ -__init(i:number, bb:flatbuffers.ByteBuffer):FieldNode { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * The number of value slots in the Arrow array at this level of a nested - * tree - * - * @returns flatbuffers.Long - */ -length():flatbuffers.Long { - return this.bb!.readInt64(this.bb_pos); -}; - -/** - * The number of observed nulls. Fields with null_count == 0 may choose not - * to write their physical validity bitmap out as a materialized buffer, - * instead setting the length of the bitmap buffer to 0. - * - * @returns flatbuffers.Long - */ -nullCount():flatbuffers.Long { - return this.bb!.readInt64(this.bb_pos + 8); -}; - -/** - * @returns number - */ -static sizeOf():number { - return 16; -} - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Long length - * @param flatbuffers.Long null_count - * @returns flatbuffers.Offset - */ -static createFieldNode(builder:flatbuffers.Builder, length: flatbuffers.Long, null_count: flatbuffers.Long):flatbuffers.Offset { - builder.prep(8, 16); - builder.writeInt64(null_count); - builder.writeInt64(length); - return builder.offset(); -}; - -} -} -/** - * Optional compression for the memory buffers constituting IPC message - * bodies. Intended for use with RecordBatch but could be used for other - * message types - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class BodyCompression { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns BodyCompression - */ -__init(i:number, bb:flatbuffers.ByteBuffer):BodyCompression { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param BodyCompression= obj - * @returns BodyCompression - */ -static getRootAsBodyCompression(bb:flatbuffers.ByteBuffer, obj?:BodyCompression):BodyCompression { - return (obj || new BodyCompression()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param BodyCompression= obj - * @returns BodyCompression - */ -static getSizePrefixedRootAsBodyCompression(bb:flatbuffers.ByteBuffer, obj?:BodyCompression):BodyCompression { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new BodyCompression()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * Compressor library - * - * @returns org.apache.arrow.flatbuf.CompressionType - */ -codec():org.apache.arrow.flatbuf.CompressionType { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? /** */ (this.bb!.readInt8(this.bb_pos + offset)) : org.apache.arrow.flatbuf.CompressionType.LZ4_FRAME; -}; - -/** - * Indicates the way the record batch body was compressed - * - * @returns org.apache.arrow.flatbuf.BodyCompressionMethod - */ -method():org.apache.arrow.flatbuf.BodyCompressionMethod { - var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? /** */ (this.bb!.readInt8(this.bb_pos + offset)) : org.apache.arrow.flatbuf.BodyCompressionMethod.BUFFER; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startBodyCompression(builder:flatbuffers.Builder) { - builder.startObject(2); -}; - -/** - * @param flatbuffers.Builder builder - * @param org.apache.arrow.flatbuf.CompressionType codec - */ -static addCodec(builder:flatbuffers.Builder, codec:org.apache.arrow.flatbuf.CompressionType) { - builder.addFieldInt8(0, codec, org.apache.arrow.flatbuf.CompressionType.LZ4_FRAME); -}; - -/** - * @param flatbuffers.Builder builder - * @param org.apache.arrow.flatbuf.BodyCompressionMethod method - */ -static addMethod(builder:flatbuffers.Builder, method:org.apache.arrow.flatbuf.BodyCompressionMethod) { - builder.addFieldInt8(1, method, org.apache.arrow.flatbuf.BodyCompressionMethod.BUFFER); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endBodyCompression(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createBodyCompression(builder:flatbuffers.Builder, codec:org.apache.arrow.flatbuf.CompressionType, method:org.apache.arrow.flatbuf.BodyCompressionMethod):flatbuffers.Offset { - BodyCompression.startBodyCompression(builder); - BodyCompression.addCodec(builder, codec); - BodyCompression.addMethod(builder, method); - return BodyCompression.endBodyCompression(builder); -} -} -} -/** - * A data header describing the shared memory layout of a "record" or "row" - * batch. Some systems call this a "row batch" internally and others a "record - * batch". - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class RecordBatch { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns RecordBatch - */ -__init(i:number, bb:flatbuffers.ByteBuffer):RecordBatch { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param RecordBatch= obj - * @returns RecordBatch - */ -static getRootAsRecordBatch(bb:flatbuffers.ByteBuffer, obj?:RecordBatch):RecordBatch { - return (obj || new RecordBatch()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param RecordBatch= obj - * @returns RecordBatch - */ -static getSizePrefixedRootAsRecordBatch(bb:flatbuffers.ByteBuffer, obj?:RecordBatch):RecordBatch { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new RecordBatch()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * number of records / rows. The arrays in the batch should all have this - * length - * - * @returns flatbuffers.Long - */ -length():flatbuffers.Long { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? this.bb!.readInt64(this.bb_pos + offset) : this.bb!.createLong(0, 0); -}; - -/** - * Nodes correspond to the pre-ordered flattened logical schema - * - * @param number index - * @param org.apache.arrow.flatbuf.FieldNode= obj - * @returns org.apache.arrow.flatbuf.FieldNode - */ -nodes(index: number, obj?:org.apache.arrow.flatbuf.FieldNode):org.apache.arrow.flatbuf.FieldNode|null { - var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? (obj || new org.apache.arrow.flatbuf.FieldNode()).__init(this.bb!.__vector(this.bb_pos + offset) + index * 16, this.bb!) : null; -}; - -/** - * @returns number - */ -nodesLength():number { - var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; -}; - -/** - * Buffers correspond to the pre-ordered flattened buffer tree - * - * The number of buffers appended to this list depends on the schema. For - * example, most primitive arrays will have 2 buffers, 1 for the validity - * bitmap and 1 for the values. For struct arrays, there will only be a - * single buffer for the validity (nulls) bitmap - * - * @param number index - * @param org.apache.arrow.flatbuf.Buffer= obj - * @returns org.apache.arrow.flatbuf.Buffer - */ -buffers(index: number, obj?:NS17716817176095924048.org.apache.arrow.flatbuf.Buffer):NS17716817176095924048.org.apache.arrow.flatbuf.Buffer|null { - var offset = this.bb!.__offset(this.bb_pos, 8); - return offset ? (obj || new NS17716817176095924048.org.apache.arrow.flatbuf.Buffer()).__init(this.bb!.__vector(this.bb_pos + offset) + index * 16, this.bb!) : null; -}; - -/** - * @returns number - */ -buffersLength():number { - var offset = this.bb!.__offset(this.bb_pos, 8); - return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; -}; - -/** - * Optional compression of the message body - * - * @param org.apache.arrow.flatbuf.BodyCompression= obj - * @returns org.apache.arrow.flatbuf.BodyCompression|null - */ -compression(obj?:org.apache.arrow.flatbuf.BodyCompression):org.apache.arrow.flatbuf.BodyCompression|null { - var offset = this.bb!.__offset(this.bb_pos, 10); - return offset ? (obj || new org.apache.arrow.flatbuf.BodyCompression()).__init(this.bb!.__indirect(this.bb_pos + offset), this.bb!) : null; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startRecordBatch(builder:flatbuffers.Builder) { - builder.startObject(4); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Long length - */ -static addLength(builder:flatbuffers.Builder, length:flatbuffers.Long) { - builder.addFieldInt64(0, length, builder.createLong(0, 0)); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset nodesOffset - */ -static addNodes(builder:flatbuffers.Builder, nodesOffset:flatbuffers.Offset) { - builder.addFieldOffset(1, nodesOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param number numElems - */ -static startNodesVector(builder:flatbuffers.Builder, numElems:number) { - builder.startVector(16, numElems, 8); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset buffersOffset - */ -static addBuffers(builder:flatbuffers.Builder, buffersOffset:flatbuffers.Offset) { - builder.addFieldOffset(2, buffersOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param number numElems - */ -static startBuffersVector(builder:flatbuffers.Builder, numElems:number) { - builder.startVector(16, numElems, 8); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset compressionOffset - */ -static addCompression(builder:flatbuffers.Builder, compressionOffset:flatbuffers.Offset) { - builder.addFieldOffset(3, compressionOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endRecordBatch(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -} -} -/** - * For sending dictionary encoding information. Any Field can be - * dictionary-encoded, but in this case none of its children may be - * dictionary-encoded. - * There is one vector / column per dictionary, but that vector / column - * may be spread across multiple dictionary batches by using the isDelta - * flag - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class DictionaryBatch { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns DictionaryBatch - */ -__init(i:number, bb:flatbuffers.ByteBuffer):DictionaryBatch { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param DictionaryBatch= obj - * @returns DictionaryBatch - */ -static getRootAsDictionaryBatch(bb:flatbuffers.ByteBuffer, obj?:DictionaryBatch):DictionaryBatch { - return (obj || new DictionaryBatch()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param DictionaryBatch= obj - * @returns DictionaryBatch - */ -static getSizePrefixedRootAsDictionaryBatch(bb:flatbuffers.ByteBuffer, obj?:DictionaryBatch):DictionaryBatch { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new DictionaryBatch()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @returns flatbuffers.Long - */ -id():flatbuffers.Long { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? this.bb!.readInt64(this.bb_pos + offset) : this.bb!.createLong(0, 0); -}; - -/** - * @param org.apache.arrow.flatbuf.RecordBatch= obj - * @returns org.apache.arrow.flatbuf.RecordBatch|null - */ -data(obj?:org.apache.arrow.flatbuf.RecordBatch):org.apache.arrow.flatbuf.RecordBatch|null { - var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? (obj || new org.apache.arrow.flatbuf.RecordBatch()).__init(this.bb!.__indirect(this.bb_pos + offset), this.bb!) : null; -}; - -/** - * If isDelta is true the values in the dictionary are to be appended to a - * dictionary with the indicated id. If isDelta is false this dictionary - * should replace the existing dictionary. - * - * @returns boolean - */ -isDelta():boolean { - var offset = this.bb!.__offset(this.bb_pos, 8); - return offset ? !!this.bb!.readInt8(this.bb_pos + offset) : false; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startDictionaryBatch(builder:flatbuffers.Builder) { - builder.startObject(3); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Long id - */ -static addId(builder:flatbuffers.Builder, id:flatbuffers.Long) { - builder.addFieldInt64(0, id, builder.createLong(0, 0)); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset dataOffset - */ -static addData(builder:flatbuffers.Builder, dataOffset:flatbuffers.Offset) { - builder.addFieldOffset(1, dataOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param boolean isDelta - */ -static addIsDelta(builder:flatbuffers.Builder, isDelta:boolean) { - builder.addFieldInt8(2, +isDelta, +false); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endDictionaryBatch(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -} -} -/** - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Message { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Message - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Message { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Message= obj - * @returns Message - */ -static getRootAsMessage(bb:flatbuffers.ByteBuffer, obj?:Message):Message { - return (obj || new Message()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Message= obj - * @returns Message - */ -static getSizePrefixedRootAsMessage(bb:flatbuffers.ByteBuffer, obj?:Message):Message { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new Message()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @returns org.apache.arrow.flatbuf.MetadataVersion - */ -version():NS17716817176095924048.org.apache.arrow.flatbuf.MetadataVersion { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : NS17716817176095924048.org.apache.arrow.flatbuf.MetadataVersion.V1; -}; - -/** - * @returns org.apache.arrow.flatbuf.MessageHeader - */ -headerType():org.apache.arrow.flatbuf.MessageHeader { - var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? /** */ (this.bb!.readUint8(this.bb_pos + offset)) : org.apache.arrow.flatbuf.MessageHeader.NONE; -}; - -/** - * @param flatbuffers.Table obj - * @returns ?flatbuffers.Table - */ -header(obj:T):T|null { - var offset = this.bb!.__offset(this.bb_pos, 8); - return offset ? this.bb!.__union(obj, this.bb_pos + offset) : null; -}; - -/** - * @returns flatbuffers.Long - */ -bodyLength():flatbuffers.Long { - var offset = this.bb!.__offset(this.bb_pos, 10); - return offset ? this.bb!.readInt64(this.bb_pos + offset) : this.bb!.createLong(0, 0); -}; - -/** - * @param number index - * @param org.apache.arrow.flatbuf.KeyValue= obj - * @returns org.apache.arrow.flatbuf.KeyValue - */ -customMetadata(index: number, obj?:NS17716817176095924048.org.apache.arrow.flatbuf.KeyValue):NS17716817176095924048.org.apache.arrow.flatbuf.KeyValue|null { - var offset = this.bb!.__offset(this.bb_pos, 12); - return offset ? (obj || new NS17716817176095924048.org.apache.arrow.flatbuf.KeyValue()).__init(this.bb!.__indirect(this.bb!.__vector(this.bb_pos + offset) + index * 4), this.bb!) : null; -}; - -/** - * @returns number - */ -customMetadataLength():number { - var offset = this.bb!.__offset(this.bb_pos, 12); - return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startMessage(builder:flatbuffers.Builder) { - builder.startObject(5); -}; - -/** - * @param flatbuffers.Builder builder - * @param org.apache.arrow.flatbuf.MetadataVersion version - */ -static addVersion(builder:flatbuffers.Builder, version:NS17716817176095924048.org.apache.arrow.flatbuf.MetadataVersion) { - builder.addFieldInt16(0, version, NS17716817176095924048.org.apache.arrow.flatbuf.MetadataVersion.V1); -}; - -/** - * @param flatbuffers.Builder builder - * @param org.apache.arrow.flatbuf.MessageHeader headerType - */ -static addHeaderType(builder:flatbuffers.Builder, headerType:org.apache.arrow.flatbuf.MessageHeader) { - builder.addFieldInt8(1, headerType, org.apache.arrow.flatbuf.MessageHeader.NONE); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset headerOffset - */ -static addHeader(builder:flatbuffers.Builder, headerOffset:flatbuffers.Offset) { - builder.addFieldOffset(2, headerOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Long bodyLength - */ -static addBodyLength(builder:flatbuffers.Builder, bodyLength:flatbuffers.Long) { - builder.addFieldInt64(3, bodyLength, builder.createLong(0, 0)); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset customMetadataOffset - */ -static addCustomMetadata(builder:flatbuffers.Builder, customMetadataOffset:flatbuffers.Offset) { - builder.addFieldOffset(4, customMetadataOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param Array. data - * @returns flatbuffers.Offset - */ -static createCustomMetadataVector(builder:flatbuffers.Builder, data:flatbuffers.Offset[]):flatbuffers.Offset { - builder.startVector(4, data.length, 4); - for (var i = data.length - 1; i >= 0; i--) { - builder.addOffset(data[i]); - } - return builder.endVector(); -}; - -/** - * @param flatbuffers.Builder builder - * @param number numElems - */ -static startCustomMetadataVector(builder:flatbuffers.Builder, numElems:number) { - builder.startVector(4, numElems, 4); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endMessage(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset offset - */ -static finishMessageBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) { - builder.finish(offset); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset offset - */ -static finishSizePrefixedMessageBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) { - builder.finish(offset, undefined, true); -}; - -static createMessage(builder:flatbuffers.Builder, version:NS17716817176095924048.org.apache.arrow.flatbuf.MetadataVersion, headerType:org.apache.arrow.flatbuf.MessageHeader, headerOffset:flatbuffers.Offset, bodyLength:flatbuffers.Long, customMetadataOffset:flatbuffers.Offset):flatbuffers.Offset { - Message.startMessage(builder); - Message.addVersion(builder, version); - Message.addHeaderType(builder, headerType); - Message.addHeader(builder, headerOffset); - Message.addBodyLength(builder, bodyLength); - Message.addCustomMetadata(builder, customMetadataOffset); - return Message.endMessage(builder); -} -} -} diff --git a/proto/raw-js-openapi/src/arrow/flight/flatbuf/Schema_generated.ts b/proto/raw-js-openapi/src/arrow/flight/flatbuf/Schema_generated.ts deleted file mode 100644 index 1c485ed3b78..00000000000 --- a/proto/raw-js-openapi/src/arrow/flight/flatbuf/Schema_generated.ts +++ /dev/null @@ -1,2807 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -/** - * @enum {number} - */ -export namespace org.apache.arrow.flatbuf{ -export enum MetadataVersion{ - /** - * 0.1.0 (October 2016). - */ - V1= 0, - - /** - * 0.2.0 (February 2017). Non-backwards compatible with V1. - */ - V2= 1, - - /** - * 0.3.0 -> 0.7.1 (May - December 2017). Non-backwards compatible with V2. - */ - V3= 2, - - /** - * >= 0.8.0 (December 2017). Non-backwards compatible with V3. - */ - V4= 3, - - /** - * >= 1.0.0 (July 2020. Backwards compatible with V4 (V5 readers can read V4 - * metadata and IPC messages). Implementations are recommended to provide a - * V4 compatibility mode with V5 format changes disabled. - * - * Incompatible changes between V4 and V5: - * - Union buffer layout has changed. In V5, Unions don't have a validity - * bitmap buffer. - */ - V5= 4 -}; -} - -/** - * Represents Arrow Features that might not have full support - * within implementations. This is intended to be used in - * two scenarios: - * 1. A mechanism for readers of Arrow Streams - * and files to understand that the stream or file makes - * use of a feature that isn't supported or unknown to - * the implementation (and therefore can meet the Arrow - * forward compatibility guarantees). - * 2. A means of negotiating between a client and server - * what features a stream is allowed to use. The enums - * values here are intented to represent higher level - * features, additional details maybe negotiated - * with key-value pairs specific to the protocol. - * - * Enums added to this list should be assigned power-of-two values - * to facilitate exchanging and comparing bitmaps for supported - * features. - * - * @enum {number} - */ -export namespace org.apache.arrow.flatbuf{ -export enum Feature{ - /** - * Needed to make flatbuffers happy. - */ - UNUSED= 0, - - /** - * The stream makes use of multiple full dictionaries with the - * same ID and assumes clients implement dictionary replacement - * correctly. - */ - DICTIONARY_REPLACEMENT= 1, - - /** - * The stream makes use of compressed bodies as described - * in Message.fbs. - */ - COMPRESSED_BODY= 2 -}; -} - -/** - * @enum {number} - */ -export namespace org.apache.arrow.flatbuf{ -export enum UnionMode{ - Sparse= 0, - Dense= 1 -}; -} - -/** - * @enum {number} - */ -export namespace org.apache.arrow.flatbuf{ -export enum Precision{ - HALF= 0, - SINGLE= 1, - DOUBLE= 2 -}; -} - -/** - * @enum {number} - */ -export namespace org.apache.arrow.flatbuf{ -export enum DateUnit{ - DAY= 0, - MILLISECOND= 1 -}; -} - -/** - * @enum {number} - */ -export namespace org.apache.arrow.flatbuf{ -export enum TimeUnit{ - SECOND= 0, - MILLISECOND= 1, - MICROSECOND= 2, - NANOSECOND= 3 -}; -} - -/** - * @enum {number} - */ -export namespace org.apache.arrow.flatbuf{ -export enum IntervalUnit{ - YEAR_MONTH= 0, - DAY_TIME= 1 -}; -} - -/** - * ---------------------------------------------------------------------- - * Top-level Type value, enabling extensible type-specific metadata. We can - * add new logical types to Type without breaking backwards compatibility - * - * @enum {number} - */ -export namespace org.apache.arrow.flatbuf{ -export enum Type{ - NONE= 0, - Null= 1, - Int= 2, - FloatingPoint= 3, - Binary= 4, - Utf8= 5, - Bool= 6, - Decimal= 7, - Date= 8, - Time= 9, - Timestamp= 10, - Interval= 11, - List= 12, - Struct_= 13, - Union= 14, - FixedSizeBinary= 15, - FixedSizeList= 16, - Map= 17, - Duration= 18, - LargeBinary= 19, - LargeUtf8= 20, - LargeList= 21 -}; - -export function unionToType( - type: Type, - accessor: (obj:org.apache.arrow.flatbuf.Binary|org.apache.arrow.flatbuf.Bool|org.apache.arrow.flatbuf.Date|org.apache.arrow.flatbuf.Decimal|org.apache.arrow.flatbuf.Duration|org.apache.arrow.flatbuf.FixedSizeBinary|org.apache.arrow.flatbuf.FixedSizeList|org.apache.arrow.flatbuf.FloatingPoint|org.apache.arrow.flatbuf.Int|org.apache.arrow.flatbuf.Interval|org.apache.arrow.flatbuf.LargeBinary|org.apache.arrow.flatbuf.LargeList|org.apache.arrow.flatbuf.LargeUtf8|org.apache.arrow.flatbuf.List|org.apache.arrow.flatbuf.Map|org.apache.arrow.flatbuf.Null|org.apache.arrow.flatbuf.Struct_|org.apache.arrow.flatbuf.Time|org.apache.arrow.flatbuf.Timestamp|org.apache.arrow.flatbuf.Union|org.apache.arrow.flatbuf.Utf8) => org.apache.arrow.flatbuf.Binary|org.apache.arrow.flatbuf.Bool|org.apache.arrow.flatbuf.Date|org.apache.arrow.flatbuf.Decimal|org.apache.arrow.flatbuf.Duration|org.apache.arrow.flatbuf.FixedSizeBinary|org.apache.arrow.flatbuf.FixedSizeList|org.apache.arrow.flatbuf.FloatingPoint|org.apache.arrow.flatbuf.Int|org.apache.arrow.flatbuf.Interval|org.apache.arrow.flatbuf.LargeBinary|org.apache.arrow.flatbuf.LargeList|org.apache.arrow.flatbuf.LargeUtf8|org.apache.arrow.flatbuf.List|org.apache.arrow.flatbuf.Map|org.apache.arrow.flatbuf.Null|org.apache.arrow.flatbuf.Struct_|org.apache.arrow.flatbuf.Time|org.apache.arrow.flatbuf.Timestamp|org.apache.arrow.flatbuf.Union|org.apache.arrow.flatbuf.Utf8|null -): org.apache.arrow.flatbuf.Binary|org.apache.arrow.flatbuf.Bool|org.apache.arrow.flatbuf.Date|org.apache.arrow.flatbuf.Decimal|org.apache.arrow.flatbuf.Duration|org.apache.arrow.flatbuf.FixedSizeBinary|org.apache.arrow.flatbuf.FixedSizeList|org.apache.arrow.flatbuf.FloatingPoint|org.apache.arrow.flatbuf.Int|org.apache.arrow.flatbuf.Interval|org.apache.arrow.flatbuf.LargeBinary|org.apache.arrow.flatbuf.LargeList|org.apache.arrow.flatbuf.LargeUtf8|org.apache.arrow.flatbuf.List|org.apache.arrow.flatbuf.Map|org.apache.arrow.flatbuf.Null|org.apache.arrow.flatbuf.Struct_|org.apache.arrow.flatbuf.Time|org.apache.arrow.flatbuf.Timestamp|org.apache.arrow.flatbuf.Union|org.apache.arrow.flatbuf.Utf8|null { - switch(org.apache.arrow.flatbuf.Type[type]) { - case 'NONE': return null; - case 'Null': return accessor(new org.apache.arrow.flatbuf.Null())! as org.apache.arrow.flatbuf.Null; - case 'Int': return accessor(new org.apache.arrow.flatbuf.Int())! as org.apache.arrow.flatbuf.Int; - case 'FloatingPoint': return accessor(new org.apache.arrow.flatbuf.FloatingPoint())! as org.apache.arrow.flatbuf.FloatingPoint; - case 'Binary': return accessor(new org.apache.arrow.flatbuf.Binary())! as org.apache.arrow.flatbuf.Binary; - case 'Utf8': return accessor(new org.apache.arrow.flatbuf.Utf8())! as org.apache.arrow.flatbuf.Utf8; - case 'Bool': return accessor(new org.apache.arrow.flatbuf.Bool())! as org.apache.arrow.flatbuf.Bool; - case 'Decimal': return accessor(new org.apache.arrow.flatbuf.Decimal())! as org.apache.arrow.flatbuf.Decimal; - case 'Date': return accessor(new org.apache.arrow.flatbuf.Date())! as org.apache.arrow.flatbuf.Date; - case 'Time': return accessor(new org.apache.arrow.flatbuf.Time())! as org.apache.arrow.flatbuf.Time; - case 'Timestamp': return accessor(new org.apache.arrow.flatbuf.Timestamp())! as org.apache.arrow.flatbuf.Timestamp; - case 'Interval': return accessor(new org.apache.arrow.flatbuf.Interval())! as org.apache.arrow.flatbuf.Interval; - case 'List': return accessor(new org.apache.arrow.flatbuf.List())! as org.apache.arrow.flatbuf.List; - case 'Struct_': return accessor(new org.apache.arrow.flatbuf.Struct_())! as org.apache.arrow.flatbuf.Struct_; - case 'Union': return accessor(new org.apache.arrow.flatbuf.Union())! as org.apache.arrow.flatbuf.Union; - case 'FixedSizeBinary': return accessor(new org.apache.arrow.flatbuf.FixedSizeBinary())! as org.apache.arrow.flatbuf.FixedSizeBinary; - case 'FixedSizeList': return accessor(new org.apache.arrow.flatbuf.FixedSizeList())! as org.apache.arrow.flatbuf.FixedSizeList; - case 'Map': return accessor(new org.apache.arrow.flatbuf.Map())! as org.apache.arrow.flatbuf.Map; - case 'Duration': return accessor(new org.apache.arrow.flatbuf.Duration())! as org.apache.arrow.flatbuf.Duration; - case 'LargeBinary': return accessor(new org.apache.arrow.flatbuf.LargeBinary())! as org.apache.arrow.flatbuf.LargeBinary; - case 'LargeUtf8': return accessor(new org.apache.arrow.flatbuf.LargeUtf8())! as org.apache.arrow.flatbuf.LargeUtf8; - case 'LargeList': return accessor(new org.apache.arrow.flatbuf.LargeList())! as org.apache.arrow.flatbuf.LargeList; - default: return null; - } -} - -export function unionListToType( - type: Type, - accessor: (index: number, obj:org.apache.arrow.flatbuf.Binary|org.apache.arrow.flatbuf.Bool|org.apache.arrow.flatbuf.Date|org.apache.arrow.flatbuf.Decimal|org.apache.arrow.flatbuf.Duration|org.apache.arrow.flatbuf.FixedSizeBinary|org.apache.arrow.flatbuf.FixedSizeList|org.apache.arrow.flatbuf.FloatingPoint|org.apache.arrow.flatbuf.Int|org.apache.arrow.flatbuf.Interval|org.apache.arrow.flatbuf.LargeBinary|org.apache.arrow.flatbuf.LargeList|org.apache.arrow.flatbuf.LargeUtf8|org.apache.arrow.flatbuf.List|org.apache.arrow.flatbuf.Map|org.apache.arrow.flatbuf.Null|org.apache.arrow.flatbuf.Struct_|org.apache.arrow.flatbuf.Time|org.apache.arrow.flatbuf.Timestamp|org.apache.arrow.flatbuf.Union|org.apache.arrow.flatbuf.Utf8) => org.apache.arrow.flatbuf.Binary|org.apache.arrow.flatbuf.Bool|org.apache.arrow.flatbuf.Date|org.apache.arrow.flatbuf.Decimal|org.apache.arrow.flatbuf.Duration|org.apache.arrow.flatbuf.FixedSizeBinary|org.apache.arrow.flatbuf.FixedSizeList|org.apache.arrow.flatbuf.FloatingPoint|org.apache.arrow.flatbuf.Int|org.apache.arrow.flatbuf.Interval|org.apache.arrow.flatbuf.LargeBinary|org.apache.arrow.flatbuf.LargeList|org.apache.arrow.flatbuf.LargeUtf8|org.apache.arrow.flatbuf.List|org.apache.arrow.flatbuf.Map|org.apache.arrow.flatbuf.Null|org.apache.arrow.flatbuf.Struct_|org.apache.arrow.flatbuf.Time|org.apache.arrow.flatbuf.Timestamp|org.apache.arrow.flatbuf.Union|org.apache.arrow.flatbuf.Utf8|null, - index: number -): org.apache.arrow.flatbuf.Binary|org.apache.arrow.flatbuf.Bool|org.apache.arrow.flatbuf.Date|org.apache.arrow.flatbuf.Decimal|org.apache.arrow.flatbuf.Duration|org.apache.arrow.flatbuf.FixedSizeBinary|org.apache.arrow.flatbuf.FixedSizeList|org.apache.arrow.flatbuf.FloatingPoint|org.apache.arrow.flatbuf.Int|org.apache.arrow.flatbuf.Interval|org.apache.arrow.flatbuf.LargeBinary|org.apache.arrow.flatbuf.LargeList|org.apache.arrow.flatbuf.LargeUtf8|org.apache.arrow.flatbuf.List|org.apache.arrow.flatbuf.Map|org.apache.arrow.flatbuf.Null|org.apache.arrow.flatbuf.Struct_|org.apache.arrow.flatbuf.Time|org.apache.arrow.flatbuf.Timestamp|org.apache.arrow.flatbuf.Union|org.apache.arrow.flatbuf.Utf8|null { - switch(org.apache.arrow.flatbuf.Type[type]) { - case 'NONE': return null; - case 'Null': return accessor(index, new org.apache.arrow.flatbuf.Null())! as org.apache.arrow.flatbuf.Null; - case 'Int': return accessor(index, new org.apache.arrow.flatbuf.Int())! as org.apache.arrow.flatbuf.Int; - case 'FloatingPoint': return accessor(index, new org.apache.arrow.flatbuf.FloatingPoint())! as org.apache.arrow.flatbuf.FloatingPoint; - case 'Binary': return accessor(index, new org.apache.arrow.flatbuf.Binary())! as org.apache.arrow.flatbuf.Binary; - case 'Utf8': return accessor(index, new org.apache.arrow.flatbuf.Utf8())! as org.apache.arrow.flatbuf.Utf8; - case 'Bool': return accessor(index, new org.apache.arrow.flatbuf.Bool())! as org.apache.arrow.flatbuf.Bool; - case 'Decimal': return accessor(index, new org.apache.arrow.flatbuf.Decimal())! as org.apache.arrow.flatbuf.Decimal; - case 'Date': return accessor(index, new org.apache.arrow.flatbuf.Date())! as org.apache.arrow.flatbuf.Date; - case 'Time': return accessor(index, new org.apache.arrow.flatbuf.Time())! as org.apache.arrow.flatbuf.Time; - case 'Timestamp': return accessor(index, new org.apache.arrow.flatbuf.Timestamp())! as org.apache.arrow.flatbuf.Timestamp; - case 'Interval': return accessor(index, new org.apache.arrow.flatbuf.Interval())! as org.apache.arrow.flatbuf.Interval; - case 'List': return accessor(index, new org.apache.arrow.flatbuf.List())! as org.apache.arrow.flatbuf.List; - case 'Struct_': return accessor(index, new org.apache.arrow.flatbuf.Struct_())! as org.apache.arrow.flatbuf.Struct_; - case 'Union': return accessor(index, new org.apache.arrow.flatbuf.Union())! as org.apache.arrow.flatbuf.Union; - case 'FixedSizeBinary': return accessor(index, new org.apache.arrow.flatbuf.FixedSizeBinary())! as org.apache.arrow.flatbuf.FixedSizeBinary; - case 'FixedSizeList': return accessor(index, new org.apache.arrow.flatbuf.FixedSizeList())! as org.apache.arrow.flatbuf.FixedSizeList; - case 'Map': return accessor(index, new org.apache.arrow.flatbuf.Map())! as org.apache.arrow.flatbuf.Map; - case 'Duration': return accessor(index, new org.apache.arrow.flatbuf.Duration())! as org.apache.arrow.flatbuf.Duration; - case 'LargeBinary': return accessor(index, new org.apache.arrow.flatbuf.LargeBinary())! as org.apache.arrow.flatbuf.LargeBinary; - case 'LargeUtf8': return accessor(index, new org.apache.arrow.flatbuf.LargeUtf8())! as org.apache.arrow.flatbuf.LargeUtf8; - case 'LargeList': return accessor(index, new org.apache.arrow.flatbuf.LargeList())! as org.apache.arrow.flatbuf.LargeList; - default: return null; - } -} -} - -/** - * ---------------------------------------------------------------------- - * Dictionary encoding metadata - * Maintained for forwards compatibility, in the future - * Dictionaries might be explicit maps between integers and values - * allowing for non-contiguous index values - * - * @enum {number} - */ -export namespace org.apache.arrow.flatbuf{ -export enum DictionaryKind{ - DenseArray= 0 -}; -} - -/** - * ---------------------------------------------------------------------- - * Endianness of the platform producing the data - * - * @enum {number} - */ -export namespace org.apache.arrow.flatbuf{ -export enum Endianness{ - Little= 0, - Big= 1 -}; -} - -/** - * These are stored in the flatbuffer in the Type union below - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Null { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Null - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Null { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Null= obj - * @returns Null - */ -static getRootAsNull(bb:flatbuffers.ByteBuffer, obj?:Null):Null { - return (obj || new Null()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Null= obj - * @returns Null - */ -static getSizePrefixedRootAsNull(bb:flatbuffers.ByteBuffer, obj?:Null):Null { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new Null()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.Builder builder - */ -static startNull(builder:flatbuffers.Builder) { - builder.startObject(0); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endNull(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createNull(builder:flatbuffers.Builder):flatbuffers.Offset { - Null.startNull(builder); - return Null.endNull(builder); -} -} -} -/** - * A Struct_ in the flatbuffer metadata is the same as an Arrow Struct - * (according to the physical memory layout). We used Struct_ here as - * Struct is a reserved word in Flatbuffers - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Struct_ { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Struct_ - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Struct_ { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Struct_= obj - * @returns Struct_ - */ -static getRootAsStruct_(bb:flatbuffers.ByteBuffer, obj?:Struct_):Struct_ { - return (obj || new Struct_()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Struct_= obj - * @returns Struct_ - */ -static getSizePrefixedRootAsStruct_(bb:flatbuffers.ByteBuffer, obj?:Struct_):Struct_ { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new Struct_()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.Builder builder - */ -static startStruct_(builder:flatbuffers.Builder) { - builder.startObject(0); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endStruct_(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createStruct_(builder:flatbuffers.Builder):flatbuffers.Offset { - Struct_.startStruct_(builder); - return Struct_.endStruct_(builder); -} -} -} -/** - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class List { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns List - */ -__init(i:number, bb:flatbuffers.ByteBuffer):List { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param List= obj - * @returns List - */ -static getRootAsList(bb:flatbuffers.ByteBuffer, obj?:List):List { - return (obj || new List()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param List= obj - * @returns List - */ -static getSizePrefixedRootAsList(bb:flatbuffers.ByteBuffer, obj?:List):List { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new List()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.Builder builder - */ -static startList(builder:flatbuffers.Builder) { - builder.startObject(0); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endList(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createList(builder:flatbuffers.Builder):flatbuffers.Offset { - List.startList(builder); - return List.endList(builder); -} -} -} -/** - * Same as List, but with 64-bit offsets, allowing to represent - * extremely large data values. - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class LargeList { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns LargeList - */ -__init(i:number, bb:flatbuffers.ByteBuffer):LargeList { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param LargeList= obj - * @returns LargeList - */ -static getRootAsLargeList(bb:flatbuffers.ByteBuffer, obj?:LargeList):LargeList { - return (obj || new LargeList()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param LargeList= obj - * @returns LargeList - */ -static getSizePrefixedRootAsLargeList(bb:flatbuffers.ByteBuffer, obj?:LargeList):LargeList { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new LargeList()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.Builder builder - */ -static startLargeList(builder:flatbuffers.Builder) { - builder.startObject(0); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endLargeList(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createLargeList(builder:flatbuffers.Builder):flatbuffers.Offset { - LargeList.startLargeList(builder); - return LargeList.endLargeList(builder); -} -} -} -/** - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class FixedSizeList { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns FixedSizeList - */ -__init(i:number, bb:flatbuffers.ByteBuffer):FixedSizeList { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param FixedSizeList= obj - * @returns FixedSizeList - */ -static getRootAsFixedSizeList(bb:flatbuffers.ByteBuffer, obj?:FixedSizeList):FixedSizeList { - return (obj || new FixedSizeList()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param FixedSizeList= obj - * @returns FixedSizeList - */ -static getSizePrefixedRootAsFixedSizeList(bb:flatbuffers.ByteBuffer, obj?:FixedSizeList):FixedSizeList { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new FixedSizeList()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * Number of list items per value - * - * @returns number - */ -listSize():number { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? this.bb!.readInt32(this.bb_pos + offset) : 0; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startFixedSizeList(builder:flatbuffers.Builder) { - builder.startObject(1); -}; - -/** - * @param flatbuffers.Builder builder - * @param number listSize - */ -static addListSize(builder:flatbuffers.Builder, listSize:number) { - builder.addFieldInt32(0, listSize, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endFixedSizeList(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createFixedSizeList(builder:flatbuffers.Builder, listSize:number):flatbuffers.Offset { - FixedSizeList.startFixedSizeList(builder); - FixedSizeList.addListSize(builder, listSize); - return FixedSizeList.endFixedSizeList(builder); -} -} -} -/** - * A Map is a logical nested type that is represented as - * - * List> - * - * In this layout, the keys and values are each respectively contiguous. We do - * not constrain the key and value types, so the application is responsible - * for ensuring that the keys are hashable and unique. Whether the keys are sorted - * may be set in the metadata for this field. - * - * In a field with Map type, the field has a child Struct field, which then - * has two children: key type and the second the value type. The names of the - * child fields may be respectively "entries", "key", and "value", but this is - * not enforced. - * - * Map - * ```text - * - child[0] entries: Struct - * - child[0] key: K - * - child[1] value: V - * ``` - * Neither the "entries" field nor the "key" field may be nullable. - * - * The metadata is structured so that Arrow systems without special handling - * for Map can make Map an alias for List. The "layout" attribute for the Map - * field must have the same contents as a List. - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Map { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Map - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Map { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Map= obj - * @returns Map - */ -static getRootAsMap(bb:flatbuffers.ByteBuffer, obj?:Map):Map { - return (obj || new Map()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Map= obj - * @returns Map - */ -static getSizePrefixedRootAsMap(bb:flatbuffers.ByteBuffer, obj?:Map):Map { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new Map()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * Set to true if the keys within each value are sorted - * - * @returns boolean - */ -keysSorted():boolean { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? !!this.bb!.readInt8(this.bb_pos + offset) : false; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startMap(builder:flatbuffers.Builder) { - builder.startObject(1); -}; - -/** - * @param flatbuffers.Builder builder - * @param boolean keysSorted - */ -static addKeysSorted(builder:flatbuffers.Builder, keysSorted:boolean) { - builder.addFieldInt8(0, +keysSorted, +false); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endMap(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createMap(builder:flatbuffers.Builder, keysSorted:boolean):flatbuffers.Offset { - Map.startMap(builder); - Map.addKeysSorted(builder, keysSorted); - return Map.endMap(builder); -} -} -} -/** - * A union is a complex type with children in Field - * By default ids in the type vector refer to the offsets in the children - * optionally typeIds provides an indirection between the child offset and the type id - * for each child `typeIds[offset]` is the id used in the type vector - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Union { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Union - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Union { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Union= obj - * @returns Union - */ -static getRootAsUnion(bb:flatbuffers.ByteBuffer, obj?:Union):Union { - return (obj || new Union()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Union= obj - * @returns Union - */ -static getSizePrefixedRootAsUnion(bb:flatbuffers.ByteBuffer, obj?:Union):Union { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new Union()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @returns org.apache.arrow.flatbuf.UnionMode - */ -mode():org.apache.arrow.flatbuf.UnionMode { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.UnionMode.Sparse; -}; - -/** - * @param number index - * @returns number - */ -typeIds(index: number):number|null { - var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? this.bb!.readInt32(this.bb!.__vector(this.bb_pos + offset) + index * 4) : 0; -}; - -/** - * @returns number - */ -typeIdsLength():number { - var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; -}; - -/** - * @returns Int32Array - */ -typeIdsArray():Int32Array|null { - var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? new Int32Array(this.bb!.bytes().buffer, this.bb!.bytes().byteOffset + this.bb!.__vector(this.bb_pos + offset), this.bb!.__vector_len(this.bb_pos + offset)) : null; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startUnion(builder:flatbuffers.Builder) { - builder.startObject(2); -}; - -/** - * @param flatbuffers.Builder builder - * @param org.apache.arrow.flatbuf.UnionMode mode - */ -static addMode(builder:flatbuffers.Builder, mode:org.apache.arrow.flatbuf.UnionMode) { - builder.addFieldInt16(0, mode, org.apache.arrow.flatbuf.UnionMode.Sparse); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset typeIdsOffset - */ -static addTypeIds(builder:flatbuffers.Builder, typeIdsOffset:flatbuffers.Offset) { - builder.addFieldOffset(1, typeIdsOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param Array. data - * @returns flatbuffers.Offset - */ -static createTypeIdsVector(builder:flatbuffers.Builder, data:number[]|Int32Array):flatbuffers.Offset; -/** - * @deprecated This Uint8Array overload will be removed in the future. - */ -static createTypeIdsVector(builder:flatbuffers.Builder, data:number[]|Uint8Array):flatbuffers.Offset; -static createTypeIdsVector(builder:flatbuffers.Builder, data:number[]|Int32Array|Uint8Array):flatbuffers.Offset { - builder.startVector(4, data.length, 4); - for (var i = data.length - 1; i >= 0; i--) { - builder.addInt32(data[i]); - } - return builder.endVector(); -}; - -/** - * @param flatbuffers.Builder builder - * @param number numElems - */ -static startTypeIdsVector(builder:flatbuffers.Builder, numElems:number) { - builder.startVector(4, numElems, 4); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endUnion(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createUnion(builder:flatbuffers.Builder, mode:org.apache.arrow.flatbuf.UnionMode, typeIdsOffset:flatbuffers.Offset):flatbuffers.Offset { - Union.startUnion(builder); - Union.addMode(builder, mode); - Union.addTypeIds(builder, typeIdsOffset); - return Union.endUnion(builder); -} -} -} -/** - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Int { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Int - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Int { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Int= obj - * @returns Int - */ -static getRootAsInt(bb:flatbuffers.ByteBuffer, obj?:Int):Int { - return (obj || new Int()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Int= obj - * @returns Int - */ -static getSizePrefixedRootAsInt(bb:flatbuffers.ByteBuffer, obj?:Int):Int { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new Int()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @returns number - */ -bitWidth():number { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? this.bb!.readInt32(this.bb_pos + offset) : 0; -}; - -/** - * @returns boolean - */ -isSigned():boolean { - var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? !!this.bb!.readInt8(this.bb_pos + offset) : false; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startInt(builder:flatbuffers.Builder) { - builder.startObject(2); -}; - -/** - * @param flatbuffers.Builder builder - * @param number bitWidth - */ -static addBitWidth(builder:flatbuffers.Builder, bitWidth:number) { - builder.addFieldInt32(0, bitWidth, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param boolean isSigned - */ -static addIsSigned(builder:flatbuffers.Builder, isSigned:boolean) { - builder.addFieldInt8(1, +isSigned, +false); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endInt(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createInt(builder:flatbuffers.Builder, bitWidth:number, isSigned:boolean):flatbuffers.Offset { - Int.startInt(builder); - Int.addBitWidth(builder, bitWidth); - Int.addIsSigned(builder, isSigned); - return Int.endInt(builder); -} -} -} -/** - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class FloatingPoint { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns FloatingPoint - */ -__init(i:number, bb:flatbuffers.ByteBuffer):FloatingPoint { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param FloatingPoint= obj - * @returns FloatingPoint - */ -static getRootAsFloatingPoint(bb:flatbuffers.ByteBuffer, obj?:FloatingPoint):FloatingPoint { - return (obj || new FloatingPoint()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param FloatingPoint= obj - * @returns FloatingPoint - */ -static getSizePrefixedRootAsFloatingPoint(bb:flatbuffers.ByteBuffer, obj?:FloatingPoint):FloatingPoint { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new FloatingPoint()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @returns org.apache.arrow.flatbuf.Precision - */ -precision():org.apache.arrow.flatbuf.Precision { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.Precision.HALF; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startFloatingPoint(builder:flatbuffers.Builder) { - builder.startObject(1); -}; - -/** - * @param flatbuffers.Builder builder - * @param org.apache.arrow.flatbuf.Precision precision - */ -static addPrecision(builder:flatbuffers.Builder, precision:org.apache.arrow.flatbuf.Precision) { - builder.addFieldInt16(0, precision, org.apache.arrow.flatbuf.Precision.HALF); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endFloatingPoint(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createFloatingPoint(builder:flatbuffers.Builder, precision:org.apache.arrow.flatbuf.Precision):flatbuffers.Offset { - FloatingPoint.startFloatingPoint(builder); - FloatingPoint.addPrecision(builder, precision); - return FloatingPoint.endFloatingPoint(builder); -} -} -} -/** - * Unicode with UTF-8 encoding - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Utf8 { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Utf8 - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Utf8 { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Utf8= obj - * @returns Utf8 - */ -static getRootAsUtf8(bb:flatbuffers.ByteBuffer, obj?:Utf8):Utf8 { - return (obj || new Utf8()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Utf8= obj - * @returns Utf8 - */ -static getSizePrefixedRootAsUtf8(bb:flatbuffers.ByteBuffer, obj?:Utf8):Utf8 { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new Utf8()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.Builder builder - */ -static startUtf8(builder:flatbuffers.Builder) { - builder.startObject(0); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endUtf8(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createUtf8(builder:flatbuffers.Builder):flatbuffers.Offset { - Utf8.startUtf8(builder); - return Utf8.endUtf8(builder); -} -} -} -/** - * Opaque binary data - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Binary { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Binary - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Binary { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Binary= obj - * @returns Binary - */ -static getRootAsBinary(bb:flatbuffers.ByteBuffer, obj?:Binary):Binary { - return (obj || new Binary()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Binary= obj - * @returns Binary - */ -static getSizePrefixedRootAsBinary(bb:flatbuffers.ByteBuffer, obj?:Binary):Binary { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new Binary()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.Builder builder - */ -static startBinary(builder:flatbuffers.Builder) { - builder.startObject(0); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endBinary(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createBinary(builder:flatbuffers.Builder):flatbuffers.Offset { - Binary.startBinary(builder); - return Binary.endBinary(builder); -} -} -} -/** - * Same as Utf8, but with 64-bit offsets, allowing to represent - * extremely large data values. - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class LargeUtf8 { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns LargeUtf8 - */ -__init(i:number, bb:flatbuffers.ByteBuffer):LargeUtf8 { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param LargeUtf8= obj - * @returns LargeUtf8 - */ -static getRootAsLargeUtf8(bb:flatbuffers.ByteBuffer, obj?:LargeUtf8):LargeUtf8 { - return (obj || new LargeUtf8()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param LargeUtf8= obj - * @returns LargeUtf8 - */ -static getSizePrefixedRootAsLargeUtf8(bb:flatbuffers.ByteBuffer, obj?:LargeUtf8):LargeUtf8 { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new LargeUtf8()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.Builder builder - */ -static startLargeUtf8(builder:flatbuffers.Builder) { - builder.startObject(0); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endLargeUtf8(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createLargeUtf8(builder:flatbuffers.Builder):flatbuffers.Offset { - LargeUtf8.startLargeUtf8(builder); - return LargeUtf8.endLargeUtf8(builder); -} -} -} -/** - * Same as Binary, but with 64-bit offsets, allowing to represent - * extremely large data values. - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class LargeBinary { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns LargeBinary - */ -__init(i:number, bb:flatbuffers.ByteBuffer):LargeBinary { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param LargeBinary= obj - * @returns LargeBinary - */ -static getRootAsLargeBinary(bb:flatbuffers.ByteBuffer, obj?:LargeBinary):LargeBinary { - return (obj || new LargeBinary()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param LargeBinary= obj - * @returns LargeBinary - */ -static getSizePrefixedRootAsLargeBinary(bb:flatbuffers.ByteBuffer, obj?:LargeBinary):LargeBinary { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new LargeBinary()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.Builder builder - */ -static startLargeBinary(builder:flatbuffers.Builder) { - builder.startObject(0); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endLargeBinary(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createLargeBinary(builder:flatbuffers.Builder):flatbuffers.Offset { - LargeBinary.startLargeBinary(builder); - return LargeBinary.endLargeBinary(builder); -} -} -} -/** - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class FixedSizeBinary { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns FixedSizeBinary - */ -__init(i:number, bb:flatbuffers.ByteBuffer):FixedSizeBinary { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param FixedSizeBinary= obj - * @returns FixedSizeBinary - */ -static getRootAsFixedSizeBinary(bb:flatbuffers.ByteBuffer, obj?:FixedSizeBinary):FixedSizeBinary { - return (obj || new FixedSizeBinary()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param FixedSizeBinary= obj - * @returns FixedSizeBinary - */ -static getSizePrefixedRootAsFixedSizeBinary(bb:flatbuffers.ByteBuffer, obj?:FixedSizeBinary):FixedSizeBinary { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new FixedSizeBinary()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * Number of bytes per value - * - * @returns number - */ -byteWidth():number { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? this.bb!.readInt32(this.bb_pos + offset) : 0; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startFixedSizeBinary(builder:flatbuffers.Builder) { - builder.startObject(1); -}; - -/** - * @param flatbuffers.Builder builder - * @param number byteWidth - */ -static addByteWidth(builder:flatbuffers.Builder, byteWidth:number) { - builder.addFieldInt32(0, byteWidth, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endFixedSizeBinary(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createFixedSizeBinary(builder:flatbuffers.Builder, byteWidth:number):flatbuffers.Offset { - FixedSizeBinary.startFixedSizeBinary(builder); - FixedSizeBinary.addByteWidth(builder, byteWidth); - return FixedSizeBinary.endFixedSizeBinary(builder); -} -} -} -/** - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Bool { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Bool - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Bool { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Bool= obj - * @returns Bool - */ -static getRootAsBool(bb:flatbuffers.ByteBuffer, obj?:Bool):Bool { - return (obj || new Bool()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Bool= obj - * @returns Bool - */ -static getSizePrefixedRootAsBool(bb:flatbuffers.ByteBuffer, obj?:Bool):Bool { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new Bool()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.Builder builder - */ -static startBool(builder:flatbuffers.Builder) { - builder.startObject(0); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endBool(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createBool(builder:flatbuffers.Builder):flatbuffers.Offset { - Bool.startBool(builder); - return Bool.endBool(builder); -} -} -} -/** - * Exact decimal value represented as an integer value in two's - * complement. Currently only 128-bit (16-byte) and 256-bit (32-byte) integers - * are used. The representation uses the endianness indicated - * in the Schema. - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Decimal { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Decimal - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Decimal { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Decimal= obj - * @returns Decimal - */ -static getRootAsDecimal(bb:flatbuffers.ByteBuffer, obj?:Decimal):Decimal { - return (obj || new Decimal()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Decimal= obj - * @returns Decimal - */ -static getSizePrefixedRootAsDecimal(bb:flatbuffers.ByteBuffer, obj?:Decimal):Decimal { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new Decimal()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * Total number of decimal digits - * - * @returns number - */ -precision():number { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? this.bb!.readInt32(this.bb_pos + offset) : 0; -}; - -/** - * Number of digits after the decimal point "." - * - * @returns number - */ -scale():number { - var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? this.bb!.readInt32(this.bb_pos + offset) : 0; -}; - -/** - * Number of bits per value. The only accepted widths are 128 and 256. - * We use bitWidth for consistency with Int::bitWidth. - * - * @returns number - */ -bitWidth():number { - var offset = this.bb!.__offset(this.bb_pos, 8); - return offset ? this.bb!.readInt32(this.bb_pos + offset) : 128; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startDecimal(builder:flatbuffers.Builder) { - builder.startObject(3); -}; - -/** - * @param flatbuffers.Builder builder - * @param number precision - */ -static addPrecision(builder:flatbuffers.Builder, precision:number) { - builder.addFieldInt32(0, precision, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param number scale - */ -static addScale(builder:flatbuffers.Builder, scale:number) { - builder.addFieldInt32(1, scale, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param number bitWidth - */ -static addBitWidth(builder:flatbuffers.Builder, bitWidth:number) { - builder.addFieldInt32(2, bitWidth, 128); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endDecimal(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createDecimal(builder:flatbuffers.Builder, precision:number, scale:number, bitWidth:number):flatbuffers.Offset { - Decimal.startDecimal(builder); - Decimal.addPrecision(builder, precision); - Decimal.addScale(builder, scale); - Decimal.addBitWidth(builder, bitWidth); - return Decimal.endDecimal(builder); -} -} -} -/** - * Date is either a 32-bit or 64-bit type representing elapsed time since UNIX - * epoch (1970-01-01), stored in either of two units: - * - * * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no - * leap seconds), where the values are evenly divisible by 86400000 - * * Days (32 bits) since the UNIX epoch - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Date { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Date - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Date { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Date= obj - * @returns Date - */ -static getRootAsDate(bb:flatbuffers.ByteBuffer, obj?:Date):Date { - return (obj || new Date()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Date= obj - * @returns Date - */ -static getSizePrefixedRootAsDate(bb:flatbuffers.ByteBuffer, obj?:Date):Date { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new Date()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @returns org.apache.arrow.flatbuf.DateUnit - */ -unit():org.apache.arrow.flatbuf.DateUnit { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.DateUnit.MILLISECOND; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startDate(builder:flatbuffers.Builder) { - builder.startObject(1); -}; - -/** - * @param flatbuffers.Builder builder - * @param org.apache.arrow.flatbuf.DateUnit unit - */ -static addUnit(builder:flatbuffers.Builder, unit:org.apache.arrow.flatbuf.DateUnit) { - builder.addFieldInt16(0, unit, org.apache.arrow.flatbuf.DateUnit.MILLISECOND); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endDate(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createDate(builder:flatbuffers.Builder, unit:org.apache.arrow.flatbuf.DateUnit):flatbuffers.Offset { - Date.startDate(builder); - Date.addUnit(builder, unit); - return Date.endDate(builder); -} -} -} -/** - * Time type. The physical storage type depends on the unit - * - SECOND and MILLISECOND: 32 bits - * - MICROSECOND and NANOSECOND: 64 bits - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Time { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Time - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Time { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Time= obj - * @returns Time - */ -static getRootAsTime(bb:flatbuffers.ByteBuffer, obj?:Time):Time { - return (obj || new Time()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Time= obj - * @returns Time - */ -static getSizePrefixedRootAsTime(bb:flatbuffers.ByteBuffer, obj?:Time):Time { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new Time()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @returns org.apache.arrow.flatbuf.TimeUnit - */ -unit():org.apache.arrow.flatbuf.TimeUnit { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.TimeUnit.MILLISECOND; -}; - -/** - * @returns number - */ -bitWidth():number { - var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? this.bb!.readInt32(this.bb_pos + offset) : 32; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startTime(builder:flatbuffers.Builder) { - builder.startObject(2); -}; - -/** - * @param flatbuffers.Builder builder - * @param org.apache.arrow.flatbuf.TimeUnit unit - */ -static addUnit(builder:flatbuffers.Builder, unit:org.apache.arrow.flatbuf.TimeUnit) { - builder.addFieldInt16(0, unit, org.apache.arrow.flatbuf.TimeUnit.MILLISECOND); -}; - -/** - * @param flatbuffers.Builder builder - * @param number bitWidth - */ -static addBitWidth(builder:flatbuffers.Builder, bitWidth:number) { - builder.addFieldInt32(1, bitWidth, 32); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endTime(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createTime(builder:flatbuffers.Builder, unit:org.apache.arrow.flatbuf.TimeUnit, bitWidth:number):flatbuffers.Offset { - Time.startTime(builder); - Time.addUnit(builder, unit); - Time.addBitWidth(builder, bitWidth); - return Time.endTime(builder); -} -} -} -/** - * Time elapsed from the Unix epoch, 00:00:00.000 on 1 January 1970, excluding - * leap seconds, as a 64-bit integer. Note that UNIX time does not include - * leap seconds. - * - * Date & time libraries often have multiple different data types for temporal - * data. In order to ease interoperability between different implementations the - * Arrow project has some recommendations for encoding these types into a Timestamp - * column. - * - * An "instant" represents a single moment in time that has no meaningful time zone - * or the time zone is unknown. A column of instants can also contain values from - * multiple time zones. To encode an instant set the timezone string to "UTC". - * - * A "zoned date-time" represents a single moment in time that has a meaningful - * reference time zone. To encode a zoned date-time as a Timestamp set the timezone - * string to the name of the timezone. There is some ambiguity between an instant - * and a zoned date-time with the UTC time zone. Both of these are stored the same. - * Typically, this distinction does not matter. If it does, then an application should - * use custom metadata or an extension type to distinguish between the two cases. - * - * An "offset date-time" represents a single moment in time combined with a meaningful - * offset from UTC. To encode an offset date-time as a Timestamp set the timezone string - * to the numeric time zone offset string (e.g. "+03:00"). - * - * A "local date-time" does not represent a single moment in time. It represents a wall - * clock time combined with a date. Because of daylight savings time there may multiple - * instants that correspond to a single local date-time in any given time zone. A - * local date-time is often stored as a struct or a Date32/Time64 pair. However, it can - * also be encoded into a Timestamp column. To do so the value should be the the time - * elapsed from the Unix epoch so that a wall clock in UTC would display the desired time. - * The timezone string should be set to null or the empty string. - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Timestamp { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Timestamp - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Timestamp { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Timestamp= obj - * @returns Timestamp - */ -static getRootAsTimestamp(bb:flatbuffers.ByteBuffer, obj?:Timestamp):Timestamp { - return (obj || new Timestamp()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Timestamp= obj - * @returns Timestamp - */ -static getSizePrefixedRootAsTimestamp(bb:flatbuffers.ByteBuffer, obj?:Timestamp):Timestamp { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new Timestamp()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @returns org.apache.arrow.flatbuf.TimeUnit - */ -unit():org.apache.arrow.flatbuf.TimeUnit { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.TimeUnit.SECOND; -}; - -/** - * The time zone is a string indicating the name of a time zone, one of: - * - * * As used in the Olson time zone database (the "tz database" or - * "tzdata"), such as "America/New_York" - * * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30 - * - * Whether a timezone string is present indicates different semantics about - * the data: - * - * * If the time zone is null or an empty string, the data is a local date-time - * and does not represent a single moment in time. Instead it represents a wall clock - * time and care should be taken to avoid interpreting it semantically as an instant. - * - * * If the time zone is set to a valid value, values can be displayed as - * "localized" to that time zone, even though the underlying 64-bit - * integers are identical to the same data stored in UTC. Converting - * between time zones is a metadata-only operation and does not change the - * underlying values - * - * @param flatbuffers.Encoding= optionalEncoding - * @returns string|Uint8Array|null - */ -timezone():string|null -timezone(optionalEncoding:flatbuffers.Encoding):string|Uint8Array|null -timezone(optionalEncoding?:any):string|Uint8Array|null { - var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startTimestamp(builder:flatbuffers.Builder) { - builder.startObject(2); -}; - -/** - * @param flatbuffers.Builder builder - * @param org.apache.arrow.flatbuf.TimeUnit unit - */ -static addUnit(builder:flatbuffers.Builder, unit:org.apache.arrow.flatbuf.TimeUnit) { - builder.addFieldInt16(0, unit, org.apache.arrow.flatbuf.TimeUnit.SECOND); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset timezoneOffset - */ -static addTimezone(builder:flatbuffers.Builder, timezoneOffset:flatbuffers.Offset) { - builder.addFieldOffset(1, timezoneOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endTimestamp(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createTimestamp(builder:flatbuffers.Builder, unit:org.apache.arrow.flatbuf.TimeUnit, timezoneOffset:flatbuffers.Offset):flatbuffers.Offset { - Timestamp.startTimestamp(builder); - Timestamp.addUnit(builder, unit); - Timestamp.addTimezone(builder, timezoneOffset); - return Timestamp.endTimestamp(builder); -} -} -} -/** - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Interval { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Interval - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Interval { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Interval= obj - * @returns Interval - */ -static getRootAsInterval(bb:flatbuffers.ByteBuffer, obj?:Interval):Interval { - return (obj || new Interval()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Interval= obj - * @returns Interval - */ -static getSizePrefixedRootAsInterval(bb:flatbuffers.ByteBuffer, obj?:Interval):Interval { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new Interval()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @returns org.apache.arrow.flatbuf.IntervalUnit - */ -unit():org.apache.arrow.flatbuf.IntervalUnit { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.IntervalUnit.YEAR_MONTH; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startInterval(builder:flatbuffers.Builder) { - builder.startObject(1); -}; - -/** - * @param flatbuffers.Builder builder - * @param org.apache.arrow.flatbuf.IntervalUnit unit - */ -static addUnit(builder:flatbuffers.Builder, unit:org.apache.arrow.flatbuf.IntervalUnit) { - builder.addFieldInt16(0, unit, org.apache.arrow.flatbuf.IntervalUnit.YEAR_MONTH); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endInterval(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createInterval(builder:flatbuffers.Builder, unit:org.apache.arrow.flatbuf.IntervalUnit):flatbuffers.Offset { - Interval.startInterval(builder); - Interval.addUnit(builder, unit); - return Interval.endInterval(builder); -} -} -} -/** - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Duration { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Duration - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Duration { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Duration= obj - * @returns Duration - */ -static getRootAsDuration(bb:flatbuffers.ByteBuffer, obj?:Duration):Duration { - return (obj || new Duration()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Duration= obj - * @returns Duration - */ -static getSizePrefixedRootAsDuration(bb:flatbuffers.ByteBuffer, obj?:Duration):Duration { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new Duration()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @returns org.apache.arrow.flatbuf.TimeUnit - */ -unit():org.apache.arrow.flatbuf.TimeUnit { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.TimeUnit.MILLISECOND; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startDuration(builder:flatbuffers.Builder) { - builder.startObject(1); -}; - -/** - * @param flatbuffers.Builder builder - * @param org.apache.arrow.flatbuf.TimeUnit unit - */ -static addUnit(builder:flatbuffers.Builder, unit:org.apache.arrow.flatbuf.TimeUnit) { - builder.addFieldInt16(0, unit, org.apache.arrow.flatbuf.TimeUnit.MILLISECOND); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endDuration(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createDuration(builder:flatbuffers.Builder, unit:org.apache.arrow.flatbuf.TimeUnit):flatbuffers.Offset { - Duration.startDuration(builder); - Duration.addUnit(builder, unit); - return Duration.endDuration(builder); -} -} -} -/** - * ---------------------------------------------------------------------- - * user defined key value pairs to add custom metadata to arrow - * key namespacing is the responsibility of the user - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class KeyValue { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns KeyValue - */ -__init(i:number, bb:flatbuffers.ByteBuffer):KeyValue { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param KeyValue= obj - * @returns KeyValue - */ -static getRootAsKeyValue(bb:flatbuffers.ByteBuffer, obj?:KeyValue):KeyValue { - return (obj || new KeyValue()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param KeyValue= obj - * @returns KeyValue - */ -static getSizePrefixedRootAsKeyValue(bb:flatbuffers.ByteBuffer, obj?:KeyValue):KeyValue { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new KeyValue()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.Encoding= optionalEncoding - * @returns string|Uint8Array|null - */ -key():string|null -key(optionalEncoding:flatbuffers.Encoding):string|Uint8Array|null -key(optionalEncoding?:any):string|Uint8Array|null { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null; -}; - -/** - * @param flatbuffers.Encoding= optionalEncoding - * @returns string|Uint8Array|null - */ -value():string|null -value(optionalEncoding:flatbuffers.Encoding):string|Uint8Array|null -value(optionalEncoding?:any):string|Uint8Array|null { - var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startKeyValue(builder:flatbuffers.Builder) { - builder.startObject(2); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset keyOffset - */ -static addKey(builder:flatbuffers.Builder, keyOffset:flatbuffers.Offset) { - builder.addFieldOffset(0, keyOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset valueOffset - */ -static addValue(builder:flatbuffers.Builder, valueOffset:flatbuffers.Offset) { - builder.addFieldOffset(1, valueOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endKeyValue(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -static createKeyValue(builder:flatbuffers.Builder, keyOffset:flatbuffers.Offset, valueOffset:flatbuffers.Offset):flatbuffers.Offset { - KeyValue.startKeyValue(builder); - KeyValue.addKey(builder, keyOffset); - KeyValue.addValue(builder, valueOffset); - return KeyValue.endKeyValue(builder); -} -} -} -/** - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class DictionaryEncoding { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns DictionaryEncoding - */ -__init(i:number, bb:flatbuffers.ByteBuffer):DictionaryEncoding { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param DictionaryEncoding= obj - * @returns DictionaryEncoding - */ -static getRootAsDictionaryEncoding(bb:flatbuffers.ByteBuffer, obj?:DictionaryEncoding):DictionaryEncoding { - return (obj || new DictionaryEncoding()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param DictionaryEncoding= obj - * @returns DictionaryEncoding - */ -static getSizePrefixedRootAsDictionaryEncoding(bb:flatbuffers.ByteBuffer, obj?:DictionaryEncoding):DictionaryEncoding { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new DictionaryEncoding()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * The known dictionary id in the application where this data is used. In - * the file or streaming formats, the dictionary ids are found in the - * DictionaryBatch messages - * - * @returns flatbuffers.Long - */ -id():flatbuffers.Long { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? this.bb!.readInt64(this.bb_pos + offset) : this.bb!.createLong(0, 0); -}; - -/** - * The dictionary indices are constrained to be non-negative integers. If - * this field is null, the indices must be signed int32. To maximize - * cross-language compatibility and performance, implementations are - * recommended to prefer signed integer types over unsigned integer types - * and to avoid uint64 indices unless they are required by an application. - * - * @param org.apache.arrow.flatbuf.Int= obj - * @returns org.apache.arrow.flatbuf.Int|null - */ -indexType(obj?:org.apache.arrow.flatbuf.Int):org.apache.arrow.flatbuf.Int|null { - var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? (obj || new org.apache.arrow.flatbuf.Int()).__init(this.bb!.__indirect(this.bb_pos + offset), this.bb!) : null; -}; - -/** - * By default, dictionaries are not ordered, or the order does not have - * semantic meaning. In some statistical, applications, dictionary-encoding - * is used to represent ordered categorical data, and we provide a way to - * preserve that metadata here - * - * @returns boolean - */ -isOrdered():boolean { - var offset = this.bb!.__offset(this.bb_pos, 8); - return offset ? !!this.bb!.readInt8(this.bb_pos + offset) : false; -}; - -/** - * @returns org.apache.arrow.flatbuf.DictionaryKind - */ -dictionaryKind():org.apache.arrow.flatbuf.DictionaryKind { - var offset = this.bb!.__offset(this.bb_pos, 10); - return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.DictionaryKind.DenseArray; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startDictionaryEncoding(builder:flatbuffers.Builder) { - builder.startObject(4); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Long id - */ -static addId(builder:flatbuffers.Builder, id:flatbuffers.Long) { - builder.addFieldInt64(0, id, builder.createLong(0, 0)); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset indexTypeOffset - */ -static addIndexType(builder:flatbuffers.Builder, indexTypeOffset:flatbuffers.Offset) { - builder.addFieldOffset(1, indexTypeOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param boolean isOrdered - */ -static addIsOrdered(builder:flatbuffers.Builder, isOrdered:boolean) { - builder.addFieldInt8(2, +isOrdered, +false); -}; - -/** - * @param flatbuffers.Builder builder - * @param org.apache.arrow.flatbuf.DictionaryKind dictionaryKind - */ -static addDictionaryKind(builder:flatbuffers.Builder, dictionaryKind:org.apache.arrow.flatbuf.DictionaryKind) { - builder.addFieldInt16(3, dictionaryKind, org.apache.arrow.flatbuf.DictionaryKind.DenseArray); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endDictionaryEncoding(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -} -} -/** - * ---------------------------------------------------------------------- - * A field represents a named column in a record / row batch or child of a - * nested type. - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Field { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Field - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Field { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Field= obj - * @returns Field - */ -static getRootAsField(bb:flatbuffers.ByteBuffer, obj?:Field):Field { - return (obj || new Field()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Field= obj - * @returns Field - */ -static getSizePrefixedRootAsField(bb:flatbuffers.ByteBuffer, obj?:Field):Field { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new Field()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * Name is not required, in i.e. a List - * - * @param flatbuffers.Encoding= optionalEncoding - * @returns string|Uint8Array|null - */ -name():string|null -name(optionalEncoding:flatbuffers.Encoding):string|Uint8Array|null -name(optionalEncoding?:any):string|Uint8Array|null { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null; -}; - -/** - * Whether or not this field can contain nulls. Should be true in general. - * - * @returns boolean - */ -nullable():boolean { - var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? !!this.bb!.readInt8(this.bb_pos + offset) : false; -}; - -/** - * @returns org.apache.arrow.flatbuf.Type - */ -typeType():org.apache.arrow.flatbuf.Type { - var offset = this.bb!.__offset(this.bb_pos, 8); - return offset ? /** */ (this.bb!.readUint8(this.bb_pos + offset)) : org.apache.arrow.flatbuf.Type.NONE; -}; - -/** - * This is the type of the decoded value if the field is dictionary encoded. - * - * @param flatbuffers.Table obj - * @returns ?flatbuffers.Table - */ -type(obj:T):T|null { - var offset = this.bb!.__offset(this.bb_pos, 10); - return offset ? this.bb!.__union(obj, this.bb_pos + offset) : null; -}; - -/** - * Present only if the field is dictionary encoded. - * - * @param org.apache.arrow.flatbuf.DictionaryEncoding= obj - * @returns org.apache.arrow.flatbuf.DictionaryEncoding|null - */ -dictionary(obj?:org.apache.arrow.flatbuf.DictionaryEncoding):org.apache.arrow.flatbuf.DictionaryEncoding|null { - var offset = this.bb!.__offset(this.bb_pos, 12); - return offset ? (obj || new org.apache.arrow.flatbuf.DictionaryEncoding()).__init(this.bb!.__indirect(this.bb_pos + offset), this.bb!) : null; -}; - -/** - * children apply only to nested data types like Struct, List and Union. For - * primitive types children will have length 0. - * - * @param number index - * @param org.apache.arrow.flatbuf.Field= obj - * @returns org.apache.arrow.flatbuf.Field - */ -children(index: number, obj?:org.apache.arrow.flatbuf.Field):org.apache.arrow.flatbuf.Field|null { - var offset = this.bb!.__offset(this.bb_pos, 14); - return offset ? (obj || new org.apache.arrow.flatbuf.Field()).__init(this.bb!.__indirect(this.bb!.__vector(this.bb_pos + offset) + index * 4), this.bb!) : null; -}; - -/** - * @returns number - */ -childrenLength():number { - var offset = this.bb!.__offset(this.bb_pos, 14); - return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; -}; - -/** - * User-defined metadata - * - * @param number index - * @param org.apache.arrow.flatbuf.KeyValue= obj - * @returns org.apache.arrow.flatbuf.KeyValue - */ -customMetadata(index: number, obj?:org.apache.arrow.flatbuf.KeyValue):org.apache.arrow.flatbuf.KeyValue|null { - var offset = this.bb!.__offset(this.bb_pos, 16); - return offset ? (obj || new org.apache.arrow.flatbuf.KeyValue()).__init(this.bb!.__indirect(this.bb!.__vector(this.bb_pos + offset) + index * 4), this.bb!) : null; -}; - -/** - * @returns number - */ -customMetadataLength():number { - var offset = this.bb!.__offset(this.bb_pos, 16); - return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startField(builder:flatbuffers.Builder) { - builder.startObject(7); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset nameOffset - */ -static addName(builder:flatbuffers.Builder, nameOffset:flatbuffers.Offset) { - builder.addFieldOffset(0, nameOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param boolean nullable - */ -static addNullable(builder:flatbuffers.Builder, nullable:boolean) { - builder.addFieldInt8(1, +nullable, +false); -}; - -/** - * @param flatbuffers.Builder builder - * @param org.apache.arrow.flatbuf.Type typeType - */ -static addTypeType(builder:flatbuffers.Builder, typeType:org.apache.arrow.flatbuf.Type) { - builder.addFieldInt8(2, typeType, org.apache.arrow.flatbuf.Type.NONE); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset typeOffset - */ -static addType(builder:flatbuffers.Builder, typeOffset:flatbuffers.Offset) { - builder.addFieldOffset(3, typeOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset dictionaryOffset - */ -static addDictionary(builder:flatbuffers.Builder, dictionaryOffset:flatbuffers.Offset) { - builder.addFieldOffset(4, dictionaryOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset childrenOffset - */ -static addChildren(builder:flatbuffers.Builder, childrenOffset:flatbuffers.Offset) { - builder.addFieldOffset(5, childrenOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param Array. data - * @returns flatbuffers.Offset - */ -static createChildrenVector(builder:flatbuffers.Builder, data:flatbuffers.Offset[]):flatbuffers.Offset { - builder.startVector(4, data.length, 4); - for (var i = data.length - 1; i >= 0; i--) { - builder.addOffset(data[i]); - } - return builder.endVector(); -}; - -/** - * @param flatbuffers.Builder builder - * @param number numElems - */ -static startChildrenVector(builder:flatbuffers.Builder, numElems:number) { - builder.startVector(4, numElems, 4); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset customMetadataOffset - */ -static addCustomMetadata(builder:flatbuffers.Builder, customMetadataOffset:flatbuffers.Offset) { - builder.addFieldOffset(6, customMetadataOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param Array. data - * @returns flatbuffers.Offset - */ -static createCustomMetadataVector(builder:flatbuffers.Builder, data:flatbuffers.Offset[]):flatbuffers.Offset { - builder.startVector(4, data.length, 4); - for (var i = data.length - 1; i >= 0; i--) { - builder.addOffset(data[i]); - } - return builder.endVector(); -}; - -/** - * @param flatbuffers.Builder builder - * @param number numElems - */ -static startCustomMetadataVector(builder:flatbuffers.Builder, numElems:number) { - builder.startVector(4, numElems, 4); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endField(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -} -} -/** - * ---------------------------------------------------------------------- - * A Buffer represents a single contiguous memory segment - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Buffer { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Buffer - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Buffer { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * The relative offset into the shared memory page where the bytes for this - * buffer starts - * - * @returns flatbuffers.Long - */ -offset():flatbuffers.Long { - return this.bb!.readInt64(this.bb_pos); -}; - -/** - * The absolute length (in bytes) of the memory buffer. The memory is found - * from offset (inclusive) to offset + length (non-inclusive). When building - * messages using the encapsulated IPC message, padding bytes may be written - * after a buffer, but such padding bytes do not need to be accounted for in - * the size here. - * - * @returns flatbuffers.Long - */ -length():flatbuffers.Long { - return this.bb!.readInt64(this.bb_pos + 8); -}; - -/** - * @returns number - */ -static sizeOf():number { - return 16; -} - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Long offset - * @param flatbuffers.Long length - * @returns flatbuffers.Offset - */ -static createBuffer(builder:flatbuffers.Builder, offset: flatbuffers.Long, length: flatbuffers.Long):flatbuffers.Offset { - builder.prep(8, 16); - builder.writeInt64(length); - builder.writeInt64(offset); - return builder.offset(); -}; - -} -} -/** - * ---------------------------------------------------------------------- - * A Schema describes the columns in a row batch - * - * @constructor - */ -export namespace org.apache.arrow.flatbuf{ -export class Schema { - bb: flatbuffers.ByteBuffer|null = null; - - bb_pos:number = 0; -/** - * @param number i - * @param flatbuffers.ByteBuffer bb - * @returns Schema - */ -__init(i:number, bb:flatbuffers.ByteBuffer):Schema { - this.bb_pos = i; - this.bb = bb; - return this; -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Schema= obj - * @returns Schema - */ -static getRootAsSchema(bb:flatbuffers.ByteBuffer, obj?:Schema):Schema { - return (obj || new Schema()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * @param flatbuffers.ByteBuffer bb - * @param Schema= obj - * @returns Schema - */ -static getSizePrefixedRootAsSchema(bb:flatbuffers.ByteBuffer, obj?:Schema):Schema { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new Schema()).__init(bb.readInt32(bb.position()) + bb.position(), bb); -}; - -/** - * endianness of the buffer - * it is Little Endian by default - * if endianness doesn't match the underlying system then the vectors need to be converted - * - * @returns org.apache.arrow.flatbuf.Endianness - */ -endianness():org.apache.arrow.flatbuf.Endianness { - var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.Endianness.Little; -}; - -/** - * @param number index - * @param org.apache.arrow.flatbuf.Field= obj - * @returns org.apache.arrow.flatbuf.Field - */ -fields(index: number, obj?:org.apache.arrow.flatbuf.Field):org.apache.arrow.flatbuf.Field|null { - var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? (obj || new org.apache.arrow.flatbuf.Field()).__init(this.bb!.__indirect(this.bb!.__vector(this.bb_pos + offset) + index * 4), this.bb!) : null; -}; - -/** - * @returns number - */ -fieldsLength():number { - var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; -}; - -/** - * @param number index - * @param org.apache.arrow.flatbuf.KeyValue= obj - * @returns org.apache.arrow.flatbuf.KeyValue - */ -customMetadata(index: number, obj?:org.apache.arrow.flatbuf.KeyValue):org.apache.arrow.flatbuf.KeyValue|null { - var offset = this.bb!.__offset(this.bb_pos, 8); - return offset ? (obj || new org.apache.arrow.flatbuf.KeyValue()).__init(this.bb!.__indirect(this.bb!.__vector(this.bb_pos + offset) + index * 4), this.bb!) : null; -}; - -/** - * @returns number - */ -customMetadataLength():number { - var offset = this.bb!.__offset(this.bb_pos, 8); - return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; -}; - -/** - * Features used in the stream/file. - * - * @param number index - * @returns flatbuffers.Long - */ -features(index: number):flatbuffers.Long|null { - var offset = this.bb!.__offset(this.bb_pos, 10); - return offset ? /** */ (this.bb!.readInt64(this.bb!.__vector(this.bb_pos + offset) + index * 8)) : this.bb!.createLong(0, 0); -}; - -/** - * @returns number - */ -featuresLength():number { - var offset = this.bb!.__offset(this.bb_pos, 10); - return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; -}; - -/** - * @param flatbuffers.Builder builder - */ -static startSchema(builder:flatbuffers.Builder) { - builder.startObject(4); -}; - -/** - * @param flatbuffers.Builder builder - * @param org.apache.arrow.flatbuf.Endianness endianness - */ -static addEndianness(builder:flatbuffers.Builder, endianness:org.apache.arrow.flatbuf.Endianness) { - builder.addFieldInt16(0, endianness, org.apache.arrow.flatbuf.Endianness.Little); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset fieldsOffset - */ -static addFields(builder:flatbuffers.Builder, fieldsOffset:flatbuffers.Offset) { - builder.addFieldOffset(1, fieldsOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param Array. data - * @returns flatbuffers.Offset - */ -static createFieldsVector(builder:flatbuffers.Builder, data:flatbuffers.Offset[]):flatbuffers.Offset { - builder.startVector(4, data.length, 4); - for (var i = data.length - 1; i >= 0; i--) { - builder.addOffset(data[i]); - } - return builder.endVector(); -}; - -/** - * @param flatbuffers.Builder builder - * @param number numElems - */ -static startFieldsVector(builder:flatbuffers.Builder, numElems:number) { - builder.startVector(4, numElems, 4); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset customMetadataOffset - */ -static addCustomMetadata(builder:flatbuffers.Builder, customMetadataOffset:flatbuffers.Offset) { - builder.addFieldOffset(2, customMetadataOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param Array. data - * @returns flatbuffers.Offset - */ -static createCustomMetadataVector(builder:flatbuffers.Builder, data:flatbuffers.Offset[]):flatbuffers.Offset { - builder.startVector(4, data.length, 4); - for (var i = data.length - 1; i >= 0; i--) { - builder.addOffset(data[i]); - } - return builder.endVector(); -}; - -/** - * @param flatbuffers.Builder builder - * @param number numElems - */ -static startCustomMetadataVector(builder:flatbuffers.Builder, numElems:number) { - builder.startVector(4, numElems, 4); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset featuresOffset - */ -static addFeatures(builder:flatbuffers.Builder, featuresOffset:flatbuffers.Offset) { - builder.addFieldOffset(3, featuresOffset, 0); -}; - -/** - * @param flatbuffers.Builder builder - * @param Array. data - * @returns flatbuffers.Offset - */ -static createFeaturesVector(builder:flatbuffers.Builder, data:flatbuffers.Long[]):flatbuffers.Offset { - builder.startVector(8, data.length, 8); - for (var i = data.length - 1; i >= 0; i--) { - builder.addInt64(data[i]); - } - return builder.endVector(); -}; - -/** - * @param flatbuffers.Builder builder - * @param number numElems - */ -static startFeaturesVector(builder:flatbuffers.Builder, numElems:number) { - builder.startVector(8, numElems, 8); -}; - -/** - * @param flatbuffers.Builder builder - * @returns flatbuffers.Offset - */ -static endSchema(builder:flatbuffers.Builder):flatbuffers.Offset { - var offset = builder.endObject(); - return offset; -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset offset - */ -static finishSchemaBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) { - builder.finish(offset); -}; - -/** - * @param flatbuffers.Builder builder - * @param flatbuffers.Offset offset - */ -static finishSizePrefixedSchemaBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) { - builder.finish(offset, undefined, true); -}; - -static createSchema(builder:flatbuffers.Builder, endianness:org.apache.arrow.flatbuf.Endianness, fieldsOffset:flatbuffers.Offset, customMetadataOffset:flatbuffers.Offset, featuresOffset:flatbuffers.Offset):flatbuffers.Offset { - Schema.startSchema(builder); - Schema.addEndianness(builder, endianness); - Schema.addFields(builder, fieldsOffset); - Schema.addCustomMetadata(builder, customMetadataOffset); - Schema.addFeatures(builder, featuresOffset); - return Schema.endSchema(builder); -} -} -} diff --git a/proto/raw-js-openapi/src/index.js b/proto/raw-js-openapi/src/index.js index 1e9c688f910..4e2312ce3aa 100644 --- a/proto/raw-js-openapi/src/index.js +++ b/proto/raw-js-openapi/src/index.js @@ -28,11 +28,6 @@ var browserHeaders = require("browser-headers"); var grpcWeb = require("@improbable-eng/grpc-web");//usually .grpc var jspb = require("google-protobuf"); -var flatbuffers = require("flatbuffers").flatbuffers; -var barrage = require("@deephaven/barrage"); - -var message = require('./arrow/flight/flatbuf/Message_generated'); -var schema = require('./arrow/flight/flatbuf/Schema_generated'); var io = { deephaven: { proto: { @@ -58,17 +53,8 @@ var io = { deephaven: { hierarchicaltable_pb, hierarchicaltable_pb_service }, - barrage: { - "flatbuf": { - "Barrage_generated": barrage, - } - } }}; var arrow = { flight: { - flatbuf: { - Message_generated: message, - Schema_generated: schema, - }, protocol: { Flight_pb, Flight_pb_service, @@ -80,7 +66,6 @@ var dhinternal = { browserHeaders, jspb, grpcWeb,//TODO need to expand this to the specific things we need - flatbuffers, io, arrow }; diff --git a/py/client-ticking/README.md b/py/client-ticking/README.md index 6a7ece5be63..2a01e869c7a 100644 --- a/py/client-ticking/README.md +++ b/py/client-ticking/README.md @@ -72,7 +72,7 @@ cd ${DHROOT}/py/client-ticking ```sh # Ensure the DHCPP environment variable is set per the instructions above rm -rf build dist # Ensure we clean the remnants of any pre-existing build. -DEEPHAVEN_VERSION=$(../../gradlew :printVersion -q) CFLAGS="-I${DHCPP}/include" LDFLAGS="-L${DHCPP}/lib" python3 setup.py build_ext -i +DEEPHAVEN_VERSION=$(../../gradlew :printVersion -q) CPPFLAGS="-I${DHCPP}/include" LDFLAGS="-L${DHCPP}/lib" python3 setup.py build_ext -i ``` #### Install pydeephaven-ticking diff --git a/py/client-ticking/pyClientTickingWheel/entrypoint.sh b/py/client-ticking/pyClientTickingWheel/entrypoint.sh index 413a6a011d7..af3d784a28a 100755 --- a/py/client-ticking/pyClientTickingWheel/entrypoint.sh +++ b/py/client-ticking/pyClientTickingWheel/entrypoint.sh @@ -13,7 +13,7 @@ rm -f ./*.cpp ./*.so PATH="/opt/python/${PYTHON_TAG}/bin:$PATH" MAKEFLAGS="-j${NCPUS}" \ - CFLAGS="-I${DHCPP}/include" \ + CPPFLAGS="-I${DHCPP}/include" \ LDFLAGS="-L${DHCPP}/lib" \ DEEPHAVEN_VERSION="${DEEPHAVEN_VERSION}" \ python setup.py build_ext -i diff --git a/py/embedded-server/build.gradle b/py/embedded-server/build.gradle index e0bc16be9db..2479a7d0b38 100644 --- a/py/embedded-server/build.gradle +++ b/py/embedded-server/build.gradle @@ -27,7 +27,7 @@ dependencies { } } -def testPyClient = Docker.registerDockerTask(project, 'testPyClient') { +def testEmbeddedServer = Docker.registerDockerTask(project, 'testEmbeddedServer') { copyIn { from('tests') { into 'project/tests' @@ -62,4 +62,4 @@ def testPyClient = Docker.registerDockerTask(project, 'testPyClient') { } } -tasks.getByName('check').dependsOn(testPyClient) +tasks.getByName('check').dependsOn(testEmbeddedServer) diff --git a/py/embedded-server/deephaven_server/server.py b/py/embedded-server/deephaven_server/server.py index 6bae0ef5ee6..f888dad73d2 100644 --- a/py/embedded-server/deephaven_server/server.py +++ b/py/embedded-server/deephaven_server/server.py @@ -5,6 +5,7 @@ from typing import List, Optional import sys +import atexit from .start_jvm import start_jvm @@ -167,6 +168,9 @@ def __init__( # Keep a reference to the server so we know it is running Server.instance = self + # On halt, prevent the JVM from writing to sys.out and sys.err + atexit.register(self.j_server.prepareForShutdown) + def start(self): """ Starts the server. Presently once the server is started, it cannot be stopped until the diff --git a/py/embedded-server/java-runtime/src/main/java/io/deephaven/python/server/EmbeddedServer.java b/py/embedded-server/java-runtime/src/main/java/io/deephaven/python/server/EmbeddedServer.java index 7e18ccca1bb..3bc427f8904 100644 --- a/py/embedded-server/java-runtime/src/main/java/io/deephaven/python/server/EmbeddedServer.java +++ b/py/embedded-server/java-runtime/src/main/java/io/deephaven/python/server/EmbeddedServer.java @@ -36,6 +36,7 @@ import io.deephaven.server.session.ObfuscatingErrorTransformerModule; import io.deephaven.server.session.SslConfigModule; import io.deephaven.time.calendar.CalendarsFromConfigurationModule; +import io.deephaven.util.process.ProcessEnvironment; import org.jpy.PyModule; import org.jpy.PyObject; @@ -167,6 +168,15 @@ public void start() throws Exception { Bootstrap.printf("Server started on port %d%n", getPort()); } + public void prepareForShutdown() { + // Stop any DH-started threads and work + ProcessEnvironment.getGlobalShutdownManager().maybeInvokeTasks(); + + // Shut down our wrapped stdout/stderr instances + System.out.close(); + System.err.close(); + } + public int getPort() { return server.server().getPort(); } diff --git a/py/server/deephaven/_table_reader.py b/py/server/deephaven/_table_reader.py index 4c9265745c5..49fa82c9772 100644 --- a/py/server/deephaven/_table_reader.py +++ b/py/server/deephaven/_table_reader.py @@ -9,7 +9,7 @@ import numpy as np from deephaven import update_graph -from deephaven.column import Column +from deephaven.column import ColumnDefinition from deephaven.jcompat import to_sequence from deephaven.numpy import _column_to_numpy_array from deephaven.table import Table @@ -18,7 +18,7 @@ T = TypeVar('T') -def _col_defs(table: Table, cols: Union[str, Sequence[str]]) -> Sequence[Column]: +def _col_defs(table: Table, cols: Union[str, Sequence[str]]) -> Sequence[ColumnDefinition]: if not cols: col_defs = table.columns else: @@ -31,7 +31,7 @@ def _col_defs(table: Table, cols: Union[str, Sequence[str]]) -> Sequence[Column] def _table_reader_all(table: Table, cols: Optional[Union[str, Sequence[str]]] = None, *, - emitter: Callable[[Sequence[Column], jpy.JType], T], row_set: jpy.JType, + emitter: Callable[[Sequence[ColumnDefinition], jpy.JType], T], row_set: jpy.JType, prev: bool = False) -> T: """ Reads all the rows in the given row set of a table. The emitter converts the Java data into a desired Python object. @@ -103,7 +103,7 @@ def _table_reader_all_dict(table: Table, cols: Optional[Union[str, Sequence[str] def _table_reader_chunk(table: Table, cols: Optional[Union[str, Sequence[str]]] = None, *, - emitter: Callable[[Sequence[Column], jpy.JType], Iterable[T]], row_set: jpy.JType, + emitter: Callable[[Sequence[ColumnDefinition], jpy.JType], Iterable[T]], row_set: jpy.JType, chunk_size: int = 2048, prev: bool = False) \ -> Generator[T, None, None]: """ Returns a generator that reads one chunk of rows at a time from the table. The emitter converts the Java chunk @@ -178,7 +178,7 @@ def _table_reader_chunk_dict(table: Table, cols: Optional[Union[str, Sequence[st Raises: ValueError """ - def _emitter(col_defs: Sequence[Column], j_array: jpy.JType) -> Generator[Dict[str, np.ndarray], None, None]: + def _emitter(col_defs: Sequence[ColumnDefinition], j_array: jpy.JType) -> Generator[Dict[str, np.ndarray], None, None]: yield {col_def.name: _column_to_numpy_array(col_def, j_array[i]) for i, col_def in enumerate(col_defs)} return _table_reader_chunk(table, cols, emitter=_emitter, row_set=row_set, chunk_size=chunk_size, prev=prev) @@ -210,9 +210,9 @@ def _table_reader_chunk_tuple(table: Table, cols: Optional[Union[str, Sequence[s Raises: ValueError """ - named_tuple_class = namedtuple(tuple_name, cols or [col.name for col in table.columns], rename=False) + named_tuple_class = namedtuple(tuple_name, cols or table.column_names, rename=False) - def _emitter(col_defs: Sequence[Column], j_array: jpy.JType) -> Generator[Tuple[np.ndarray], None, None]: + def _emitter(col_defs: Sequence[ColumnDefinition], j_array: jpy.JType) -> Generator[Tuple[np.ndarray], None, None]: yield named_tuple_class._make([_column_to_numpy_array(col_def, j_array[i]) for i, col_def in enumerate(col_defs)]) return _table_reader_chunk(table, cols, emitter=_emitter, row_set=table.j_table.getRowSet(), chunk_size=chunk_size, prev=False) @@ -242,7 +242,7 @@ def _table_reader_row_dict(table: Table, cols: Optional[Union[str, Sequence[str] Raises: ValueError """ - def _emitter(col_defs: Sequence[Column], j_array: jpy.JType) -> Iterable[Dict[str, Any]]: + def _emitter(col_defs: Sequence[ColumnDefinition], j_array: jpy.JType) -> Iterable[Dict[str, Any]]: make_dict = lambda values: {col_def.name: value for col_def, value in zip(col_defs, values)} mvs = [memoryview(j_array[i]) if col_def.data_type.is_primitive else j_array[i] for i, col_def in enumerate(col_defs)] return map(make_dict, zip(*mvs)) @@ -275,9 +275,9 @@ def _table_reader_row_tuple(table: Table, cols: Optional[Union[str, Sequence[str Raises: ValueError """ - named_tuple_class = namedtuple(tuple_name, cols or [col.name for col in table.columns], rename=False) + named_tuple_class = namedtuple(tuple_name, cols or table.column_names, rename=False) - def _emitter(col_defs: Sequence[Column], j_array: jpy.JType) -> Iterable[Tuple[Any, ...]]: + def _emitter(col_defs: Sequence[ColumnDefinition], j_array: jpy.JType) -> Iterable[Tuple[Any, ...]]: mvs = [memoryview(j_array[i]) if col_def.data_type.is_primitive else j_array[i] for i, col_def in enumerate(col_defs)] return map(named_tuple_class._make, zip(*mvs)) diff --git a/py/server/deephaven/_udf.py b/py/server/deephaven/_udf.py index dabe1cb2c0b..5163a51b577 100644 --- a/py/server/deephaven/_udf.py +++ b/py/server/deephaven/_udf.py @@ -202,7 +202,11 @@ def prepare_auto_arg_conv(self, encoded_arg_types: str) -> bool: parameters.""" # numba and numpy ufuncs don't need auto argument conversion as they handle the conversion themselves and the # arg types are already verified by the query engine - if isinstance(self.fn, (numba.np.ufunc.gufunc.GUFunc, numba.np.ufunc.dufunc.DUFunc)) or isinstance(self.fn, numpy.ufunc): + if numba: + if isinstance(self.fn, (numba.np.ufunc.gufunc.GUFunc, numba.np.ufunc.dufunc.DUFunc)): + return False + + if isinstance(self.fn, numpy.ufunc): return False if not self.params or not encoded_arg_types: diff --git a/py/server/deephaven/column.py b/py/server/deephaven/column.py index bbb5f5008b9..88839233ff0 100644 --- a/py/server/deephaven/column.py +++ b/py/server/deephaven/column.py @@ -4,16 +4,17 @@ """ This module implements the Column class and functions that work with Columns. """ -from dataclasses import dataclass, field from enum import Enum -from typing import Sequence, Any +from functools import cached_property +from typing import Sequence, Any, Optional +from warnings import warn import jpy import deephaven.dtypes as dtypes from deephaven import DHError -from deephaven.dtypes import DType -from deephaven.dtypes import _instant_array +from deephaven.dtypes import DType, _instant_array, from_jtype +from deephaven._wrapper import JObjectWrapper _JColumnHeader = jpy.get_type("io.deephaven.qst.column.header.ColumnHeader") _JColumn = jpy.get_type("io.deephaven.qst.column.Column") @@ -32,46 +33,151 @@ def __repr__(self): return self.name -@dataclass -class Column: - """ A Column object represents a column definition in a Deephaven Table. """ - name: str - data_type: DType - component_type: DType = None - column_type: ColumnType = ColumnType.NORMAL +class ColumnDefinition(JObjectWrapper): + """A Deephaven column definition.""" - @property - def j_column_header(self): - return _JColumnHeader.of(self.name, self.data_type.qst_type) + j_object_type = _JColumnDefinition + + def __init__(self, j_column_definition: jpy.JType): + self.j_column_definition = j_column_definition @property - def j_column_definition(self): - if hasattr(self.data_type.j_type, 'jclass'): - j_data_type = self.data_type.j_type.jclass - else: - j_data_type = self.data_type.qst_type.clazz() - j_component_type = self.component_type.qst_type.clazz() if self.component_type else None - j_column_type = self.column_type.value - return _JColumnDefinition.fromGenericType(self.name, j_data_type, j_component_type, j_column_type) - - -@dataclass -class InputColumn(Column): - """ An InputColumn represents a user defined column with some input data. """ - input_data: Any = field(default=None) - - def __post_init__(self): + def j_object(self) -> jpy.JType: + return self.j_column_definition + + @cached_property + def name(self) -> str: + """The column name.""" + return self.j_column_definition.getName() + + @cached_property + def data_type(self) -> DType: + """The column data type.""" + return from_jtype(self.j_column_definition.getDataType()) + + @cached_property + def component_type(self) -> Optional[DType]: + """The column component type.""" + return from_jtype(self.j_column_definition.getComponentType()) + + @cached_property + def column_type(self) -> ColumnType: + """The column type.""" + return ColumnType(self.j_column_definition.getColumnType()) + + +class Column(ColumnDefinition): + """A Column object represents a column definition in a Deephaven Table. Deprecated for removal next release, prefer col_def.""" + + def __init__( + self, + name: str, + data_type: DType, + component_type: DType = None, + column_type: ColumnType = ColumnType.NORMAL, + ): + """Deprecated for removal next release, prefer col_def.""" + warn( + "Column is deprecated for removal next release, prefer col_def", + DeprecationWarning, + stacklevel=2, + ) + super().__init__( + col_def(name, data_type, component_type, column_type).j_column_definition + ) + + +class InputColumn: + """An InputColumn represents a user defined column with some input data.""" + + def __init__( + self, + name: str = None, + data_type: DType = None, + component_type: DType = None, + column_type: ColumnType = ColumnType.NORMAL, + input_data: Any = None, + ): + """Creates an InputColumn. + Args: + name (str): the column name + data_type (DType): the column data type + component_type (Optional[DType]): the column component type, None by default + column_type (ColumnType): the column type, NORMAL by default + input_data: Any: the input data, by default is None + + Returns: + a new InputColumn + + Raises: + DHError + """ try: - if self.input_data is None: - self.j_column = _JColumn.empty(self.j_column_header) - else: - if self.data_type.is_primitive: - self.j_column = _JColumn.ofUnsafe(self.name, dtypes.array(self.data_type, self.input_data, - remap=dtypes.null_remap(self.data_type))) - else: - self.j_column = _JColumn.of(self.j_column_header, dtypes.array(self.data_type, self.input_data)) + self._column_definition = col_def( + name, data_type, component_type, column_type + ) + self.j_column = self._to_j_column(input_data) except Exception as e: - raise DHError(e, f"failed to create an InputColumn ({self.name}).") from e + raise DHError(e, f"failed to create an InputColumn ({name}).") from e + + def _to_j_column(self, input_data: Any = None) -> jpy.JType: + if input_data is None: + return _JColumn.empty( + _JColumnHeader.of( + self._column_definition.name, + self._column_definition.data_type.qst_type, + ) + ) + if self._column_definition.data_type.is_primitive: + return _JColumn.ofUnsafe( + self._column_definition.name, + dtypes.array( + self._column_definition.data_type, + input_data, + remap=dtypes.null_remap(self._column_definition.data_type), + ), + ) + return _JColumn.of( + _JColumnHeader.of( + self._column_definition.name, self._column_definition.data_type.qst_type + ), + dtypes.array(self._column_definition.data_type, input_data), + ) + + +def col_def( + name: str, + data_type: DType, + component_type: Optional[DType] = None, + column_type: ColumnType = ColumnType.NORMAL, +) -> ColumnDefinition: + """Creates a ColumnDefinition. + + Args: + name (str): the column name + data_type (DType): the column data type + component_type (Optional[DType]): the column component type, None by default + column_type (ColumnType): the column type, ColumnType.NORMAL by default + + Returns: + a new ColumnDefinition + + Raises: + DHError + """ + try: + return ColumnDefinition( + _JColumnDefinition.fromGenericType( + name, + data_type.j_type.jclass + if hasattr(data_type.j_type, "jclass") + else data_type.qst_type.clazz(), + component_type.qst_type.clazz() if component_type else None, + column_type.value, + ) + ) + except Exception as e: + raise DHError(e, f"failed to create a ColumnDefinition ({name}).") from e def bool_col(name: str, data: Sequence) -> InputColumn: diff --git a/py/server/deephaven/experimental/iceberg.py b/py/server/deephaven/experimental/iceberg.py index 7506bc95a25..16fcd08f37d 100644 --- a/py/server/deephaven/experimental/iceberg.py +++ b/py/server/deephaven/experimental/iceberg.py @@ -8,16 +8,14 @@ from deephaven import DHError from deephaven._wrapper import JObjectWrapper -from deephaven.column import Column -from deephaven.dtypes import DType from deephaven.experimental import s3 +from deephaven.table import Table, TableDefinition, TableDefinitionLike -from deephaven.jcompat import j_table_definition - -from deephaven.table import Table +from deephaven.jcompat import j_hashmap _JIcebergInstructions = jpy.get_type("io.deephaven.iceberg.util.IcebergInstructions") _JIcebergCatalogAdapter = jpy.get_type("io.deephaven.iceberg.util.IcebergCatalogAdapter") +_JIcebergTools = jpy.get_type("io.deephaven.iceberg.util.IcebergTools") # IcebergToolsS3 is an optional library try: @@ -39,14 +37,14 @@ class IcebergInstructions(JObjectWrapper): j_object_type = _JIcebergInstructions def __init__(self, - table_definition: Optional[Union[Dict[str, DType], List[Column]]] = None, + table_definition: Optional[TableDefinitionLike] = None, data_instructions: Optional[s3.S3Instructions] = None, column_renames: Optional[Dict[str, str]] = None): """ Initializes the instructions using the provided parameters. Args: - table_definition (Optional[Union[Dict[str, DType], List[Column], None]]): the table definition; if omitted, + table_definition (Optional[TableDefinitionLike]): the table definition; if omitted, the definition is inferred from the Iceberg schema. Setting a definition guarantees the returned table will have that definition. This is useful for specifying a subset of the Iceberg schema columns. data_instructions (Optional[s3.S3Instructions]): Special instructions for reading data files, useful when @@ -62,7 +60,7 @@ def __init__(self, builder = self.j_object_type.builder() if table_definition is not None: - builder.tableDefinition(j_table_definition(table_definition)) + builder.tableDefinition(TableDefinition(table_definition).j_table_definition) if data_instructions is not None: builder.dataInstructions(data_instructions.j_object) @@ -250,3 +248,74 @@ def adapter_aws_glue( except Exception as e: raise DHError(e, "Failed to build Iceberg Catalog Adapter") from e + +def adapter( + name: Optional[str] = None, + properties: Optional[Dict[str, str]] = None, + hadoopConfig: Optional[Dict[str, str]] = None +) -> IcebergCatalogAdapter: + """ + Create an Iceberg catalog adapter from configuration properties. These properties map to the Iceberg catalog Java + API properties and are used to select the catalog and file IO implementations. + + The minimal set of properties required to create an Iceberg catalog are the following: + - `catalog-impl` or `type` - the Java catalog implementation to use. When providing `catalog-impl`, the + implementing Java class should be provided (e.g. `org.apache.iceberg.rest.RESTCatalog` or + `org.apache.iceberg.aws.glue.GlueCatalog`). Choices for `type` include `hive`, `hadoop`, `rest`, `glue`, + `nessie`, `jdbc`. + - `uri` - the URI of the catalog + + Other common properties include: + - `warehouse` - the root path of the data warehouse. + - `client.region` - the region of the AWS client. + - `s3.access-key-id` - the S3 access key for reading files. + - `s3.secret-access-key` - the S3 secret access key for reading files. + - `s3.endpoint` - the S3 endpoint to connect to. + + Example usage #1 - REST catalog with an S3 backend (using MinIO): + ``` + from deephaven.experimental import iceberg + + adapter = iceberg.adapter_generic(name="generic-adapter", properties={ + "type" : "rest", + "uri" : "http://rest:8181", + "client.region" : "us-east-1", + "s3.access-key-id" : "admin", + "s3.secret-access-key" : "password", + "s3.endpoint" : "http://minio:9000" + }) + ``` + + Example usage #2 - AWS Glue catalog: + ``` + from deephaven.experimental import iceberg + + ## Note: region and credential information are loaded by the catalog from the environment + adapter = iceberg.adapter_generic(name="generic-adapter", properties={ + "type" : "glue", + "uri" : "s3://lab-warehouse/sales", + }); + ``` + + Args: + name (Optional[str]): a descriptive name of the catalog; if omitted the catalog name is inferred from the + catalog URI property. + properties (Optional[Dict[str, str]]): the properties of the catalog to load + hadoopConfig (Optional[Dict[str, str]]): hadoop configuration properties for the catalog to load + + Returns: + IcebergCatalogAdapter: the catalog adapter created from the provided properties + + Raises: + DHError: If unable to build the catalog adapter + """ + + try: + return IcebergCatalogAdapter( + _JIcebergTools.createAdapter( + name, + j_hashmap(properties if properties is not None else {}), + j_hashmap(hadoopConfig if hadoopConfig is not None else {}))) + except Exception as e: + raise DHError(e, "Failed to build Iceberg Catalog Adapter") from e + diff --git a/py/server/deephaven/experimental/s3.py b/py/server/deephaven/experimental/s3.py index db6168aca16..ade4e5f8629 100644 --- a/py/server/deephaven/experimental/s3.py +++ b/py/server/deephaven/experimental/s3.py @@ -1,16 +1,13 @@ # # Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending # -import datetime from typing import Optional, Union import jpy -import numpy as np -import pandas as pd -from deephaven import time, DHError +from deephaven import DHError from deephaven._wrapper import JObjectWrapper -from deephaven.dtypes import Duration +from deephaven.time import DurationLike, to_j_duration # If we move S3 to a permanent module, we should remove this try/except block and just import the types directly. try: @@ -38,10 +35,8 @@ def __init__(self, max_concurrent_requests: Optional[int] = None, read_ahead_count: Optional[int] = None, fragment_size: Optional[int] = None, - connection_timeout: Union[ - Duration, int, str, datetime.timedelta, np.timedelta64, pd.Timedelta, None] = None, - read_timeout: Union[ - Duration, int, str, datetime.timedelta, np.timedelta64, pd.Timedelta, None] = None, + connection_timeout: Optional[DurationLike] = None, + read_timeout: Optional[DurationLike] = None, access_key_id: Optional[str] = None, secret_access_key: Optional[str] = None, anonymous_access: bool = False, @@ -54,19 +49,19 @@ def __init__(self, Args: region_name (str): the region name for reading parquet files. If not provided, the default region will be - picked by the AWS SDK from 'aws.region' system property, "AWS_REGION" environment variable, the - {user.home}/.aws/credentials or {user.home}/.aws/config files, or from EC2 metadata service, if running in - EC2. + picked by the AWS SDK from 'aws.region' system property, "AWS_REGION" environment variable, the + {user.home}/.aws/credentials or {user.home}/.aws/config files, or from EC2 metadata service, if running + in EC2. max_concurrent_requests (int): the maximum number of concurrent requests for reading files, default is 256. read_ahead_count (int): the number of fragments to send asynchronous read requests for while reading the current fragment. Defaults to 32, which means fetch the next 32 fragments in advance when reading the current fragment. fragment_size (int): the maximum size of each fragment to read, defaults to 64 KiB. If there are fewer bytes remaining in the file, the fetched fragment can be smaller. - connection_timeout (Union[Duration, int, str, datetime.timedelta, np.timedelta64, pd.Timedelta]): + connection_timeout (DurationLike): the amount of time to wait when initially establishing a connection before giving up and timing out, can be expressed as an integer in nanoseconds, a time interval string, e.g. "PT00:00:00.001" or "PT1s", or other time duration types. Default to 2 seconds. - read_timeout (Union[Duration, int, str, datetime.timedelta, np.timedelta64, pd.Timedelta]): + read_timeout (DurationLike): the amount of time to wait when reading a fragment before giving up and timing out, can be expressed as an integer in nanoseconds, a time interval string, e.g. "PT00:00:00.001" or "PT1s", or other time duration types. Default to 2 seconds. @@ -111,10 +106,10 @@ def __init__(self, builder.fragmentSize(fragment_size) if connection_timeout is not None: - builder.connectionTimeout(time.to_j_duration(connection_timeout)) + builder.connectionTimeout(to_j_duration(connection_timeout)) if read_timeout is not None: - builder.readTimeout(time.to_j_duration(read_timeout)) + builder.readTimeout(to_j_duration(read_timeout)) if ((access_key_id is not None and secret_access_key is None) or (access_key_id is None and secret_access_key is not None)): diff --git a/py/server/deephaven/jcompat.py b/py/server/deephaven/jcompat.py index d807cb472f3..98e6a04565e 100644 --- a/py/server/deephaven/jcompat.py +++ b/py/server/deephaven/jcompat.py @@ -5,7 +5,8 @@ """ This module provides Java compatibility support including convenience functions to create some widely used Java data structures from corresponding Python ones in order to be able to call Java methods. """ -from typing import Any, Callable, Dict, Iterable, List, Sequence, Set, TypeVar, Union, Optional +from typing import Any, Callable, Dict, Iterable, List, Sequence, Set, TypeVar, Union, Optional, Mapping +from warnings import warn import jpy import numpy as np @@ -14,7 +15,7 @@ from deephaven import dtypes, DHError from deephaven._wrapper import unwrap, wrap_j_object, JObjectWrapper from deephaven.dtypes import DType, _PRIMITIVE_DTYPE_NULL_MAP -from deephaven.column import Column +from deephaven.column import ColumnDefinition _NULL_BOOLEAN_AS_BYTE = jpy.get_type("io.deephaven.util.BooleanUtils").NULL_BOOLEAN_AS_BYTE _JPrimitiveArrayConversionUtility = jpy.get_type("io.deephaven.integrations.common.PrimitiveArrayConversionUtility") @@ -327,11 +328,20 @@ def _j_array_to_series(dtype: DType, j_array: jpy.JType, conv_null: bool) -> pd. return s -def j_table_definition(table_definition: Union[Dict[str, DType], List[Column], None]) -> Optional[jpy.JType]: - """Produce a Deephaven TableDefinition from user input. +# Note: unable to import TableDefinitionLike due to circular ref (table.py -> agg.py -> jcompat.py) +def j_table_definition( + table_definition: Union[ + "TableDefinition", + Mapping[str, dtypes.DType], + Iterable[ColumnDefinition], + jpy.JType, + None, + ], +) -> Optional[jpy.JType]: + """Deprecated for removal next release, prefer TableDefinition. Produce a Deephaven TableDefinition from user input. Args: - table_definition (Union[Dict[str, DType], List[Column], None]): the table definition as a dictionary of column + table_definition (Optional[TableDefinitionLike]): the table definition as a dictionary of column names and their corresponding data types or a list of Column objects Returns: @@ -340,22 +350,18 @@ def j_table_definition(table_definition: Union[Dict[str, DType], List[Column], N Raises: DHError """ - if table_definition is None: - return None - elif isinstance(table_definition, Dict): - return _JTableDefinition.of( - [ - Column(name=name, data_type=dtype).j_column_definition - for name, dtype in table_definition.items() - ] - ) - elif isinstance(table_definition, List): - return _JTableDefinition.of( - [col.j_column_definition for col in table_definition] - ) - else: - raise DHError(f"Unexpected table_definition type: {type(table_definition)}") + warn( + "j_table_definition is deprecated for removal next release, prefer TableDefinition", + DeprecationWarning, + stacklevel=2, + ) + from deephaven.table import TableDefinition + return ( + TableDefinition(table_definition).j_table_definition + if table_definition + else None + ) class AutoCloseable(JObjectWrapper): """A context manager wrapper to allow Java AutoCloseable to be used in with statements. diff --git a/py/server/deephaven/json/__init__.py b/py/server/deephaven/json/__init__.py index 77957733df8..3cc9b986e33 100644 --- a/py/server/deephaven/json/__init__.py +++ b/py/server/deephaven/json/__init__.py @@ -7,38 +7,45 @@ example, the JSON object .. code-block:: json + { "name": "Foo", "age": 42, "location": { "lat": 45.018269, "lon": -93.473892 } } can be modelled with the dictionary .. code-block:: python + { "name": str, "age": int, "location": { "lat": float, "lon": float } } Notice that this allows for the nested modelling of JSON values. Other common constructions involve the modelling of JSON arrays. For example, a variable-length JSON array where the elements are the same type .. code-block:: json + [42, 31, ..., 12345] can be modelled with a single-element list containing the element type .. code-block:: python + [ int ] If the JSON array is a fixed size and each elements' type is known, for example .. code-block:: json + ["Foo", 42, [45.018269, -93.473892]] can be modelled with a tuple containing each type .. code-block:: python + (str, int, (float, float)) Notice again that this allows for the nested modelling of JSON values. Of course, these constructions can be all be used together. For example, the JSON object .. code-block:: json + { "name": "Foo", "locations": [ @@ -51,6 +58,7 @@ can be modelled as .. code-block:: python + {"name": str, "locations": [(float, float)]} See the methods in this module more more details on modelling JSON values. @@ -64,7 +72,7 @@ from deephaven import dtypes from deephaven._wrapper import JObjectWrapper -from deephaven.time import to_j_instant +from deephaven.time import InstantLike, to_j_instant from deephaven._jpy import strict_cast @@ -200,7 +208,9 @@ def json_val(json_value_type: JsonValueType) -> JsonValue: class RepeatedFieldBehavior(Enum): """ The behavior to use when a repeated field is encountered in a JSON object. For example, + .. code-block:: json + { "foo": 42, "foo": 43 @@ -222,6 +232,7 @@ class ObjectField: simplify by just using the JsonValueType. For example, .. code-block:: python + { "name": ObjectField(str), "age": ObjectField(int), @@ -230,6 +241,7 @@ class ObjectField: could be simplified to .. code-block:: python + { "name": str, "age": int, @@ -294,22 +306,26 @@ def object_val( """Creates an object value. For example, the JSON object .. code-block:: json + { "name": "foo", "age": 42 } might be modelled as the object type .. code-block:: python + object_val({ "name": str, "age": int }) In contexts where the user needs to create a JsonValueType and isn't changing any default values, the user can simplify by using a Dict[str, Union[JsonValueType, ObjectField]]. For example, .. code-block:: python + some_method(object_val({ "name": str, "age": int })) could be simplified to .. code-block:: python + some_method({ "name": str, "age": int }) Args: @@ -356,9 +372,11 @@ def typed_object_val( """Creates a type-discriminated object value. For example, the JSON objects .. code-block:: json + { "type": "trade", "symbol": "FOO", "price": 70.03, "size": 42 } .. code-block:: json + { "type": "quote", "symbol": "BAR", "bid": 10.01, "ask": 10.05 } might be modelled as a type-discriminated object with "type" as the type field, "symbol" as a shared field, with a @@ -366,6 +384,7 @@ def typed_object_val( field: .. code-block:: python + typed_object_val( "type", {"symbol": str}, @@ -432,22 +451,26 @@ def array_val( """Creates a "typed array", where all elements of the array have the same element type. For example, the JSON array .. code-block:: json + [1, 42, 43, 13] might be modelled as an array of ints .. code-block:: python + array_val(int) In contexts where the user needs to create a JsonValueType and isn't changing any default values, the user can simplify by using a list with a single element type. For example, .. code-block:: python + some_method(array_val(element)) could be simplified to .. code-block:: python + some_method([element]) Args: @@ -474,6 +497,7 @@ def object_entries_val( variable and all the values types are the same. For example, the JSON object .. code-block:: json + { "foo": 1, "bar": 42, @@ -485,6 +509,7 @@ def object_entries_val( might be modelled as the object kv type .. code-block:: python + object_entries_val(int) Args: @@ -511,16 +536,19 @@ def tuple_val( """Creates a tuple value. For example, the JSON array .. code-block:: json + ["foo", 42, 5.72] might be modelled as the tuple type .. code-block:: python + tuple_val((str, int, float)) To provide meaningful names, a dictionary can be used: .. code-block:: python + tuple_val({"name": str, "age": int, "height": float}) otherwise, default names based on the indexes of the values will be used. @@ -529,11 +557,13 @@ def tuple_val( names, the user can simplify passing through a python tuple type. For example, .. code-block:: python + some_method(tuple_val((tuple_type_1, tuple_type_2))) could be simplified to .. code-block:: python + some_method((tuple_type_1, tuple_type_2)) Args: @@ -571,22 +601,26 @@ def bool_val( """Creates a bool value. For example, the JSON boolean .. code-block:: json + True might be modelled as the bool type .. code-block:: python + bool_val() In contexts where the user needs to create a JsonValueType and isn't changing any default values, the user can simplify by using the python built-in bool type. For example, .. code-block:: python + some_method(bool_val()) could be simplified to .. code-block:: python + some_method(bool) Args: @@ -623,11 +657,13 @@ def char_val( """Creates a char value. For example, the JSON string .. code-block:: json + "F" might be modelled as the char type .. code-block:: python + char_val() Args: @@ -664,11 +700,13 @@ def byte_val( """Creates a byte (signed 8-bit) value. For example, the JSON integer .. code-block:: json + 42 might be modelled as the byte type .. code-block:: python + byte_val() Args: @@ -709,11 +747,13 @@ def short_val( """Creates a short (signed 16-bit) value. For example, the JSON integer .. code-block:: json + 30000 might be modelled as the short type .. code-block:: python + short_val() Args: @@ -754,11 +794,13 @@ def int_val( """Creates an int (signed 32-bit) value. For example, the JSON integer .. code-block:: json + 100000 might be modelled as the int type .. code-block:: python + int_val() Args: @@ -799,22 +841,26 @@ def long_val( """Creates a long (signed 64-bit) value. For example, the JSON integer .. code-block:: json + 8000000000 might be modelled as the long type .. code-block:: python + long_val() In contexts where the user needs to create a JsonValueType and isn't changing any default values, the user can simplify by using the python built-in long type. For example, .. code-block:: python + some_method(long_val()) could be simplified to .. code-block:: python + some_method(int) Args: @@ -854,11 +900,13 @@ def float_val( """Creates a float (signed 32-bit) value. For example, the JSON decimal .. code-block:: json + 42.42 might be modelled as the float type .. code-block:: python + float_val() Args: @@ -897,22 +945,26 @@ def double_val( """Creates a double (signed 64-bit) value. For example, the JSON decimal .. code-block:: json + 42.42424242 might be modelled as the double type .. code-block:: python + double_val() In contexts where the user needs to create a JsonValueType and isn't changing any default values, the user can simplify by using the python built-in float type. For example, .. code-block:: python + some_method(double_val()) could be simplified to .. code-block:: python + some_method(float) Args: @@ -953,22 +1005,26 @@ def string_val( """Creates a String value. For example, the JSON string .. code-block:: json + "Hello, world!" might be modelled as the string type .. code-block:: python + string_val() In contexts where the user needs to create a JsonValueType and isn't changing any default values, the user can simplify by using the python built-in str type. For example, .. code-block:: python + some_method(string_val()) could be simplified to .. code-block:: python + some_method(str) Args: @@ -1000,44 +1056,49 @@ def string_val( return JsonValue(builder.build()) -# TODO(deephaven-core#5269): Create deephaven.time time-type aliases def instant_val( allow_missing: bool = True, allow_null: bool = True, number_format: Literal[None, "s", "ms", "us", "ns"] = None, allow_decimal: bool = False, - on_missing: Optional[Any] = None, - on_null: Optional[Any] = None, + on_missing: Optional[InstantLike] = None, + on_null: Optional[InstantLike] = None, ) -> JsonValue: """Creates an Instant value. For example, the JSON string .. code-block:: json + "2009-02-13T23:31:30.123456789Z" might be modelled as the Instant type .. code-block:: python + instant_val() In another example, the JSON decimal .. code-block:: json + 1234567890.123456789 might be modelled as the Instant type .. code-block:: python + instant_val(number_format="s", allow_decimal=True) In contexts where the user needs to create a JsonValueType and isn't changing any default values, the user can simplify by using the python datetime.datetime type. For example, .. code-block:: python + some_method(instant_val()) could be simplified to .. code-block:: python + some_method(datetime.datetime) Args: @@ -1048,8 +1109,8 @@ def instant_val( the epoch. When not set, a JSON string in the ISO-8601 format is expected. allow_decimal (bool): if the Instant value is allowed to be a JSON decimal type, default is False. Only valid when number_format is specified. - on_missing (Optional[Any]): the value to use when the JSON value is missing and allow_missing is True, default is None. - on_null (Optional[Any]): the value to use when the JSON value is null and allow_null is True, default is None. + on_missing (Optional[InstantLike]): the value to use when the JSON value is missing and allow_missing is True, default is None. + on_null (Optional[InstantLike]): the value to use when the JSON value is null and allow_null is True, default is None. Returns: the Instant value @@ -1106,11 +1167,13 @@ def big_integer_val( """Creates a BigInteger value. For example, the JSON integer .. code-block:: json + 123456789012345678901 might be modelled as the BigInteger type .. code-block:: python + big_integer_val() Args: @@ -1150,11 +1213,13 @@ def big_decimal_val( """Creates a BigDecimal value. For example, the JSON decimal .. code-block:: json + 123456789012345678901.42 might be modelled as the BigDecimal type .. code-block:: python + big_decimal_val() Args: @@ -1207,11 +1272,13 @@ def skip_val( This may be useful in combination with an object type where allow_unknown_fields=False. For example, the JSON object .. code-block:: json + { "name": "foo", "age": 42 } might be modelled as the object type .. code-block:: python + object_val({ "name": str, "age": skip_val() }, allow_unknown_fields=False) Args: diff --git a/py/server/deephaven/learn/__init__.py b/py/server/deephaven/learn/__init__.py index 54261ad1d21..45439cb6bf0 100644 --- a/py/server/deephaven/learn/__init__.py +++ b/py/server/deephaven/learn/__init__.py @@ -71,7 +71,7 @@ def _validate(inputs: Input, outputs: Output, table: Table): input_columns_list = [input_.input.getColNames()[i] for input_ in inputs for i in range(len(input_.input.getColNames()))] input_columns = set(input_columns_list) - table_columns = {col.name for col in table.columns} + table_columns = set(table.definition.keys()) if table_columns >= input_columns: if outputs is not None: output_columns_list = [output.output.getColName() for output in outputs] @@ -99,7 +99,7 @@ def _create_non_conflicting_col_name(table: Table, base_col_name: str) -> str: Returns: column name that is not present in the table. """ - table_col_names = set([col.name for col in table.columns]) + table_col_names = set(table.definition.keys()) if base_col_name not in table_col_names: return base_col_name else: diff --git a/py/server/deephaven/numpy.py b/py/server/deephaven/numpy.py index c87f24ea40c..33dcef3bc14 100644 --- a/py/server/deephaven/numpy.py +++ b/py/server/deephaven/numpy.py @@ -8,10 +8,10 @@ import jpy import numpy as np -from deephaven.dtypes import DType, BusinessCalendar +from deephaven.dtypes import BusinessCalendar from deephaven import DHError, dtypes, new_table -from deephaven.column import Column, InputColumn +from deephaven.column import InputColumn, ColumnDefinition from deephaven.dtypes import DType from deephaven.jcompat import _j_array_to_numpy_array from deephaven.table import Table @@ -27,11 +27,11 @@ def _to_column_name(name: str) -> str: return re.sub(r"\s+", "_", tmp_name) -def _column_to_numpy_array(col_def: Column, j_array: jpy.JType) -> np.ndarray: +def _column_to_numpy_array(col_def: ColumnDefinition, j_array: jpy.JType) -> np.ndarray: """ Produces a numpy array from the given Java array and the Table column definition. Args: - col_def (Column): the column definition + col_def (ColumnDefinition): the column definition j_array (jpy.JType): the Java array Returns: @@ -48,7 +48,7 @@ def _column_to_numpy_array(col_def: Column, j_array: jpy.JType) -> np.ndarray: raise DHError(e, f"failed to create a numpy array for the column {col_def.name}") from e -def _columns_to_2d_numpy_array(col_def: Column, j_arrays: List[jpy.JType]) -> np.ndarray: +def _columns_to_2d_numpy_array(col_def: ColumnDefinition, j_arrays: List[jpy.JType]) -> np.ndarray: """ Produces a 2d numpy array from the given Java arrays of the same component type and the Table column definition """ try: @@ -95,15 +95,15 @@ def to_numpy(table: Table, cols: List[str] = None) -> np.ndarray: if table.is_refreshing: table = table.snapshot() - col_def_dict = {col.name: col for col in table.columns} + table_def = table.definition if not cols: - cols = list(col_def_dict.keys()) + cols = list(table_def.keys()) else: - diff_set = set(cols) - set(col_def_dict.keys()) + diff_set = set(cols) - set(table_def.keys()) if diff_set: raise DHError(message=f"columns - {list(diff_set)} not found") - col_defs = [col_def_dict[col] for col in cols] + col_defs = [table_def[col] for col in cols] if len(set([col_def.data_type for col_def in col_defs])) != 1: raise DHError(message="columns must be of the same data type.") diff --git a/py/server/deephaven/pandas.py b/py/server/deephaven/pandas.py index d946de5e391..c937bc47b94 100644 --- a/py/server/deephaven/pandas.py +++ b/py/server/deephaven/pandas.py @@ -11,7 +11,7 @@ import pyarrow as pa from deephaven import DHError, new_table, dtypes, arrow -from deephaven.column import Column +from deephaven.column import ColumnDefinition from deephaven.constants import NULL_BYTE, NULL_SHORT, NULL_INT, NULL_LONG, NULL_FLOAT, NULL_DOUBLE, NULL_CHAR from deephaven.jcompat import _j_array_to_series from deephaven.numpy import _make_input_column @@ -22,12 +22,12 @@ _is_dtype_backend_supported = pd.__version__ >= "2.0.0" -def _column_to_series(table: Table, col_def: Column, conv_null: bool) -> pd.Series: +def _column_to_series(table: Table, col_def: ColumnDefinition, conv_null: bool) -> pd.Series: """Produce a copy of the specified column as a pandas.Series object. Args: table (Table): the table - col_def (Column): the column definition + col_def (ColumnDefinition): the column definition conv_null (bool): whether to check for Deephaven nulls in the data and automatically replace them with pd.NA. @@ -133,17 +133,17 @@ def to_pandas(table: Table, cols: List[str] = None, if table.is_refreshing: table = table.snapshot() - col_def_dict = {col.name: col for col in table.columns} + table_def = table.definition if not cols: - cols = list(col_def_dict.keys()) + cols = list(table_def.keys()) else: - diff_set = set(cols) - set(col_def_dict.keys()) + diff_set = set(cols) - set(table_def.keys()) if diff_set: raise DHError(message=f"columns - {list(diff_set)} not found") data = {} for col in cols: - series = _column_to_series(table, col_def_dict[col], conv_null) + series = _column_to_series(table, table_def[col], conv_null) data[col] = series return pd.DataFrame(data=data, columns=cols, copy=False) @@ -201,12 +201,15 @@ def _map_na(array: [np.ndarray, pd.api.extensions.ExtensionArray]): return array -def to_table(df: pd.DataFrame, cols: List[str] = None) -> Table: +def to_table(df: pd.DataFrame, cols: List[str] = None, infer_objects: bool = True) -> Table: """Creates a new table from a pandas DataFrame. Args: df (DataFrame): the pandas DataFrame instance cols (List[str]): the dataframe column names, default is None which means including all columns in the DataFrame + infer_objects (bool): whether to infer the best possible types for columns of the generic 'object' type in the + DataFrame before creating the table, default is True. When True, pandas convert_dtypes() method is called to + perform the conversion. Note that any conversion will make a copy of the data. Returns: a Deephaven table @@ -222,11 +225,19 @@ def to_table(df: pd.DataFrame, cols: List[str] = None) -> Table: if diff_set: raise DHError(message=f"columns - {list(diff_set)} not found") + # if infer_objects is True, convert object dtypes to the best possible types supporting pd.NA + converted_df = df + if infer_objects: + converted_df = df[cols] + for col in cols: + if df.dtypes[col] == object: + converted_df[col] = df[col].convert_dtypes() + # if any arrow backed column is present, create a pyarrow table first, then upload to DH, if error occurs, fall # back to the numpy-array based approach - if _is_dtype_backend_supported and any(isinstance(df[col].dtype, pd.ArrowDtype) for col in cols): + if _is_dtype_backend_supported and any(isinstance(converted_df[col].dtype, pd.ArrowDtype) for col in cols): try: - pa_table = pa.Table.from_pandas(df=df, columns=cols) + pa_table = pa.Table.from_pandas(df=converted_df, columns=cols) dh_table = arrow.to_table(pa_table) return dh_table except: @@ -235,9 +246,9 @@ def to_table(df: pd.DataFrame, cols: List[str] = None) -> Table: try: input_cols = [] for col in cols: - np_array = df.get(col).values - if isinstance(df.dtypes[col], pd.CategoricalDtype): - dtype = df.dtypes[col].categories.dtype + np_array = converted_df.get(col).values + if isinstance(converted_df.dtypes[col], pd.CategoricalDtype): + dtype = converted_df.dtypes[col].categories.dtype else: dtype = np_array.dtype dh_dtype = dtypes.from_np_dtype(dtype) diff --git a/py/server/deephaven/parquet.py b/py/server/deephaven/parquet.py index 61614c37061..188a3be9bdb 100644 --- a/py/server/deephaven/parquet.py +++ b/py/server/deephaven/parquet.py @@ -11,10 +11,8 @@ import jpy from deephaven import DHError -from deephaven.column import Column -from deephaven.dtypes import DType -from deephaven.jcompat import j_array_list, j_table_definition -from deephaven.table import Table, PartitionedTable +from deephaven.jcompat import j_array_list +from deephaven.table import Table, TableDefinition, TableDefinitionLike, PartitionedTable from deephaven.experimental import s3 _JParquetTools = jpy.get_type("io.deephaven.parquet.table.ParquetTools") @@ -69,7 +67,7 @@ def _build_parquet_instructions( generate_metadata_files: Optional[bool] = None, base_name: Optional[str] = None, file_layout: Optional[ParquetFileLayout] = None, - table_definition: Optional[Union[Dict[str, DType], List[Column]]] = None, + table_definition: Optional[TableDefinitionLike] = None, index_columns: Optional[Sequence[Sequence[str]]] = None, special_instructions: Optional[s3.S3Instructions] = None, ): @@ -135,7 +133,7 @@ def _build_parquet_instructions( builder.setFileLayout(_j_file_layout(file_layout)) if table_definition is not None: - builder.setTableDefinition(j_table_definition(table_definition)) + builder.setTableDefinition(TableDefinition(table_definition).j_table_definition) if index_columns: builder.addAllIndexColumns(_j_list_of_list_of_string(index_columns)) @@ -166,7 +164,7 @@ def read( is_legacy_parquet: bool = False, is_refreshing: bool = False, file_layout: Optional[ParquetFileLayout] = None, - table_definition: Union[Dict[str, DType], List[Column], None] = None, + table_definition: Optional[TableDefinitionLike] = None, special_instructions: Optional[s3.S3Instructions] = None, ) -> Table: """ Reads in a table from a single parquet, metadata file, or directory with recognized layout. @@ -179,7 +177,7 @@ def read( is_refreshing (bool): if the parquet data represents a refreshing source file_layout (Optional[ParquetFileLayout]): the parquet file layout, by default None. When None, the layout is inferred. - table_definition (Union[Dict[str, DType], List[Column], None]): the table definition, by default None. When None, + table_definition (Optional[TableDefinitionLike]): the table definition, by default None. When None, the definition is inferred from the parquet file(s). Setting a definition guarantees the returned table will have that definition. This is useful for bootstrapping purposes when the initially partitioned directory is empty and is_refreshing=True. It is also useful for specifying a subset of the parquet definition. @@ -235,7 +233,7 @@ def delete(path: str) -> None: def write( table: Table, path: str, - table_definition: Optional[Union[Dict[str, DType], List[Column]]] = None, + table_definition: Optional[TableDefinitionLike] = None, col_instructions: Optional[List[ColumnInstruction]] = None, compression_codec_name: Optional[str] = None, max_dictionary_keys: Optional[int] = None, @@ -252,7 +250,7 @@ def write( path (str): the destination file path or URI; the file name should end in a ".parquet" extension. If the path includes any non-existing directories, they are created. If there is an error, any intermediate directories previously created are removed; note this makes this method unsafe for concurrent use - table_definition (Optional[Union[Dict[str, DType], List[Column]]): the table definition to use for writing, + table_definition (Optional[TableDefinitionLike]): the table definition to use for writing, instead of the definitions implied by the table. Default is None, which means use the column definitions implied by the table. This definition can be used to skip some columns or add additional columns with null values. @@ -302,7 +300,7 @@ def write( def write_partitioned( table: Union[Table, PartitionedTable], destination_dir: str, - table_definition: Optional[Union[Dict[str, DType], List[Column]]] = None, + table_definition: Optional[TableDefinitionLike] = None, col_instructions: Optional[List[ColumnInstruction]] = None, compression_codec_name: Optional[str] = None, max_dictionary_keys: Optional[int] = None, @@ -324,7 +322,7 @@ def write_partitioned( destination_dir (str): The path or URI to the destination root directory in which the partitioned parquet data will be stored in a nested directory structure format. Non-existing directories in the provided path will be created. - table_definition (Optional[Union[Dict[str, DType], List[Column]]): the table definition to use for writing, + table_definition (Optional[TableDefinitionLike]): the table definition to use for writing, instead of the definitions implied by the table. Default is None, which means use the column definitions implied by the table. This definition can be used to skip some columns or add additional columns with null values. @@ -388,7 +386,7 @@ def write_partitioned( def batch_write( tables: List[Table], paths: List[str], - table_definition: Optional[Union[Dict[str, DType], List[Column]]] = None, + table_definition: Optional[TableDefinitionLike] = None, col_instructions: Optional[List[ColumnInstruction]] = None, compression_codec_name: Optional[str] = None, max_dictionary_keys: Optional[int] = None, @@ -407,7 +405,7 @@ def batch_write( paths (List[str]): the destination paths or URIs. Any non-existing directories in the paths provided are created. If there is an error, any intermediate directories previously created are removed; note this makes this method unsafe for concurrent use - table_definition (Optional[Union[Dict[str, DType], List[Column]]]): the table definition to use for writing. + table_definition (Optional[TableDefinitionLike]): the table definition to use for writing. This definition can be used to skip some columns or add additional columns with null values. Default is None, which means if all tables have the same definition, use the common table definition implied by the tables. Otherwise, this parameter must be specified. diff --git a/py/server/deephaven/replay.py b/py/server/deephaven/replay.py index 78c8e018af1..b578c5e3d66 100644 --- a/py/server/deephaven/replay.py +++ b/py/server/deephaven/replay.py @@ -4,16 +4,12 @@ """ This module provides support for replaying historical data. """ -from typing import Union import jpy -import datetime -import numpy as np -import pandas as pd - -from deephaven import dtypes, DHError, time +from deephaven import DHError, time from deephaven._wrapper import JObjectWrapper from deephaven.table import Table +from deephaven.time import InstantLike _JReplayer = jpy.get_type("io.deephaven.engine.table.impl.replay.Replayer") @@ -27,14 +23,13 @@ class TableReplayer(JObjectWrapper): j_object_type = _JReplayer - def __init__(self, start_time: Union[dtypes.Instant, int, str, datetime.datetime, np.datetime64, pd.Timestamp], - end_time: Union[dtypes.Instant, int, str, datetime.datetime, np.datetime64, pd.Timestamp]): + def __init__(self, start_time: InstantLike, end_time: InstantLike): """Initializes the replayer. Args: - start_time (Union[dtypes.Instant, int, str, datetime.datetime, np.datetime64, pd.Timestamp]): + start_time (InstantLike): replay start time. Integer values are nanoseconds since the Epoch. - end_time (Union[dtypes.Instant, int, str, datetime.datetime, np.datetime64, pd.Timestamp]): + end_time (InstantLike): replay end time. Integer values are nanoseconds since the Epoch. Raises: diff --git a/py/server/deephaven/stream/kafka/consumer.py b/py/server/deephaven/stream/kafka/consumer.py index 01d33e7463b..abfe1a4f818 100644 --- a/py/server/deephaven/stream/kafka/consumer.py +++ b/py/server/deephaven/stream/kafka/consumer.py @@ -9,11 +9,11 @@ from deephaven import dtypes from deephaven._wrapper import JObjectWrapper -from deephaven.column import Column +from deephaven.column import col_def from deephaven.dherror import DHError from deephaven.dtypes import DType from deephaven.jcompat import j_hashmap, j_properties, j_array_list -from deephaven.table import Table, PartitionedTable +from deephaven.table import Table, TableDefinition, TableDefinitionLike, PartitionedTable _JKafkaTools = jpy.get_type("io.deephaven.kafka.KafkaTools") _JKafkaTools_Consume = jpy.get_type("io.deephaven.kafka.KafkaTools$Consume") @@ -427,13 +427,13 @@ def avro_spec( raise DHError(e, "failed to create a Kafka key/value spec") from e -def json_spec(col_defs: Union[Dict[str, DType], List[Tuple[str, DType]]], mapping: Dict = None) -> KeyValueSpec: +def json_spec(col_defs: Union[TableDefinitionLike, List[Tuple[str, DType]]], mapping: Dict = None) -> KeyValueSpec: """Creates a spec for how to use JSON data when consuming a Kafka stream to a Deephaven table. Args: - col_defs (Union[Dict[str, DType], List[Tuple[str, DType]]): the column definitions, either a map of column - names and Deephaven types, or a list of tuples with two elements, a string for column name and a Deephaven - type for column data type. + col_defs (Union[TableDefinitionLike, List[Tuple[str, DType]]): the table definition, preferably specified as + TableDefinitionLike. A list of tuples with two elements, a string for column name and a Deephaven type for + column data type also works, but is deprecated for removal. mapping (Dict): a dict mapping JSON fields to column names defined in the col_defs argument. Fields starting with a '/' character are interpreted as a JSON Pointer (see RFC 6901, ISSN: 2070-1721 for details, essentially nested fields are represented like "/parent/nested"). @@ -448,10 +448,20 @@ def json_spec(col_defs: Union[Dict[str, DType], List[Tuple[str, DType]]], mappin DHError """ try: - if isinstance(col_defs, dict): - col_defs = [Column(k, v).j_column_definition for k, v in col_defs.items()] + try: + table_def = TableDefinition(col_defs) + except DHError: + table_def = None + + if table_def: + col_defs = [col.j_column_definition for col in table_def.values()] else: - col_defs = [Column(*t).j_column_definition for t in col_defs] + warn( + 'json_spec col_defs for List[Tuple[str, DType]] is deprecated for removal, prefer TableDefinitionLike', + DeprecationWarning, + stacklevel=2, + ) + col_defs = [col_def(*t).j_column_definition for t in col_defs] if mapping is None: return KeyValueSpec(j_spec=_JKafkaTools_Consume.jsonSpec(col_defs)) diff --git a/py/server/deephaven/stream/table_publisher.py b/py/server/deephaven/stream/table_publisher.py index a6c65f47885..57ead700e79 100644 --- a/py/server/deephaven/stream/table_publisher.py +++ b/py/server/deephaven/stream/table_publisher.py @@ -8,11 +8,9 @@ from typing import Callable, Dict, Optional, Tuple, Union, List from deephaven._wrapper import JObjectWrapper -from deephaven.column import Column -from deephaven.dtypes import DType from deephaven.execution_context import get_exec_ctx -from deephaven.jcompat import j_lambda, j_runnable, j_table_definition -from deephaven.table import Table +from deephaven.jcompat import j_lambda, j_runnable +from deephaven.table import Table, TableDefinition, TableDefinitionLike from deephaven.update_graph import UpdateGraph _JTableDefinition = jpy.get_type("io.deephaven.engine.table.TableDefinition") @@ -75,7 +73,7 @@ def publish_failure(self, failure: Exception) -> None: def table_publisher( name: str, - col_defs: Union[Dict[str, DType], List[Column]], + col_defs: TableDefinitionLike, on_flush_callback: Optional[Callable[[TablePublisher], None]] = None, on_shutdown_callback: Optional[Callable[[], None]] = None, update_graph: Optional[UpdateGraph] = None, @@ -85,7 +83,7 @@ def table_publisher( Args: name (str): the name, used for logging - col_defs (Dict[str, DType]): the column definitions for the resulting blink table + col_defs (TableDefinitionLike): the table definition for the resulting blink table on_flush_callback (Optional[Callable[[TablePublisher], None]]): the on-flush callback, if present, is called once at the beginning of each update graph cycle. This is a pattern that allows publishers to add any data they may have been batching. Do note though, this blocks the update cycle from proceeding, so @@ -107,7 +105,7 @@ def adapt_callback(_table_publisher: jpy.JType): j_table_publisher = _JTablePublisher.of( name, - j_table_definition(col_defs), + TableDefinition(col_defs).j_table_definition, j_lambda(adapt_callback, _JConsumer, None) if on_flush_callback else None, j_runnable(on_shutdown_callback) if on_shutdown_callback else None, (update_graph or get_exec_ctx().update_graph).j_update_graph, diff --git a/py/server/deephaven/table.py b/py/server/deephaven/table.py index 26557db18a1..e02307f647e 100644 --- a/py/server/deephaven/table.py +++ b/py/server/deephaven/table.py @@ -11,11 +11,13 @@ import inspect from enum import Enum from enum import auto +from functools import cached_property from typing import Any, Optional, Callable, Dict, Generator, Tuple, Literal -from typing import Sequence, List, Union, Protocol +from typing import Sequence, List, Union, Protocol, Mapping, Iterable import jpy import numpy as np +import sys from deephaven import DHError from deephaven import dtypes @@ -23,7 +25,7 @@ from deephaven._wrapper import JObjectWrapper from deephaven._wrapper import unwrap from deephaven.agg import Aggregation -from deephaven.column import Column, ColumnType +from deephaven.column import col_def, ColumnDefinition from deephaven.filters import Filter, and_, or_ from deephaven.jcompat import j_unary_operator, j_binary_operator, j_map_to_dict, j_hashmap from deephaven.jcompat import to_sequence, j_array_list @@ -407,19 +409,133 @@ def _sort_column(col, dir_): _JColumnName.of(col))) -def _td_to_columns(table_definition): - cols = [] - j_cols = table_definition.getColumnsArray() - for j_col in j_cols: - cols.append( - Column( - name=j_col.getName(), - data_type=dtypes.from_jtype(j_col.getDataType()), - component_type=dtypes.from_jtype(j_col.getComponentType()), - column_type=ColumnType(j_col.getColumnType()), +if sys.version_info >= (3, 10): + from typing import TypeAlias # novermin + + TableDefinitionLike: TypeAlias = Union[ + "TableDefinition", + Mapping[str, dtypes.DType], + Iterable[ColumnDefinition], + jpy.JType, + ] + """A Union representing objects that can be coerced into a TableDefinition.""" +else: + TableDefinitionLike = Union[ + "TableDefinition", + Mapping[str, dtypes.DType], + Iterable[ColumnDefinition], + jpy.JType, + ] + """A Union representing objects that can be coerced into a TableDefinition.""" + + +class TableDefinition(JObjectWrapper, Mapping): + """A Deephaven table definition, as a mapping from column name to ColumnDefinition.""" + + j_object_type = _JTableDefinition + + @staticmethod + def _to_j_table_definition(table_definition: TableDefinitionLike) -> jpy.JType: + if isinstance(table_definition, TableDefinition): + return table_definition.j_table_definition + if isinstance(table_definition, _JTableDefinition): + return table_definition + if isinstance(table_definition, Mapping): + for name in table_definition.keys(): + if not isinstance(name, str): + raise DHError( + f"Expected TableDefinitionLike Mapping to contain str keys, found type {type(name)}" + ) + for data_type in table_definition.values(): + if not isinstance(data_type, dtypes.DType): + raise DHError( + f"Expected TableDefinitionLike Mapping to contain DType values, found type {type(data_type)}" + ) + column_definitions = [ + col_def(name, data_type) for name, data_type in table_definition.items() + ] + elif isinstance(table_definition, Iterable): + for column_definition in table_definition: + if not isinstance(column_definition, ColumnDefinition): + raise DHError( + f"Expected TableDefinitionLike Iterable to contain ColumnDefinition values, found type {type(column_definition)}" + ) + column_definitions = table_definition + else: + raise DHError( + f"Unexpected TableDefinitionLike type: {type(table_definition)}" ) + return _JTableDefinition.of( + [col.j_column_definition for col in column_definitions] ) - return cols + + def __init__(self, table_definition: TableDefinitionLike): + """Construct a TableDefinition. + + Args: + table_definition (TableDefinitionLike): The table definition like object + + Returns: + A new TableDefinition + + Raises: + DHError + """ + self.j_table_definition = TableDefinition._to_j_table_definition( + table_definition + ) + + @property + def j_object(self) -> jpy.JType: + return self.j_table_definition + + @property + def table(self) -> Table: + """This table definition as a table.""" + return Table(_JTableTools.metaTable(self.j_table_definition)) + + def keys(self): + """The column names as a dictview.""" + return self._dict.keys() + + def items(self): + """The column name, column definition tuples as a dictview.""" + return self._dict.items() + + def values(self): + """The column definitions as a dictview.""" + return self._dict.values() + + @cached_property + def _dict(self) -> Dict[str, ColumnDefinition]: + return { + col.name: col + for col in [ + ColumnDefinition(j_col) + for j_col in self.j_table_definition.getColumnsArray() + ] + } + + def __getitem__(self, key) -> ColumnDefinition: + return self._dict[key] + + def __iter__(self): + return iter(self._dict) + + def __len__(self): + return len(self._dict) + + def __contains__(self, item): + return item in self._dict + + def __eq__(self, other): + return JObjectWrapper.__eq__(self, other) + + def __ne__(self, other): + return JObjectWrapper.__ne__(self, other) + + def __hash__(self): + return JObjectWrapper.__hash__(self) class Table(JObjectWrapper): @@ -435,11 +551,7 @@ def __init__(self, j_table: jpy.JType): self.j_table = jpy.cast(j_table, self.j_object_type) if self.j_table is None: raise DHError("j_table type is not io.deephaven.engine.table.Table") - self._definition = self.j_table.getDefinition() - self._schema = None - self._is_refreshing = None - self._update_graph = None - self._is_flat = None + self._definition = TableDefinition(self.j_table.getDefinition()) def __repr__(self): default_repr = super().__repr__() @@ -465,37 +577,37 @@ def size(self) -> int: @property def is_refreshing(self) -> bool: """Whether this table is refreshing.""" - if self._is_refreshing is None: - self._is_refreshing = self.j_table.isRefreshing() - return self._is_refreshing + return self.j_table.isRefreshing() @property def is_blink(self) -> bool: """Whether this table is a blink table.""" return _JBlinkTableTools.isBlink(self.j_table) - @property + @cached_property def update_graph(self) -> UpdateGraph: """The update graph of the table.""" - if self._update_graph is None: - self._update_graph = UpdateGraph(self.j_table.getUpdateGraph()) - return self._update_graph + return UpdateGraph(self.j_table.getUpdateGraph()) @property def is_flat(self) -> bool: """Whether this table is guaranteed to be flat, i.e. its row set will be from 0 to number of rows - 1.""" - if self._is_flat is None: - self._is_flat = self.j_table.isFlat() - return self._is_flat + return self.j_table.isFlat() @property - def columns(self) -> List[Column]: - """The column definitions of the table.""" - if self._schema: - return self._schema + def definition(self) -> TableDefinition: + """The table definition.""" + return self._definition - self._schema = _td_to_columns(self._definition) - return self._schema + @property + def column_names(self) -> List[str]: + """The column names of the table.""" + return list(self.definition.keys()) + + @property + def columns(self) -> List[ColumnDefinition]: + """The column definitions of the table.""" + return list(self.definition.values()) @property def meta_table(self) -> Table: @@ -704,6 +816,10 @@ def flatten(self) -> Table: """Returns a new version of this table with a flat row set, i.e. from 0 to number of rows - 1.""" return Table(j_table=self.j_table.flatten()) + def remove_blink(self) -> Table: + """Returns a non-blink child table, or this table if it is not a blink table.""" + return Table(j_table=self.j_table.removeBlink()) + def snapshot(self) -> Table: """Returns a static snapshot table. @@ -2338,13 +2454,8 @@ def j_object(self) -> jpy.JType: def __init__(self, j_partitioned_table): self.j_partitioned_table = j_partitioned_table - self._schema = None + self._definition = None self._table = None - self._key_columns = None - self._unique_keys = None - self._constituent_column = None - self._constituent_changes_permitted = None - self._is_refreshing = None @classmethod def from_partitioned_table(cls, @@ -2352,18 +2463,18 @@ def from_partitioned_table(cls, key_cols: Union[str, List[str]] = None, unique_keys: bool = None, constituent_column: str = None, - constituent_table_columns: List[Column] = None, + constituent_table_columns: Optional[TableDefinitionLike] = None, constituent_changes_permitted: bool = None) -> PartitionedTable: """Creates a PartitionedTable from the provided underlying partitioned Table. - Note: key_cols, unique_keys, constituent_column, constituent_table_columns, + Note: key_cols, unique_keys, constituent_column, constituent_table_definition, constituent_changes_permitted must either be all None or all have values. When they are None, their values will be inferred as follows: | * key_cols: the names of all columns with a non-Table data type | * unique_keys: False | * constituent_column: the name of the first column with a Table data type - | * constituent_table_columns: the column definitions of the first cell (constituent table) in the constituent + | * constituent_table_definition: the table definitions of the first cell (constituent table) in the constituent column. Consequently, the constituent column can't be empty. | * constituent_changes_permitted: the value of table.is_refreshing @@ -2373,7 +2484,7 @@ def from_partitioned_table(cls, key_cols (Union[str, List[str]]): the key column name(s) of 'table' unique_keys (bool): whether the keys in 'table' are guaranteed to be unique constituent_column (str): the constituent column name in 'table' - constituent_table_columns (List[Column]): the column definitions of the constituent table + constituent_table_columns (Optional[TableDefinitionLike]): the table definitions of the constituent table constituent_changes_permitted (bool): whether the values of the constituent column can change Returns: @@ -2390,7 +2501,7 @@ def from_partitioned_table(cls, return PartitionedTable(j_partitioned_table=_JPartitionedTableFactory.of(table.j_table)) if all([arg is not None for arg in none_args]): - table_def = _JTableDefinition.of([col.j_column_definition for col in constituent_table_columns]) + table_def = TableDefinition(constituent_table_columns).j_table_definition j_partitioned_table = _JPartitionedTableFactory.of(table.j_table, j_array_list(to_sequence(key_cols)), unique_keys, @@ -2407,18 +2518,18 @@ def from_partitioned_table(cls, @classmethod def from_constituent_tables(cls, tables: List[Table], - constituent_table_columns: List[Column] = None) -> PartitionedTable: + constituent_table_columns: Optional[TableDefinitionLike] = None) -> PartitionedTable: """Creates a PartitionedTable with a single column named '__CONSTITUENT__' containing the provided constituent tables. The result PartitionedTable has no key columns, and both its unique_keys and constituent_changes_permitted - properties are set to False. When constituent_table_columns isn't provided, it will be set to the column + properties are set to False. When constituent_table_definition isn't provided, it will be set to the table definitions of the first table in the provided constituent tables. Args: tables (List[Table]): the constituent tables - constituent_table_columns (List[Column]): a list of column definitions compatible with all the constituent - tables, default is None + constituent_table_columns (Optional[TableDefinitionLike]): the table definition compatible with all the + constituent tables, default is None Returns: a PartitionedTable @@ -2430,37 +2541,31 @@ def from_constituent_tables(cls, if not constituent_table_columns: return PartitionedTable(j_partitioned_table=_JPartitionedTableFactory.ofTables(to_sequence(tables))) else: - table_def = _JTableDefinition.of([col.j_column_definition for col in constituent_table_columns]) + table_def = TableDefinition(constituent_table_columns).j_table_definition return PartitionedTable(j_partitioned_table=_JPartitionedTableFactory.ofTables(table_def, to_sequence(tables))) except Exception as e: raise DHError(e, "failed to create a PartitionedTable from constituent tables.") from e - @property + @cached_property def table(self) -> Table: """The underlying partitioned table.""" - if self._table is None: - self._table = Table(j_table=self.j_partitioned_table.table()) - return self._table + return Table(j_table=self.j_partitioned_table.table()) @property def update_graph(self) -> UpdateGraph: """The underlying partitioned table's update graph.""" return self.table.update_graph - @property + @cached_property def is_refreshing(self) -> bool: """Whether the underlying partitioned table is refreshing.""" - if self._is_refreshing is None: - self._is_refreshing = self.table.is_refreshing - return self._is_refreshing + return self.table.is_refreshing - @property + @cached_property def key_columns(self) -> List[str]: """The partition key column names.""" - if self._key_columns is None: - self._key_columns = list(self.j_partitioned_table.keyColumnNames().toArray()) - return self._key_columns + return list(self.j_partitioned_table.keyColumnNames().toArray()) def keys(self) -> Table: """Returns a Table containing all the keys of the underlying partitioned table.""" @@ -2469,32 +2574,31 @@ def keys(self) -> Table: else: return self.table.select_distinct(self.key_columns) - @property + @cached_property def unique_keys(self) -> bool: """Whether the keys in the underlying table must always be unique. If keys must be unique, one can expect that self.table.select_distinct(self.key_columns) and self.table.view(self.key_columns) operations always produce equivalent tables.""" - if self._unique_keys is None: - self._unique_keys = self.j_partitioned_table.uniqueKeys() - return self._unique_keys + return self.j_partitioned_table.uniqueKeys() - @property + @cached_property def constituent_column(self) -> str: """The name of the column containing constituent tables.""" - if self._constituent_column is None: - self._constituent_column = self.j_partitioned_table.constituentColumnName() - return self._constituent_column + return self.j_partitioned_table.constituentColumnName() + + @cached_property + def constituent_table_definition(self) -> TableDefinition: + """The table definitions for constituent tables. All constituent tables in a partitioned table have the + same table definitions.""" + return TableDefinition(self.j_partitioned_table.constituentDefinition()) @property - def constituent_table_columns(self) -> List[Column]: + def constituent_table_columns(self) -> List[ColumnDefinition]: """The column definitions for constituent tables. All constituent tables in a partitioned table have the same column definitions.""" - if not self._schema: - self._schema = _td_to_columns(self.j_partitioned_table.constituentDefinition()) + return list(self.constituent_table_definition.values()) - return self._schema - - @property + @cached_property def constituent_changes_permitted(self) -> bool: """Can the constituents of the underlying partitioned table change? Specifically, can the values of the constituent column change? @@ -2509,9 +2613,7 @@ def constituent_changes_permitted(self) -> bool: if the underlying partitioned table is refreshing. Also note that the underlying partitioned table must be refreshing if it contains any refreshing constituents. """ - if self._constituent_changes_permitted is None: - self._constituent_changes_permitted = self.j_partitioned_table.constituentChangesPermitted() - return self._constituent_changes_permitted + return self.j_partitioned_table.constituentChangesPermitted() def merge(self) -> Table: """Makes a new Table that contains all the rows from all the constituent tables. In the merged result, diff --git a/py/server/deephaven/table_factory.py b/py/server/deephaven/table_factory.py index 7efff4b534c..3b425583ee6 100644 --- a/py/server/deephaven/table_factory.py +++ b/py/server/deephaven/table_factory.py @@ -4,20 +4,19 @@ """ This module provides various ways to make a Deephaven table. """ -import datetime -from typing import Callable, List, Dict, Any, Union, Sequence, Tuple, Mapping +from typing import Callable, List, Dict, Any, Union, Sequence, Tuple, Mapping, Optional import jpy -import numpy as np import pandas as pd -from deephaven import execution_context, DHError, time +from deephaven import execution_context, DHError from deephaven._wrapper import JObjectWrapper -from deephaven.column import InputColumn, Column -from deephaven.dtypes import DType, Duration, Instant +from deephaven.column import InputColumn +from deephaven.dtypes import DType from deephaven.execution_context import ExecutionContext from deephaven.jcompat import j_lambda, j_list_to_list, to_sequence -from deephaven.table import Table +from deephaven.table import Table, TableDefinition, TableDefinitionLike +from deephaven.time import DurationLike, InstantLike, to_j_duration, to_j_instant from deephaven.update_graph import auto_locking_ctx _JTableFactory = jpy.get_type("io.deephaven.engine.table.TableFactory") @@ -53,16 +52,16 @@ def empty_table(size: int) -> Table: raise DHError(e, "failed to create an empty table.") from e -def time_table(period: Union[Duration, int, str, datetime.timedelta, np.timedelta64, pd.Timedelta], - start_time: Union[None, Instant, int, str, datetime.datetime, np.datetime64, pd.Timestamp] = None, +def time_table(period: DurationLike, + start_time: Optional[InstantLike] = None, blink_table: bool = False) -> Table: """Creates a table that adds a new row on a regular interval. Args: - period (Union[dtypes.Duration, int, str, datetime.timedelta, np.timedelta64, pd.Timedelta]): + period (DurationLike): time interval between new row additions, can be expressed as an integer in nanoseconds, a time interval string, e.g. "PT00:00:00.001" or "PT1s", or other time duration types. - start_time (Union[None, Instant, int, str, datetime.datetime, np.datetime64, pd.Timestamp], optional): + start_time (Optional[InstantLike]): start time for adding new rows, defaults to None which means use the current time as the start time. blink_table (bool, optional): if the time table should be a blink table, defaults to False @@ -76,14 +75,13 @@ def time_table(period: Union[Duration, int, str, datetime.timedelta, np.timedelt try: builder = _JTableTools.timeTableBuilder() - if not isinstance(period, str) and not isinstance(period, int): - period = time.to_j_duration(period) + if period is None: + raise ValueError("period must be specified") - builder.period(period) + builder.period(to_j_duration(period)) if start_time: - start_time = time.to_j_instant(start_time) - builder.startTime(start_time) + builder.startTime(to_j_instant(start_time)) if blink_table: builder.blinkTable(blink_table) @@ -285,7 +283,7 @@ def value_names(self) -> List[str]: return j_list_to_list(self.j_input_table.getValueNames()) -def input_table(col_defs: Dict[str, DType] = None, init_table: Table = None, +def input_table(col_defs: Optional[TableDefinitionLike] = None, init_table: Table = None, key_cols: Union[str, Sequence[str]] = None) -> InputTable: """Creates an in-memory InputTable from either column definitions or an initial table. When key columns are provided, the InputTable will be keyed, otherwise it will be append-only. @@ -298,7 +296,7 @@ def input_table(col_defs: Dict[str, DType] = None, init_table: Table = None, The keyed input table has keys for each row and supports addition/deletion/modification of rows by the keys. Args: - col_defs (Dict[str, DType]): the column definitions + col_defs (Optional[TableDefinitionLike]): the table definition init_table (Table): the initial table key_cols (Union[str, Sequence[str]): the name(s) of the key column(s) @@ -316,8 +314,7 @@ def input_table(col_defs: Dict[str, DType] = None, init_table: Table = None, raise ValueError("both column definitions and init table are provided.") if col_defs: - j_arg_1 = _JTableDefinition.of( - [Column(name=n, data_type=t).j_column_definition for n, t in col_defs.items()]) + j_arg_1 = TableDefinition(col_defs).j_table_definition else: j_arg_1 = init_table.j_table diff --git a/py/server/deephaven/table_listener.py b/py/server/deephaven/table_listener.py index b8d406673d7..ecc6b0016f6 100644 --- a/py/server/deephaven/table_listener.py +++ b/py/server/deephaven/table_listener.py @@ -6,7 +6,7 @@ from abc import ABC, abstractmethod from functools import wraps from inspect import signature -from typing import Callable, Union, List, Generator, Dict, Literal, Sequence, Optional +from typing import Callable, Union, List, Generator, Dict, Sequence, Optional import jpy import numpy @@ -17,13 +17,14 @@ from deephaven.jcompat import to_sequence, j_list_to_list from deephaven.table import Table from deephaven._table_reader import _table_reader_all_dict, _table_reader_chunk_dict -from deephaven.update_graph import UpdateGraph _JPythonReplayListenerAdapter = jpy.get_type("io.deephaven.integrations.python.PythonReplayListenerAdapter") _JTableUpdate = jpy.get_type("io.deephaven.engine.table.TableUpdate") _JListenerRecorder = jpy.get_type("io.deephaven.engine.table.impl.ListenerRecorder") _JPythonMergedListenerAdapter = jpy.get_type("io.deephaven.integrations.python.PythonMergedListenerAdapter") +_DEFAULT_ON_ERROR_CALLBACK = lambda e : print(f"An error occurred during table update processing: {e}") + class TableUpdate(JObjectWrapper): """A TableUpdate object represents a table update event. It contains the added, removed, and modified rows in the table. """ @@ -188,13 +189,22 @@ def modified_columns(self) -> List[str]: class TableListener(ABC): - """An abstract table listener class that should be subclassed by any user table listener class.""" + """An abstract table listener class that should be subclassed by any user table listener class. It provides a + default implementation for the on_error method that simply prints out the error.""" @abstractmethod def on_update(self, update: TableUpdate, is_replay: bool) -> None: """The required method on a listener object that receives table updates.""" ... + def on_error(self, e: Exception) -> None: + """The callback method on a listener object that handles the received error. The default implementation simply prints the error. + + Args: + e (Exception): the exception that occurred during the listener's execution. + """ + print(f"An error occurred during table update processing: {self}, {e}") + def _listener_wrapper(table: Table): """A decorator to wrap a user listener function or on_update method to receive the numpy-converted Table updates. @@ -213,7 +223,6 @@ def wrapper(update, *args): return decorator - def _wrap_listener_func(t: Table, listener: Callable[[TableUpdate, bool], None]): n_params = len(signature(listener).parameters) if n_params != 2: @@ -229,17 +238,26 @@ def _wrap_listener_obj(t: Table, listener: TableListener): return listener +def _error_callback_wrapper(callback: Callable[[Exception], None]): + @wraps(callback) + def wrapper(e): + callback(RuntimeError(e)) + + return wrapper + + class TableListenerHandle(JObjectWrapper): """A handle to manage a table listener's lifecycle.""" j_object_type = _JPythonReplayListenerAdapter def __init__(self, t: Table, listener: Union[Callable[[TableUpdate, bool], None], TableListener], description: str = None, - dependencies: Union[Table, Sequence[Table]] = None): + dependencies: Union[Table, Sequence[Table]] = None, on_error: Callable[[Exception], None] = None): """Creates a new table listener handle with dependencies. Table change events are processed by 'listener', which can be either (1) a callable (e.g. function) or - (2) an instance of TableListener type which provides an "on_update" method. + (2) an instance of a TableListener subclass that must override the abstract "on_update" method, and optionally + override the default "on_error" method. The callable or the on_update method must have the following signatures. * (update: TableUpdate, is_replay: bool): support replaying the initial table snapshot and normal table updates @@ -265,6 +283,12 @@ def __init__(self, t: Table, listener: Union[Callable[[TableUpdate, bool], None] the listener is safe, it is not recommended because reading or operating on the result tables of those operations may not be safe. It is best to perform the operations on the dependent tables beforehand, and then add the result tables as dependencies to the listener so that they can be safely read in it. + on_error (Callable[[Exception], None]): a callback function to be invoked when an error occurs during the + listener's execution. It should only be set when the listener is a function, not when it is an instance + of TableListener. When the listener is a TableListener, TableListener.on_error will be used. + Defaults to None. When None, a default callback function will be provided that simply + prints out the received exception. If the callback function itself raises an exception, the new exception + will be logged in the Deephaven server log and will not be further processed by the server. Raises: DHError @@ -277,14 +301,23 @@ def __init__(self, t: Table, listener: Union[Callable[[TableUpdate, bool], None] self.dependencies = to_sequence(dependencies) if isinstance(listener, TableListener): + if on_error: + raise DHError(message="Invalid on_error argument for listeners of TableListener type which already have an on_error method.") self.listener_wrapped = _wrap_listener_obj(t, listener) + on_error_callback = _error_callback_wrapper(listener.on_error) elif callable(listener): self.listener_wrapped = _wrap_listener_func(t, listener) + if on_error: + on_error_callback = _error_callback_wrapper(on_error) + else: + on_error_callback = _error_callback_wrapper(_DEFAULT_ON_ERROR_CALLBACK) else: raise DHError(message="listener is neither callable nor TableListener object") try: - self.listener_adapter = _JPythonReplayListenerAdapter.create(description, t.j_table, False, self.listener_wrapped, self.dependencies) + self.listener_adapter = _JPythonReplayListenerAdapter.create(description, t.j_table, False, + self.listener_wrapped, on_error_callback, + self.dependencies) except Exception as e: raise DHError(e, "failed to create a table listener.") from e self.started = False @@ -326,7 +359,7 @@ def stop(self) -> None: def listen(t: Table, listener: Union[Callable[[TableUpdate, bool], None], TableListener], description: str = None, do_replay: bool = False, - dependencies: Union[Table, Sequence[Table]] = None) -> TableListenerHandle: + dependencies: Union[Table, Sequence[Table]] = None, on_error: Callable[[Exception], None] = None) -> TableListenerHandle: """This is a convenience function that creates a TableListenerHandle object and immediately starts it to listen for table updates. @@ -352,6 +385,12 @@ def listen(t: Table, listener: Union[Callable[[TableUpdate, bool], None], TableL the listener is safe, it is not recommended because reading or operating on the result tables of those operations may not be safe. It is best to perform the operations on the dependent tables beforehand, and then add the result tables as dependencies to the listener so that they can be safely read in it. + on_error (Callable[[Exception], None]): a callback function to be invoked when an error occurs during the + listener's execution. It should only be set when the listener is a function, not when it is an instance + of TableListener. When the listener is a TableListener, TableListener.on_error will be used. + Defaults to None. When None, a default callback function will be provided that simply + prints out the received exception. If the callback function itself raises an exception, the new exception + will be logged in the Deephaven server log and will not be further processed by the server. Returns: a TableListenerHandle @@ -359,8 +398,8 @@ def listen(t: Table, listener: Union[Callable[[TableUpdate, bool], None], TableL Raises: DHError """ - table_listener_handle = TableListenerHandle(t=t, dependencies=dependencies, listener=listener, - description=description) + table_listener_handle = TableListenerHandle(t=t, listener=listener, description=description, + dependencies=dependencies, on_error=on_error) table_listener_handle.start(do_replay=do_replay) return table_listener_handle @@ -394,7 +433,8 @@ def table_update(self) -> Optional[TableUpdate]: class MergedListener(ABC): - """An abstract multi-table listener class that should be subclassed by any user multi-table listener class.""" + """An abstract multi-table listener class that should be subclassed by any user multi-table listener class. It + provides a default implementation for the on_error method that simply prints out the error.""" @abstractmethod def on_update(self, updates: Dict[Table, TableUpdate], is_replay: bool) -> None: @@ -403,6 +443,14 @@ def on_update(self, updates: Dict[Table, TableUpdate], is_replay: bool) -> None: """ ... + def on_error(self, e: Exception) -> None: + """ The callback method on a listener object that handles the received error. The default implementation simply prints the error. + + Args: + e (Exception): the exception that occurred during the listener's execution. + """ + print(f"An error occurred during table update processing: {self}, {e}") + class MergedListenerHandle(JObjectWrapper): """A handle to manage a merged listener's lifecycle.""" @@ -413,12 +461,14 @@ def j_object(self) -> jpy.JType: return self.merged_listener_adapter def __init__(self, tables: Sequence[Table], listener: Union[Callable[[Dict[Table, TableUpdate], bool], None], MergedListener], - description: str = None, dependencies: Union[Table, Sequence[Table]] = None): + description: str = None, dependencies: Union[Table, Sequence[Table]] = None, on_error: Callable[[Exception], None] = None): """Creates a new MergedListenerHandle with the provided listener recorders and dependencies. Table change events are processed by 'listener', which can be either (1) a callable (e.g. function) or - (2) an instance of MergedListener type which provides an "on_update" method. + (2) an instance of a MergedListener subclass that must override the abstract "on_update" method, and optionally + override the default "on_error" method. + The callable or the on_update method must have the following signature. *(updates: Dict[Table, TableUpdate], is_replay: bool): support replaying the initial table snapshots and normal table updates The 'updates' parameter is a dictionary of Table to TableUpdate; @@ -444,6 +494,12 @@ def __init__(self, tables: Sequence[Table], listener: Union[Callable[[Dict[Table the listener is safe, it is not recommended because reading or operating on the result tables of those operations may not be safe. It is best to perform the operations on the dependent tables beforehand, and then add the result tables as dependencies to the listener so that they can be safely read in it. + on_error (Callable[[Exception], None]): a callback function to be invoked when an error occurs during the + listener's execution. It should only be set when the listener is a function, not when it is an instance + of MergedListener. When the listener is a MergedListener, MergedListener.on_error will be used. + Defaults to None. When None, a default callback function will be provided that simply + prints out the received exception. If the callback function itself raises an exception, the new exception + will be logged in the Deephaven server log and will not be further processed by the server. Raises: DHError @@ -457,20 +513,30 @@ def __init__(self, tables: Sequence[Table], listener: Union[Callable[[Dict[Table self.dependencies = dependencies if isinstance(listener, MergedListener): + if on_error: + raise DHError(message="Invalid on_error argument for listeners of MergedListener type which already have an on_error method.") self.listener = listener.on_update - else: + on_error_callback = _error_callback_wrapper(listener.on_error) + elif callable(listener): self.listener = listener + if on_error: + on_error_callback = _error_callback_wrapper(on_error) + else: + on_error_callback = _error_callback_wrapper(_DEFAULT_ON_ERROR_CALLBACK) + else: + raise DHError(message="listener is neither callable nor MergedListener object") + n_params = len(signature(self.listener).parameters) if n_params != 2: raise ValueError("merged listener function must have 2 parameters (updates, is_replay).") - try: self.merged_listener_adapter = _JPythonMergedListenerAdapter.create( to_sequence(self.listener_recorders), to_sequence(self.dependencies), description, - self) + self, + on_error_callback) self.started = False except Exception as e: raise DHError(e, "failed to create a merged listener adapter.") from e @@ -512,7 +578,6 @@ def start(self, do_replay: bool = False) -> None: self.started = True - def stop(self) -> None: """Stop the listener.""" if not self.started: @@ -527,8 +592,8 @@ def stop(self) -> None: def merged_listen(tables: Sequence[Table], listener: Union[Callable[[Dict[Table, TableUpdate]], None], MergedListener], - do_replay: bool = False, description: str = None, dependencies: Union[Table, Sequence[Table]] = None)\ - -> MergedListenerHandle: + do_replay: bool = False, description: str = None, dependencies: Union[Table, Sequence[Table]] = None, + on_error: Callable[[Exception], None] = None) -> MergedListenerHandle: """This is a convenience function that creates a MergedListenerHandle object and immediately starts it to listen for table updates. @@ -555,8 +620,14 @@ def merged_listen(tables: Sequence[Table], listener: Union[Callable[[Dict[Table, the listener is safe, it is not recommended because reading or operating on the result tables of those operations may not be safe. It is best to perform the operations on the dependent tables beforehand, and then add the result tables as dependencies to the listener so that they can be safely read in it. + on_error (Callable[[Exception], None]): a callback function to be invoked when an error occurs during the + listener's execution. It should only be set when the listener is a function, not when it is an instance + of MergedListener. When the listener is a MergedListener, MergedListener.on_error will be used. + Defaults to None. When None, a default callback function will be provided that simply + prints out the received exception. If the callback function itself raises an exception, the new exception + will be logged in the Deephaven server log and will not be further processed by the server. """ merged_listener_handle = MergedListenerHandle(tables=tables, listener=listener, - description=description, dependencies=dependencies) + description=description, dependencies=dependencies, on_error=on_error) merged_listener_handle.start(do_replay=do_replay) return merged_listener_handle diff --git a/py/server/deephaven/time.py b/py/server/deephaven/time.py index 208dc25c383..5868392c0c8 100644 --- a/py/server/deephaven/time.py +++ b/py/server/deephaven/time.py @@ -29,6 +29,50 @@ _NANOS_PER_SECOND = 1000000000 _NANOS_PER_MICRO = 1000 +if sys.version_info >= (3, 10): + from typing import TypeAlias # novermin + + TimeZoneLike : TypeAlias = Union[TimeZone, str, datetime.tzinfo, datetime.datetime, pandas.Timestamp] + """A Union representing objects that can be coerced into a TimeZone.""" + + LocalDateLike : TypeAlias = Union[LocalDate, str, datetime.date, datetime.datetime, numpy.datetime64, pandas.Timestamp] + """A Union representing objects that can be coerced into a LocalDate.""" + + LocalTimeLike : TypeAlias = Union[LocalTime, int, str, datetime.time, datetime.datetime, numpy.datetime64, pandas.Timestamp] + """A Union representing objects that can be coerced into a LocalTime.""" + + InstantLike : TypeAlias = Union[Instant, int, str, datetime.datetime, numpy.datetime64, pandas.Timestamp] + """A Union representing objects that can be coerced into an Instant.""" + + ZonedDateTimeLike : TypeAlias = Union[ZonedDateTime, str, datetime.datetime, numpy.datetime64, pandas.Timestamp] + """A Union representing objects that can be coerced into a ZonedDateTime.""" + + DurationLike : TypeAlias = Union[Duration, int, str, datetime.timedelta, numpy.timedelta64, pandas.Timedelta] + """A Union representing objects that can be coerced into a Duration.""" + + PeriodLike : TypeAlias = Union[Period, str, datetime.timedelta, numpy.timedelta64, pandas.Timedelta] + """A Union representing objects that can be coerced into a Period.""" +else: + TimeZoneLike = Union[TimeZone, str, datetime.tzinfo, datetime.datetime, pandas.Timestamp] + """A Union representing objects that can be coerced into a TimeZone.""" + + LocalDateLike = Union[LocalDate, str, datetime.date, datetime.datetime, numpy.datetime64, pandas.Timestamp] + """A Union representing objects that can be coerced into a LocalDate.""" + + LocalTimeLike = Union[LocalTime, int, str, datetime.time, datetime.datetime, numpy.datetime64, pandas.Timestamp] + """A Union representing objects that can be coerced into a LocalTime.""" + + InstantLike = Union[Instant, int, str, datetime.datetime, numpy.datetime64, pandas.Timestamp] + """A Union representing objects that can be coerced into an Instant.""" + + ZonedDateTimeLike = Union[ZonedDateTime, str, datetime.datetime, numpy.datetime64, pandas.Timestamp] + """A Union representing objects that can be coerced into a ZonedDateTime.""" + + DurationLike = Union[Duration, int, str, datetime.timedelta, numpy.timedelta64, pandas.Timedelta] + """A Union representing objects that can be coerced into a Duration.""" + + PeriodLike = Union[Period, str, datetime.timedelta, numpy.timedelta64, pandas.Timedelta] + """A Union representing objects that can be coerced into a Period.""" # region Clock @@ -223,15 +267,14 @@ def _tzinfo_to_j_time_zone(tzi: datetime.tzinfo) -> TimeZone: raise TypeError(f"Unsupported conversion: {str(type(tzi))} -> TimeZone\n\tDetails:\n\t{details}") -def to_j_time_zone(tz: Union[None, TimeZone, str, datetime.tzinfo, datetime.datetime, pandas.Timestamp]) -> \ - Optional[TimeZone]: +def to_j_time_zone(tz: Optional[TimeZoneLike]) -> Optional[TimeZone]: """ Converts a time zone value to a Java TimeZone. Time zone values can be None, a Java TimeZone, a string, a datetime.tzinfo, a datetime.datetime, or a pandas.Timestamp. Args: - tz (Union[None, TimeZone, str, datetime.tzinfo, datetime.datetime, pandas.Timestamp]): A time zone value. + tz (Optional[TimeZoneLike]): A time zone value. If None is provided, None is returned. If a string is provided, it is parsed as a time zone name. @@ -266,8 +309,7 @@ def to_j_time_zone(tz: Union[None, TimeZone, str, datetime.tzinfo, datetime.date raise DHError(e) from e -def to_j_local_date(dt: Union[None, LocalDate, str, datetime.date, datetime.datetime, - numpy.datetime64, pandas.Timestamp]) -> Optional[LocalDate]: +def to_j_local_date(dt: Optional[LocalDateLike]) -> Optional[LocalDate]: """ Converts a date time value to a Java LocalDate. Date time values can be None, a Java LocalDate, a string, a datetime.date, a datetime.datetime, @@ -276,8 +318,7 @@ def to_j_local_date(dt: Union[None, LocalDate, str, datetime.date, datetime.date Date strings can be formatted according to the ISO 8601 date time format as 'YYYY-MM-DD'. Args: - dt (Union[None, LocalDate, str, datetime.date, datetime.datetime, numpy.datetime64, pandas.Timestamp]): - A date time value. If None is provided, None is returned. + dt (Optional[LocalDateLike]): A date time value. If None is provided, None is returned. Returns: LocalDate @@ -305,8 +346,7 @@ def to_j_local_date(dt: Union[None, LocalDate, str, datetime.date, datetime.date raise DHError(e) from e -def to_j_local_time(dt: Union[None, LocalTime, int, str, datetime.time, datetime.datetime, - numpy.datetime64, pandas.Timestamp]) -> Optional[LocalTime]: +def to_j_local_time(dt: Optional[LocalTimeLike]) -> Optional[LocalTime]: """ Converts a date time value to a Java LocalTime. Date time values can be None, a Java LocalTime, an int, a string, a datetime.time, a datetime.datetime, @@ -317,8 +357,7 @@ def to_j_local_time(dt: Union[None, LocalTime, int, str, datetime.time, datetime Time strings can be formatted as 'hh:mm:ss[.nnnnnnnnn]'. Args: - dt (Union[None, LocalTime, int, str, datetime.time, datetime.datetime, numpy.datetime64, pandas.Timestamp]): - A date time value. If None is provided, None is returned. + dt (Optional[LocalTimeLike]): A date time value. If None is provided, None is returned. Returns: LocalTime @@ -351,8 +390,7 @@ def to_j_local_time(dt: Union[None, LocalTime, int, str, datetime.time, datetime raise DHError(e) from e -def to_j_instant(dt: Union[None, Instant, int, str, datetime.datetime, numpy.datetime64, pandas.Timestamp]) -> \ - Optional[Instant]: +def to_j_instant(dt: Optional[InstantLike]) -> Optional[Instant]: """ Converts a date time value to a Java Instant. Date time values can be None, a Java Instant, an int, a string, a datetime.datetime, @@ -366,8 +404,7 @@ def to_j_instant(dt: Union[None, Instant, int, str, datetime.datetime, numpy.dat from the Epoch. Expected date ranges are used to infer the units. Args: - dt (Union[None, Instant, int, str, datetime.datetime, numpy.datetime64, pandas.Timestamp]): A date time value. - If None is provided, None is returned. + dt (Optional[InstantLike]): A date time value. If None is provided, None is returned. Returns: Instant, TypeError @@ -403,8 +440,7 @@ def to_j_instant(dt: Union[None, Instant, int, str, datetime.datetime, numpy.dat raise DHError(e) from e -def to_j_zdt(dt: Union[None, ZonedDateTime, str, datetime.datetime, numpy.datetime64, pandas.Timestamp]) -> \ - Optional[ZonedDateTime]: +def to_j_zdt(dt: Optional[ZonedDateTimeLike]) -> Optional[ZonedDateTime]: """ Converts a date time value to a Java ZonedDateTime. Date time values can be None, a Java ZonedDateTime, a string, a datetime.datetime, @@ -419,8 +455,7 @@ def to_j_zdt(dt: Union[None, ZonedDateTime, str, datetime.datetime, numpy.dateti Converting a numpy.datetime64 to a ZonedDateTime will use the Deephaven default time zone. Args: - dt (Union[None, ZonedDateTime, str, datetime.datetime, numpy.datetime64, pandas.Timestamp]): - A date time value. If None is provided, None is returned. + dt (Optional[ZonedDateTimeLike]): A date time value. If None is provided, None is returned. Returns: ZonedDateTime @@ -455,8 +490,7 @@ def to_j_zdt(dt: Union[None, ZonedDateTime, str, datetime.datetime, numpy.dateti raise DHError(e) from e -def to_j_duration(dt: Union[None, Duration, int, str, datetime.timedelta, numpy.timedelta64, pandas.Timedelta]) -> \ - Optional[Duration]: +def to_j_duration(dt: Optional[DurationLike]) -> Optional[Duration]: """ Converts a time duration value to a Java Duration, which is a unit of time in terms of clock time (24-hour days, hours, minutes, seconds, and nanoseconds). @@ -480,8 +514,7 @@ def to_j_duration(dt: Union[None, Duration, int, str, datetime.timedelta, numpy. | "-PT-6H+3M" -- parses as "+6 hours and -3 minutes" Args: - dt (Union[None, Duration, int, str, datetime.timedelta, numpy.timedelta64, pandas.Timedelta]): - A time duration value. If None is provided, None is returned. + dt (Optional[DurationLike]): A time duration value. If None is provided, None is returned. Returns: Duration @@ -515,8 +548,7 @@ def to_j_duration(dt: Union[None, Duration, int, str, datetime.timedelta, numpy. raise DHError(e) from e -def to_j_period(dt: Union[None, Period, str, datetime.timedelta, numpy.timedelta64, pandas.Timedelta]) -> \ - Optional[Period]: +def to_j_period(dt: Optional[PeriodLike]) -> Optional[Period]: """ Converts a time duration value to a Java Period, which is a unit of time in terms of calendar time (days, weeks, months, years, etc.). @@ -537,8 +569,7 @@ def to_j_period(dt: Union[None, Period, str, datetime.timedelta, numpy.timedelta | "-P1Y2M" -- -1 Year, -2 Months Args: - dt (Union[None, Period, str, datetime.timedelta, numpy.timedelta64, pandas.Timedelta]): - A Python period or period string. If None is provided, None is returned. + dt (Optional[PeriodLike]): A Python period or period string. If None is provided, None is returned. Returns: Period diff --git a/py/server/deephaven_internal/auto_completer/__init__.py b/py/server/deephaven_internal/auto_completer/__init__.py index ffb6eea0888..8ba39520d27 100644 --- a/py/server/deephaven_internal/auto_completer/__init__.py +++ b/py/server/deephaven_internal/auto_completer/__init__.py @@ -19,7 +19,16 @@ from ._completer import Completer, Mode from jedi import preload_module, Interpreter + jedi_settings = Completer() # warm jedi up a little. We could probably off-thread this. preload_module("deephaven") Interpreter("", []).complete(1, 0) + + +def set_max_recursion_limit(limit: int) -> None: + """ + Utility method to raise/lower the limit that our autocompletion will impose on Python 3.9 and 3.10. + """ + from . import _completer + _completer.MAX_RECURSION_LIMIT = limit diff --git a/py/server/deephaven_internal/auto_completer/_completer.py b/py/server/deephaven_internal/auto_completer/_completer.py index f87c3c0b56c..39ea08cc25a 100644 --- a/py/server/deephaven_internal/auto_completer/_completer.py +++ b/py/server/deephaven_internal/auto_completer/_completer.py @@ -6,6 +6,9 @@ from typing import Any, Union, List from jedi import Interpreter, Script from jedi.api.classes import Completion, Signature +from importlib.metadata import version +import sys +import warnings class Mode(Enum): @@ -36,6 +39,13 @@ def __str__(self) -> str: } +""" +For Python 3.9 and 3.10, there is a bug in recursion which can result in a segfault. Lowering this +limit to 2000 or less seems to mitigate it. +""" +MAX_RECURSION_LIMIT = 2000 + + def wrap_python(txt: str) -> str: """ Wraps a string in a Python fenced codeblock for markdown @@ -80,6 +90,8 @@ def __init__(self): except ImportError: self.__can_jedi = False self.mode = Mode.OFF + self.recursion_limit_already_warned = False + self.check_recursion_limit(True) @property def mode(self) -> Mode: @@ -135,6 +147,7 @@ def do_completion( Modeled after Jedi language server https://github.com/pappasam/jedi-language-server/blob/main/jedi_language_server/server.py#L189 """ + self.check_recursion_limit() if not self._versions[uri] == version: # if you aren't the newest completion, you get nothing, quickly return [] @@ -253,3 +266,27 @@ def do_hover( hoverstring += '\n---\n' + wrap_plaintext(raw_docstring) return hoverstring.strip() + + def check_recursion_limit(self, suppress_warning: bool = False) -> None: + """ + Tests for python+jedi+numpy versions that are susceptible to a RecursionError/segfault issue, and lowers + the recursion limit, warning if the limit is raised externally. + """ + if sys.version_info < (3, 9) or sys.version_info >= (3, 11): + return + + if sys.getrecursionlimit() <= MAX_RECURSION_LIMIT: + return + + sys.setrecursionlimit(MAX_RECURSION_LIMIT) + + # Log a warning if the user (or some user code) seems to have tried to raise the limit again after we lowered it. + # This is not a fool-proof way to keep the limit down, and isn't meant to be, only to guard against the primary + # way we've seen to cause this issue. + if not suppress_warning and not self.recursion_limit_already_warned: + self.recursion_limit_already_warned = True + warnings.warn(f"""Recursion limit has been set to {MAX_RECURSION_LIMIT} to avoid a known segfault in Python 3.9 and 3.10 +related to RecursionErrors. This limit will be set to {MAX_RECURSION_LIMIT} whenever autocomplete takes place +to avoid this, because the jedi library sets this to 3000, above the safe limit. Disabling autocomplete +will prevent this check, as it will also prevent jedi from raising the limit. +See https://github.com/deephaven/deephaven-core/issues/5878 for more information.""") diff --git a/py/server/tests/test_barrage.py b/py/server/tests/test_barrage.py index 28c0a49b6aa..7049d2566ff 100644 --- a/py/server/tests/test_barrage.py +++ b/py/server/tests/test_barrage.py @@ -76,7 +76,7 @@ def test_subscribe(self): session = barrage_session(host="localhost", port=10000, auth_type="Anonymous") t = session.subscribe(ticket=self.shared_ticket.bytes) self.assertEqual(t.size, 1000) - self.assertEqual(len(t.columns), 2) + self.assertEqual(len(t.definition), 2) sp = t.snapshot() self.assertEqual(sp.size, 1000) t1 = t.update("Z = X + Y") @@ -119,7 +119,7 @@ def test_snapshot(self): session = barrage_session(host="localhost", port=10000, auth_type="Anonymous") t = session.snapshot(self.shared_ticket.bytes) self.assertEqual(t.size, 1000) - self.assertEqual(len(t.columns), 2) + self.assertEqual(len(t.definition), 2) t1 = t.update("Z = X + Y") self.assertEqual(t1.size, 1000) t2 = session.snapshot(self.shared_ticket.bytes) diff --git a/py/server/tests/test_column.py b/py/server/tests/test_column.py index 57c94d53fc1..6531b6c3fa3 100644 --- a/py/server/tests/test_column.py +++ b/py/server/tests/test_column.py @@ -12,7 +12,7 @@ from deephaven import DHError, dtypes, new_table, time as dhtime from deephaven import empty_table from deephaven.column import byte_col, char_col, short_col, bool_col, int_col, long_col, float_col, double_col, \ - string_col, datetime_col, jobj_col, ColumnType + string_col, datetime_col, jobj_col, ColumnType, col_def from deephaven.constants import MAX_BYTE, MAX_SHORT, MAX_INT, MAX_LONG from deephaven.jcompat import j_array_list from tests.testbase import BaseTestCase @@ -136,7 +136,8 @@ def test_datetime_col(self): inst = dhtime.to_j_instant(round(time.time())) dt = datetime.datetime.now() _ = datetime_col(name="Datetime", data=[inst, dt, None]) - self.assertEqual(_.data_type, dtypes.Instant) + self.assertEqual(_._column_definition.name, "Datetime") + self.assertEqual(_._column_definition.data_type, dtypes.Instant) ts = pd.Timestamp(dt) np_dt = np.datetime64(dt) @@ -144,17 +145,46 @@ def test_datetime_col(self): # test if we can convert to numpy datetime64 array np.array([pd.Timestamp(dt).to_numpy() for dt in data], dtype=np.datetime64) _ = datetime_col(name="Datetime", data=data) - self.assertEqual(_.data_type, dtypes.Instant) + self.assertEqual(_._column_definition.name, "Datetime") + self.assertEqual(_._column_definition.data_type, dtypes.Instant) data = np.array(['1970-01-01T00:00:00.000-07:00', '2020-01-01T01:00:00.000+07:00']) np.array([pd.Timestamp(str(dt)).to_numpy() for dt in data], dtype=np.datetime64) _ = datetime_col(name="Datetime", data=data) - self.assertEqual(_.data_type, dtypes.Instant) + self.assertEqual(_._column_definition.name, "Datetime") + self.assertEqual(_._column_definition.data_type, dtypes.Instant) data = np.array([1, -1]) data = data.astype(np.int64) _ = datetime_col(name="Datetime", data=data) - self.assertEqual(_.data_type, dtypes.Instant) + self.assertEqual(_._column_definition.name, "Datetime") + self.assertEqual(_._column_definition.data_type, dtypes.Instant) + + def test_col_def_simple(self): + foo_def = col_def("Foo", dtypes.int32) + self.assertEquals(foo_def.name, "Foo") + self.assertEquals(foo_def.data_type, dtypes.int32) + self.assertEquals(foo_def.component_type, None) + self.assertEquals(foo_def.column_type, ColumnType.NORMAL) + + def test_col_def_array(self): + foo_def = col_def("Foo", dtypes.int32_array) + self.assertEquals(foo_def.name, "Foo") + self.assertEquals(foo_def.data_type, dtypes.int32_array) + self.assertEquals(foo_def.component_type, dtypes.int32) + self.assertEquals(foo_def.column_type, ColumnType.NORMAL) + + def test_col_def_partitioning(self): + foo_def = col_def("Foo", dtypes.string, column_type=ColumnType.PARTITIONING) + self.assertEquals(foo_def.name, "Foo") + self.assertEquals(foo_def.data_type, dtypes.string) + self.assertEquals(foo_def.component_type, None) + self.assertEquals(foo_def.column_type, ColumnType.PARTITIONING) + + def test_col_def_invalid_component_type(self): + with self.assertRaises(DHError): + col_def("Foo", dtypes.int32_array, component_type=dtypes.int64) + @dataclass class CustomClass: diff --git a/py/server/tests/test_csv.py b/py/server/tests/test_csv.py index 3de09e9a570..88e291092cd 100644 --- a/py/server/tests/test_csv.py +++ b/py/server/tests/test_csv.py @@ -20,8 +20,7 @@ def test_read_header(self): col_types = [dtypes.string, dtypes.long, dtypes.float64] table_header = {k: v for k, v in zip(col_names, col_types)} t = read_csv('tests/data/test_csv.csv', header=table_header) - t_col_names = [col.name for col in t.columns] - self.assertEqual(col_names, t_col_names) + self.assertEqual(col_names, t.column_names) def test_read_error_col_type(self): col_names = ["Strings", "Longs", "Floats"] @@ -44,9 +43,9 @@ def test_read_error_quote(self): def test_write(self): t = read_csv("tests/data/small_sample.csv") write_csv(t, "./test_write.csv") - t_cols = [col.name for col in t.columns] + t_cols = t.column_names t = read_csv("./test_write.csv") - self.assertEqual(t_cols, [col.name for col in t.columns]) + self.assertEqual(t_cols, t.column_names) col_names = ["Strings", "Longs", "Floats"] col_types = [dtypes.string, dtypes.long, dtypes.float64] @@ -54,7 +53,7 @@ def test_write(self): t = read_csv('tests/data/test_csv.csv', header=table_header) write_csv(t, "./test_write.csv", cols=col_names) t = read_csv('./test_write.csv') - self.assertEqual(col_names, [c.name for c in t.columns]) + self.assertEqual(col_names, t.column_names) import os os.remove("./test_write.csv") diff --git a/py/server/tests/test_data_index.py b/py/server/tests/test_data_index.py index 5b3aad01391..14b21407f7c 100644 --- a/py/server/tests/test_data_index.py +++ b/py/server/tests/test_data_index.py @@ -47,7 +47,7 @@ def test_keys(self): self.assertEqual(["X", "Y"], self.data_index.keys) def test_backing_table(self): - self.assertEqual(3, len(self.data_index.table.columns)) + self.assertEqual(3, len(self.data_index.table.definition)) self.assertEqual(10, self.data_index.table.size) di = data_index(self.data_index.table, self.data_index.keys[0:1]) self.assertEqual(1, len(di.keys)) diff --git a/py/server/tests/test_dbc.py b/py/server/tests/test_dbc.py index 31868028e08..fd3cba4fbf2 100644 --- a/py/server/tests/test_dbc.py +++ b/py/server/tests/test_dbc.py @@ -50,7 +50,7 @@ def test_read_sql_connectorx(self): query = "SELECT t_ts, t_id, t_instrument, t_exchange, t_price, t_size FROM CRYPTO_TRADES LIMIT 10" postgres_url = "postgresql://test:test@postgres:5432/test" dh_table = read_sql(conn=postgres_url, query=query) - self.assertEqual(len(dh_table.columns), 6) + self.assertEqual(len(dh_table.definition), 6) self.assertEqual(dh_table.size, 10) with self.assertRaises(DHError) as cm: @@ -63,13 +63,13 @@ def test_read_sql(self): with self.subTest("odbc"): connection_string = 'Driver={PostgreSQL};Server=postgres;Port=5432;Database=test;Uid=test;Pwd=test;' dh_table = read_sql(conn=connection_string, query=query, driver="odbc") - self.assertEqual(len(dh_table.columns), 6) + self.assertEqual(len(dh_table.definition), 6) self.assertEqual(dh_table.size, 10) with self.subTest("adbc"): uri = "postgresql://postgres:5432/test?user=test&password=test" dh_table = read_sql(conn=uri, query=query, driver="adbc") - self.assertEqual(len(dh_table.columns), 6) + self.assertEqual(len(dh_table.definition), 6) self.assertEqual(dh_table.size, 10) if turbodbc_installed(): @@ -79,7 +79,7 @@ def test_read_sql(self): connection_string = "Driver={PostgreSQL};Server=postgres;Port=5432;Database=test;Uid=test;Pwd=test;" with turbodbc.connect(connection_string=connection_string) as conn: dh_table = read_sql(conn=conn, query=query, driver="odbc") - self.assertEqual(len(dh_table.columns), 6) + self.assertEqual(len(dh_table.definition), 6) self.assertEqual(dh_table.size, 10) with self.subTest("adbc-connection"): @@ -87,7 +87,7 @@ def test_read_sql(self): uri = "postgresql://postgres:5432/test?user=test&password=test" with adbc_driver_postgresql.dbapi.connect(uri) as conn: dh_table = read_sql(conn=conn, query=query, driver="adbc") - self.assertEqual(len(dh_table.columns), 6) + self.assertEqual(len(dh_table.definition), 6) self.assertEqual(dh_table.size, 10) with self.assertRaises(DHError) as cm: diff --git a/py/server/tests/test_experiments.py b/py/server/tests/test_experiments.py index 6de28871a98..fd04cce0c65 100644 --- a/py/server/tests/test_experiments.py +++ b/py/server/tests/test_experiments.py @@ -31,13 +31,13 @@ def test_full_outer_join(self): rt = full_outer_join(t1, t2, on="a = c") self.assertTrue(rt.is_refreshing) self.wait_ticking_table_update(rt, row_count=100, timeout=5) - self.assertEqual(len(rt.columns), len(t1.columns) + len(t2.columns)) + self.assertEqual(len(rt.definition), len(t1.definition) + len(t2.definition)) with self.subTest("full outer join with no matching keys"): t1 = empty_table(2).update(["X = i", "a = i"]) rt = full_outer_join(self.test_table, t1, joins=["Y = a"]) self.assertEqual(rt.size, t1.size * self.test_table.size) - self.assertEqual(len(rt.columns), 1 + len(self.test_table.columns)) + self.assertEqual(len(rt.definition), 1 + len(self.test_table.definition)) with self.subTest("Conflicting column names"): with self.assertRaises(DHError) as cm: @@ -52,13 +52,13 @@ def test_left_outer_join(self): rt = left_outer_join(t1, t2, on="a = c") self.assertTrue(rt.is_refreshing) self.wait_ticking_table_update(rt, row_count=100, timeout=5) - self.assertEqual(len(rt.columns), len(t1.columns) + len(t2.columns)) + self.assertEqual(len(rt.definition), len(t1.definition) + len(t2.definition)) with self.subTest("left outer join with no matching keys"): t1 = empty_table(2).update(["X = i", "a = i"]) rt = left_outer_join(self.test_table, t1, joins=["Y = a"]) self.assertEqual(rt.size, t1.size * self.test_table.size) - self.assertEqual(len(rt.columns), 1 + len(self.test_table.columns)) + self.assertEqual(len(rt.definition), 1 + len(self.test_table.definition)) with self.subTest("Conflicting column names"): with self.assertRaises(DHError) as cm: diff --git a/py/server/tests/test_iceberg.py b/py/server/tests/test_iceberg.py index 62ba31e6636..8934299b74d 100644 --- a/py/server/tests/test_iceberg.py +++ b/py/server/tests/test_iceberg.py @@ -4,7 +4,7 @@ import jpy from deephaven import dtypes -from deephaven.column import Column, ColumnType +from deephaven.column import col_def, ColumnType from tests.testbase import BaseTestCase from deephaven.experimental import s3, iceberg @@ -60,12 +60,10 @@ def test_instruction_create_with_table_definition_dict(self): def test_instruction_create_with_table_definition_list(self): table_def=[ - Column( - "Partition", dtypes.int32, column_type=ColumnType.PARTITIONING - ), - Column("x", dtypes.int32), - Column("y", dtypes.double), - Column("z", dtypes.double), + col_def("Partition", dtypes.int32, column_type=ColumnType.PARTITIONING), + col_def("x", dtypes.int32), + col_def("y", dtypes.double), + col_def("z", dtypes.double), ] iceberg_instructions = iceberg.IcebergInstructions(table_definition=table_def) diff --git a/py/server/tests/test_numpy.py b/py/server/tests/test_numpy.py index 725e69602f1..1c935ed04f7 100644 --- a/py/server/tests/test_numpy.py +++ b/py/server/tests/test_numpy.py @@ -71,14 +71,14 @@ def tearDown(self) -> None: super().tearDown() def test_to_numpy(self): - for col in self.test_table.columns: - with self.subTest(f"test single column to numpy- {col.name}"): - np_array = to_numpy(self.test_table, [col.name]) + for col_name in self.test_table.definition: + with self.subTest(f"test single column to numpy- {col_name}"): + np_array = to_numpy(self.test_table, [col_name]) self.assertEqual((2, 1), np_array.shape) - np.array_equal(np_array, self.np_array_dict[col.name]) + np.array_equal(np_array, self.np_array_dict[col_name]) try: - to_numpy(self.test_table, [col.name for col in self.test_table.columns]) + to_numpy(self.test_table, self.test_table.column_names) except DHError as e: self.assertIn("same data type", e.root_cause) @@ -90,17 +90,17 @@ def test_to_numpy(self): float_col(name="Float3", data=[1111.01111, -1111.01111]), float_col(name="Float4", data=[11111.011111, -11111.011111])] tmp_table = new_table(cols=input_cols) - np_array = to_numpy(tmp_table, [col.name for col in tmp_table.columns]) + np_array = to_numpy(tmp_table, tmp_table.column_names) self.assertEqual((2, 5), np_array.shape) def test_to_numpy_remap(self): - for col in self.test_table.columns: - with self.subTest(f"test single column to numpy - {col.name}"): - np_array = to_numpy(self.test_table, [col.name]) + for col_name in self.test_table.definition: + with self.subTest(f"test single column to numpy - {col_name}"): + np_array = to_numpy(self.test_table, [col_name]) self.assertEqual((2, 1), np_array.shape) try: - to_numpy(self.test_table, [col.name for col in self.test_table.columns]) + to_numpy(self.test_table, self.test_table.column_names) except DHError as e: self.assertIn("same data type", e.root_cause) @@ -140,12 +140,12 @@ def test_to_table(self): float_col(name="Float3", data=[1111.01111, -1111.01111]), float_col(name="Float4", data=[11111.011111, -11111.011111])] tmp_table = new_table(cols=input_cols) - np_array = to_numpy(tmp_table, [col.name for col in tmp_table.columns]) - tmp_table2 = to_table(np_array, [col.name for col in tmp_table.columns]) + np_array = to_numpy(tmp_table, tmp_table.column_names) + tmp_table2 = to_table(np_array, tmp_table.column_names) self.assert_table_equals(tmp_table2, tmp_table) with self.assertRaises(DHError) as cm: - tmp_table3 = to_table(np_array[:, [0, 1, 3]], [col.name for col in tmp_table.columns]) + tmp_table3 = to_table(np_array[:, [0, 1, 3]], tmp_table.column_names) self.assertIn("doesn't match", cm.exception.root_cause) def get_resource_path(self, resource_path) -> str: diff --git a/py/server/tests/test_pandas.py b/py/server/tests/test_pandas.py index d3f8312a3e8..04bbe942821 100644 --- a/py/server/tests/test_pandas.py +++ b/py/server/tests/test_pandas.py @@ -207,11 +207,11 @@ def test_to_table_nullable(self): self.assertIs(table.columns[6].data_type, dtypes.float32) self.assertIs(table.columns[7].data_type, dtypes.double) self.assertIs(table.columns[8].data_type, dtypes.string) - self.assertIs(table.columns[9].data_type, dtypes.PyObject) + self.assertIs(table.columns[9].data_type, dtypes.string) self.assertEqual(table.size, 3) table_string = table.to_string() - self.assertEqual(8, table_string.count("null")) + self.assertEqual(9, table_string.count("null")) self.assertEqual(2, table_string.count("NaN")) def test_arrow_backend(self): @@ -343,6 +343,27 @@ def test_to_table_readonly(self): t = to_table(df) self.assert_table_equals(source, t) + def test_infer_objects(self): + df = pd.DataFrame({ + "A": pd.Series([1, 2, 3], dtype=np.dtype("O")), + "B": pd.Series(["a", "b", "c"], dtype=np.dtype("O")), + "C": pd.Series([1.1, 2.2, 3.3], dtype=np.dtype("O")), + "D": pd.Series([True, False, True], dtype=np.dtype("O")), + "E": pd.Series( [pd.Timestamp("2021-01-01"), pd.Timestamp("2021-01-02"), pd.Timestamp("2021-01-03")], dtype=np.dtype("O")), + "F": pd.Series( [np.datetime64("2021-01-01"), np.datetime64("2021-01-02"), np.datetime64("2021-01-03")], dtype=np.dtype("O")), + }) + self.assertTrue(all(df[col].dtype == object for col in list(df))) + t = to_table(df) + self.assertEqual(t.columns[0].data_type, dtypes.int64) + self.assertEqual(t.columns[1].data_type, dtypes.string) + self.assertEqual(t.columns[2].data_type, dtypes.double) + self.assertEqual(t.columns[3].data_type, dtypes.bool_) + self.assertEqual(t.columns[4].data_type, dtypes.Instant) + self.assertEqual(t.columns[5].data_type, dtypes.Instant) + + t = to_table(df, infer_objects=False) + self.assertTrue(all([t.columns[i].data_type == dtypes.PyObject for i in range(6)])) + if __name__ == '__main__': unittest.main() diff --git a/py/server/tests/test_parquet.py b/py/server/tests/test_parquet.py index 2d49f7c82cd..21dea3db552 100644 --- a/py/server/tests/test_parquet.py +++ b/py/server/tests/test_parquet.py @@ -14,7 +14,7 @@ from deephaven import DHError, empty_table, dtypes, new_table from deephaven import arrow as dharrow -from deephaven.column import InputColumn, Column, ColumnType, string_col, int_col, char_col, long_col +from deephaven.column import InputColumn, ColumnType, col_def, string_col, int_col, char_col, long_col, short_col from deephaven.pandas import to_pandas, to_table from deephaven.parquet import (write, batch_write, read, delete, ColumnInstruction, ParquetFileLayout, write_partitioned) @@ -230,6 +230,8 @@ def get_table_with_array_data(self): "someCharArrayColumn = new char[] {i % 10 == 0 ? null : (char)i}", "someTimeArrayColumn = new Instant[] {i % 10 == 0 ? null : (Instant)DateTimeUtils.now() + i}", "someBiColumn = new java.math.BigInteger[] {i % 10 == 0 ? null : java.math.BigInteger.valueOf(i)}", + "someBdColumn = new java.math.BigDecimal[] {i % 10 == 0 ? null : " + + "java.math.BigDecimal.valueOf(ii).stripTrailingZeros()}", "nullStringArrayColumn = new String[] {(String)null}", "nullIntArrayColumn = new int[] {(int)null}", "nullLongArrayColumn = new long[] {(long)null}", @@ -240,7 +242,8 @@ def get_table_with_array_data(self): "nullByteArrayColumn = new byte[] {(byte)null}", "nullCharArrayColumn = new char[] {(char)null}", "nullTimeArrayColumn = new Instant[] {(Instant)null}", - "nullBiColumn = new java.math.BigInteger[] {(java.math.BigInteger)null}" + "nullBiColumn = new java.math.BigInteger[] {(java.math.BigInteger)null}", + "nullBdColumn = new java.math.BigDecimal[] {(java.math.BigDecimal)null}" ]) return dh_table @@ -597,12 +600,10 @@ def test_read_kv_partitioned(self): actual = read( kv_dir, table_definition=[ - Column( - "Partition", dtypes.int32, column_type=ColumnType.PARTITIONING - ), - Column("x", dtypes.int32), - Column("y", dtypes.double), - Column("z", dtypes.double), + col_def("Partition", dtypes.int32, column_type=ColumnType.PARTITIONING), + col_def("x", dtypes.int32), + col_def("y", dtypes.double), + col_def("z", dtypes.double), ], file_layout=ParquetFileLayout.KV_PARTITIONED, ) @@ -655,7 +656,7 @@ def test_write_partitioned_data(self): shutil.rmtree(root_dir) def verify_table_from_disk(table): - self.assertTrue(len(table.columns)) + self.assertTrue(len(table.definition)) self.assertTrue(table.columns[0].name == "X") self.assertTrue(table.columns[0].column_type == ColumnType.PARTITIONING) self.assert_table_equals(table.select().sort(["X", "Y"]), source.sort(["X", "Y"])) @@ -696,9 +697,9 @@ def verify_file_names(): shutil.rmtree(root_dir) table_definition = [ - Column("X", dtypes.string, column_type=ColumnType.PARTITIONING), - Column("Y", dtypes.string), - Column("Number", dtypes.int32) + col_def("X", dtypes.string, column_type=ColumnType.PARTITIONING), + col_def("Y", dtypes.string), + col_def("Number", dtypes.int32) ] write_partitioned(source, table_definition=table_definition, destination_dir=root_dir, base_name=base_name, max_dictionary_keys=max_dictionary_keys) @@ -782,6 +783,32 @@ def test_unsigned_ints(self): ]) self.assert_table_equals(table_from_disk, expected) + def test_unsigned_byte_cast(self): + data = {'uint8Col': [255, 2, 0]} + df = pandas.DataFrame(data) + df['uint8Col'] = df['uint8Col'].astype(np.uint8) + pyarrow.parquet.write_table(pyarrow.Table.from_pandas(df), 'data_from_pyarrow.parquet') + + # UByte -> Char + table_from_disk = read("data_from_pyarrow.parquet", table_definition={"uint8Col": dtypes.char}) + expected = new_table([char_col("uint8Col", [255, 2, 0])]) + self.assert_table_equals(table_from_disk, expected) + + # UByte -> Short + table_from_disk = read("data_from_pyarrow.parquet", table_definition={"uint8Col": dtypes.short}) + expected = new_table([short_col("uint8Col", [255, 2, 0])]) + self.assert_table_equals(table_from_disk, expected) + + # UByte -> Int + table_from_disk = read("data_from_pyarrow.parquet", table_definition={"uint8Col": dtypes.int32}) + expected = new_table([int_col("uint8Col", [255, 2, 0])]) + self.assert_table_equals(table_from_disk, expected) + + # UByte -> Long + table_from_disk = read("data_from_pyarrow.parquet", table_definition={"uint8Col": dtypes.long}) + expected = new_table([long_col("uint8Col", [255, 2, 0])]) + self.assert_table_equals(table_from_disk, expected) + def test_v2_pages(self): def test_v2_pages_helper(dh_table): write(dh_table, "data_from_dh.parquet") diff --git a/py/server/tests/test_partitioned_table.py b/py/server/tests/test_partitioned_table.py index 8da63d726a8..3059a6c40fb 100644 --- a/py/server/tests/test_partitioned_table.py +++ b/py/server/tests/test_partitioned_table.py @@ -65,6 +65,9 @@ def test_constituent_change_permitted(self): def test_constituent_table_columns(self): self.assertEqual(self.test_table.columns, self.partitioned_table.constituent_table_columns) + def test_constituent_table_definition(self): + self.assertEqual(self.test_table.definition, self.partitioned_table.constituent_table_definition) + def test_merge(self): t = self.partitioned_table.merge() self.assert_table_equals(t, self.test_table) @@ -188,7 +191,7 @@ def test_from_partitioned_table(self): key_cols="Y", unique_keys=True, constituent_column="aggPartition", - constituent_table_columns=test_table.columns, + constituent_table_columns=test_table.definition, constituent_changes_permitted=True, ) self.assertEqual(pt.key_columns, pt1.key_columns) @@ -201,7 +204,7 @@ def test_from_partitioned_table(self): key_cols="Y", unique_keys=True, constituent_column="Non-existing", - constituent_table_columns=test_table.columns, + constituent_table_columns=test_table.definition, constituent_changes_permitted=True, ) self.assertIn("no column named", str(cm.exception)) @@ -222,7 +225,7 @@ def test_from_constituent_tables(self): self.assertIn("IncompatibleTableDefinitionException", str(cm.exception)) with self.subTest("Compatible table definition"): - pt = PartitionedTable.from_constituent_tables([test_table, test_table1, test_table3], test_table.columns) + pt = PartitionedTable.from_constituent_tables([test_table, test_table1, test_table3], test_table.definition) def test_keys(self): keys_table = self.partitioned_table.keys() diff --git a/py/server/tests/test_pt_proxy.py b/py/server/tests/test_pt_proxy.py index 982a582fa6f..5cbe973dc0c 100644 --- a/py/server/tests/test_pt_proxy.py +++ b/py/server/tests/test_pt_proxy.py @@ -127,7 +127,7 @@ def test_USV(self): result_pt_proxy = op( self.pt_proxy, formulas=["a", "c", "Sum = a + b + c + d"]) for rct, ct in zip(result_pt_proxy.target.constituent_tables, self.pt_proxy.target.constituent_tables): - self.assertTrue(len(rct.columns) >= 3) + self.assertTrue(len(rct.definition) >= 3) self.assertLessEqual(rct.size, ct.size) def test_select_distinct(self): @@ -144,7 +144,7 @@ def test_natural_join(self): right_table = self.test_table.drop_columns(["b", "c"]).head(5) joined_pt_proxy = pt_proxy.natural_join(right_table, on="a", joins=["d", "e"]) for ct in joined_pt_proxy.target.constituent_tables: - self.assertEqual(len(ct.columns), 5) + self.assertEqual(len(ct.definition), 5) with self.subTest("Join with another Proxy"): with self.assertRaises(DHError) as cm: @@ -163,7 +163,7 @@ def test_natural_join(self): right_proxy = self.test_table.drop_columns(["b", "d"]).partition_by("c").proxy() joined_pt_proxy = pt_proxy.natural_join(right_proxy, on="a", joins="e") for ct in joined_pt_proxy.target.constituent_tables: - self.assertEqual(len(ct.columns), 4) + self.assertEqual(len(ct.definition), 4) def test_exact_join(self): with self.subTest("Join with a Table"): @@ -171,7 +171,7 @@ def test_exact_join(self): right_table = self.test_table.drop_columns(["b", "c"]).group_by('a') joined_pt_proxy = pt_proxy.exact_join(right_table, on="a", joins=["d", "e"]) for ct, jct in zip(pt_proxy.target.constituent_tables, joined_pt_proxy.target.constituent_tables): - self.assertEqual(len(jct.columns), 5) + self.assertEqual(len(jct.definition), 5) self.assertEqual(ct.size, jct.size) self.assertLessEqual(jct.size, right_table.size) @@ -180,7 +180,7 @@ def test_exact_join(self): right_proxy = self.test_table.drop_columns(["b", "d"]).partition_by("c").proxy() joined_pt_proxy = pt_proxy.exact_join(right_proxy, on="a", joins="e") for ct, jct in zip(pt_proxy.target.constituent_tables, joined_pt_proxy.target.constituent_tables): - self.assertEqual(len(jct.columns), 4) + self.assertEqual(len(jct.definition), 4) self.assertEqual(ct.size, jct.size) self.assertLessEqual(jct.size, right_table.size) @@ -247,7 +247,7 @@ def test_count_by(self): agg_pt_proxy = self.pt_proxy.count_by(col="cnt", by=["a"]) for gct, ct in zip(agg_pt_proxy.target.constituent_tables, self.pt_proxy.target.constituent_tables): self.assertLessEqual(gct.size, ct.size) - self.assertEqual(len(gct.columns), 2) + self.assertEqual(len(gct.definition), 2) def test_dedicated_agg(self): ops = [ @@ -268,7 +268,7 @@ def test_dedicated_agg(self): agg_pt_proxy = op(self.pt_proxy, by=["a", "b"]) for gct, ct in zip(agg_pt_proxy.target.constituent_tables, self.pt_proxy.target.constituent_tables): self.assertLessEqual(gct.size, ct.size) - self.assertEqual(len(gct.columns), len(ct.columns)) + self.assertEqual(len(gct.definition), len(ct.definition)) wops = [PartitionedTableProxy.weighted_avg_by, PartitionedTableProxy.weighted_sum_by, @@ -279,7 +279,7 @@ def test_dedicated_agg(self): agg_pt_proxy = wop(self.pt_proxy, wcol="e", by=["a", "b"]) for gct, ct in zip(agg_pt_proxy.target.constituent_tables, self.pt_proxy.target.constituent_tables): self.assertLessEqual(gct.size, ct.size) - self.assertEqual(len(gct.columns), len(ct.columns) - 1) + self.assertEqual(len(gct.definition), len(ct.definition) - 1) def test_agg_by(self): aggs = [ @@ -295,7 +295,7 @@ def test_agg_by(self): agg_pt_proxy = self.pt_proxy.agg_by(aggs=aggs, by=["a"]) for gct, ct in zip(agg_pt_proxy.target.constituent_tables, self.pt_proxy.target.constituent_tables): self.assertLessEqual(gct.size, ct.size) - self.assertEqual(len(gct.columns), 8) + self.assertEqual(len(gct.definition), 8) def test_agg_all_by(self): aggs = [ diff --git a/py/server/tests/test_table.py b/py/server/tests/test_table.py index 6b7ecf2168c..b37b89c8c14 100644 --- a/py/server/tests/test_table.py +++ b/py/server/tests/test_table.py @@ -14,7 +14,7 @@ from deephaven.html import to_html from deephaven.jcompat import j_hashmap from deephaven.pandas import to_pandas -from deephaven.table import Table, SearchDisplayMode, table_diff +from deephaven.table import Table, TableDefinition, SearchDisplayMode, table_diff from tests.testbase import BaseTestCase, table_equals @@ -84,9 +84,19 @@ def test_eq(self): t = self.test_table.where(["a > 500"]) self.assertNotEqual(t, self.test_table) + def test_definition(self): + expected = TableDefinition({ + "a": dtypes.int32, + "b": dtypes.int32, + "c": dtypes.int32, + "d": dtypes.int32, + "e": dtypes.int32 + }) + self.assertEquals(expected, self.test_table.definition) + def test_meta_table(self): t = self.test_table.meta_table - self.assertEqual(len(self.test_table.columns), t.size) + self.assertEqual(len(self.test_table.definition), t.size) def test_coalesce(self): t = self.test_table.update_view(["A = a * b"]) @@ -100,45 +110,45 @@ def test_flatten(self): self.assertTrue(ct.is_flat) def test_drop_columns(self): - column_names = [f.name for f in self.test_table.columns] + column_names = self.test_table.column_names result_table = self.test_table.drop_columns(cols=column_names[:-1]) - self.assertEqual(1, len(result_table.columns)) + self.assertEqual(1, len(result_table.definition)) result_table = self.test_table.drop_columns(cols=column_names[-1]) - self.assertEqual(1, len(self.test_table.columns) - len(result_table.columns)) + self.assertEqual(1, len(self.test_table.definition) - len(result_table.definition)) def test_move_columns(self): - column_names = [f.name for f in self.test_table.columns] + column_names = self.test_table.column_names cols_to_move = column_names[::2] with self.subTest("move-columns"): result_table = self.test_table.move_columns(1, cols_to_move) - result_cols = [f.name for f in result_table.columns] + result_cols = result_table.column_names self.assertEqual(cols_to_move, result_cols[1: len(cols_to_move) + 1]) with self.subTest("move-columns-up"): result_table = self.test_table.move_columns_up(cols_to_move) - result_cols = [f.name for f in result_table.columns] + result_cols = result_table.column_names self.assertEqual(cols_to_move, result_cols[: len(cols_to_move)]) with self.subTest("move-columns-down"): result_table = self.test_table.move_columns_down(cols_to_move) - result_cols = [f.name for f in result_table.columns] + result_cols = result_table.column_names self.assertEqual(cols_to_move, result_cols[-len(cols_to_move):]) cols_to_move = column_names[-1] with self.subTest("move-column"): result_table = self.test_table.move_columns(1, cols_to_move) - result_cols = [f.name for f in result_table.columns] + result_cols = result_table.column_names self.assertEqual([cols_to_move], result_cols[1: len(cols_to_move) + 1]) with self.subTest("move-column-up"): result_table = self.test_table.move_columns_up(cols_to_move) - result_cols = [f.name for f in result_table.columns] + result_cols = result_table.column_names self.assertEqual([cols_to_move], result_cols[: len(cols_to_move)]) with self.subTest("move-column-down"): result_table = self.test_table.move_columns_down(cols_to_move) - result_cols = [f.name for f in result_table.columns] + result_cols = result_table.column_names self.assertEqual([cols_to_move], result_cols[-len(cols_to_move):]) def test_rename_columns(self): @@ -147,10 +157,10 @@ def test_rename_columns(self): ] new_names = [cn.split("=")[0].strip() for cn in cols_to_rename] result_table = self.test_table.rename_columns(cols_to_rename) - result_cols = [f.name for f in result_table.columns] + result_cols = result_table.column_names self.assertEqual(new_names, result_cols[::2]) result_table = self.test_table.rename_columns(cols_to_rename[0]) - result_cols = [f.name for f in result_table.columns] + result_cols = result_table.column_names self.assertEqual(new_names[0], result_cols[::2][0]) def test_update_error(self): @@ -174,14 +184,14 @@ def test_USV(self): result_table = op( self.test_table, formulas=["a", "c", "Sum = a + b + c + d"]) self.assertIsNotNone(result_table) - self.assertTrue(len(result_table.columns) >= 3) + self.assertTrue(len(result_table.definition) >= 3) self.assertLessEqual(result_table.size, self.test_table.size) for op in ops: with self.subTest(op=op): result_table = op(self.test_table, formulas="Sum = a + b + c + d") self.assertIsNotNone(result_table) - self.assertTrue(len(result_table.columns) >= 1) + self.assertTrue(len(result_table.definition) >= 1) self.assertLessEqual(result_table.size, self.test_table.size) def test_select_distinct(self): @@ -430,10 +440,10 @@ def test_dedicated_agg(self): for wop in wops: with self.subTest(wop): result_table = wop(self.test_table, wcol='e', by=["a", "b"]) - self.assertEqual(len(result_table.columns), len(self.test_table.columns) - 1) + self.assertEqual(len(result_table.definition), len(self.test_table.definition) - 1) result_table = wop(self.test_table, wcol='e') - self.assertEqual(len(result_table.columns), len(self.test_table.columns) - 1) + self.assertEqual(len(result_table.definition), len(self.test_table.definition) - 1) def test_count_by(self): num_distinct_a = self.test_table.select_distinct(formulas=["a"]).size @@ -530,26 +540,26 @@ def test_snapshot_when(self): snapshot = self.test_table.snapshot_when(t) self.wait_ticking_table_update(snapshot, row_count=1, timeout=5) self.assertEqual(self.test_table.size, snapshot.size) - self.assertEqual(len(t.columns) + len(self.test_table.columns), len(snapshot.columns)) + self.assertEqual(len(t.definition) + len(self.test_table.definition), len(snapshot.definition)) with self.subTest("initial=True"): snapshot = self.test_table.snapshot_when(t, initial=True) self.assertEqual(self.test_table.size, snapshot.size) - self.assertEqual(len(t.columns) + len(self.test_table.columns), len(snapshot.columns)) + self.assertEqual(len(t.definition) + len(self.test_table.definition), len(snapshot.definition)) with self.subTest("stamp_cols=\"X\""): snapshot = self.test_table.snapshot_when(t, stamp_cols="X") - self.assertEqual(len(snapshot.columns), len(self.test_table.columns) + 1) + self.assertEqual(len(snapshot.definition), len(self.test_table.definition) + 1) with self.subTest("stamp_cols=[\"X\", \"Y\"]"): snapshot = self.test_table.snapshot_when(t, stamp_cols=["X", "Y"]) - self.assertEqual(len(snapshot.columns), len(self.test_table.columns) + 2) + self.assertEqual(len(snapshot.definition), len(self.test_table.definition) + 2) def test_snapshot_when_with_history(self): t = time_table("PT00:00:01") snapshot_hist = self.test_table.snapshot_when(t, history=True) self.wait_ticking_table_update(snapshot_hist, row_count=1, timeout=5) - self.assertEqual(1 + len(self.test_table.columns), len(snapshot_hist.columns)) + self.assertEqual(1 + len(self.test_table.definition), len(snapshot_hist.definition)) self.assertEqual(self.test_table.size, snapshot_hist.size) t = time_table("PT0.1S").update("X = i % 2 == 0 ? i : i - 1").sort("X").tail(10) @@ -920,6 +930,12 @@ def test_attributes(self): self.assertEqual(len(attrs), len(rt_attrs) + 1) self.assertIn("BlinkTable", set(attrs.keys()) - set(rt_attrs.keys())) + def test_remove_blink(self): + t_blink = time_table("PT1s", blink_table=True) + t_no_blink = t_blink.remove_blink() + self.assertEqual(t_blink.is_blink, True) + self.assertEqual(t_no_blink.is_blink, False) + def test_grouped_column_as_arg(self): t1 = empty_table(100).update( ["id = i % 10", "Person = random() > 0.5 ? true : random() > 0.5 ? false : true"]).sort( @@ -1020,7 +1036,7 @@ def test_range_join(self): right_table = self.test_table.select_distinct().sort("b").drop_columns("e") result_table = left_table.range_join(right_table, on=["a = a", "c < b < e"], aggs=aggs) self.assertEqual(result_table.size, left_table.size) - self.assertEqual(len(result_table.columns), len(left_table.columns) + len(aggs)) + self.assertEqual(len(result_table.definition), len(left_table.definition) + len(aggs)) with self.assertRaises(DHError): time_table("PT00:00:00.001").update("a = i").range_join(right_table, on=["a = a", "a < b < c"], aggs=aggs) diff --git a/py/server/tests/test_table_definition.py b/py/server/tests/test_table_definition.py new file mode 100644 index 00000000000..d4d2cd34f86 --- /dev/null +++ b/py/server/tests/test_table_definition.py @@ -0,0 +1,271 @@ +# +# Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +# +import unittest +from typing import Mapping +from deephaven import dtypes, new_table, DHError +from deephaven.table import TableDefinition +from deephaven.column import col_def, string_col, bool_col +from tests.testbase import BaseTestCase + + +class TableDefinitionTestCase(BaseTestCase): + def setUp(self): + super().setUp() + self.test_definition = TableDefinition( + { + "Bool": dtypes.bool_, + "Char": dtypes.char, + "Short": dtypes.short, + "Int": dtypes.int32, + "Long": dtypes.int64, + "Float": dtypes.float32, + "Double": dtypes.float64, + "String": dtypes.string, + "Instant": dtypes.Instant, + } + ) + + def tearDown(self) -> None: + self.test_definition = None + super().tearDown() + + def test_is_mapping(self): + self.assertTrue(isinstance(self.test_definition, Mapping)) + + def test_length(self): + self.assertEquals(9, len(self.test_definition)) + + def test_contains(self): + self.assertTrue("Bool" in self.test_definition) + self.assertTrue("Char" in self.test_definition) + self.assertTrue("Short" in self.test_definition) + self.assertTrue("Int" in self.test_definition) + self.assertTrue("Long" in self.test_definition) + self.assertTrue("Float" in self.test_definition) + self.assertTrue("Double" in self.test_definition) + self.assertTrue("String" in self.test_definition) + self.assertTrue("Instant" in self.test_definition) + self.assertFalse("FooBarBaz" in self.test_definition) + + def test_getitem(self): + self.assertEquals(col_def("Bool", dtypes.bool_), self.test_definition["Bool"]) + self.assertEquals(col_def("Char", dtypes.char), self.test_definition["Char"]) + self.assertEquals(col_def("Short", dtypes.short), self.test_definition["Short"]) + self.assertEquals(col_def("Int", dtypes.int32), self.test_definition["Int"]) + self.assertEquals(col_def("Long", dtypes.int64), self.test_definition["Long"]) + self.assertEquals( + col_def("Float", dtypes.float32), self.test_definition["Float"] + ) + self.assertEquals( + col_def("Double", dtypes.float64), + self.test_definition["Double"], + ) + self.assertEquals( + col_def("String", dtypes.string), self.test_definition["String"] + ) + self.assertEquals( + col_def("Instant", dtypes.Instant), + self.test_definition["Instant"], + ) + with self.assertRaises(KeyError): + self.test_definition["FooBarBaz"] + + def test_get(self): + self.assertEquals( + col_def("Bool", dtypes.bool_), self.test_definition.get("Bool") + ) + self.assertEquals( + col_def("Char", dtypes.char), self.test_definition.get("Char") + ) + self.assertEquals( + col_def("Short", dtypes.short), + self.test_definition.get("Short"), + ) + self.assertEquals(col_def("Int", dtypes.int32), self.test_definition.get("Int")) + self.assertEquals( + col_def("Long", dtypes.int64), self.test_definition.get("Long") + ) + self.assertEquals( + col_def("Float", dtypes.float32), + self.test_definition.get("Float"), + ) + self.assertEquals( + col_def("Double", dtypes.float64), + self.test_definition.get("Double"), + ) + self.assertEquals( + col_def("String", dtypes.string), + self.test_definition.get("String"), + ) + self.assertEquals( + col_def("Instant", dtypes.Instant), + self.test_definition.get("Instant"), + ) + self.assertEquals(None, self.test_definition.get("FooBarBaz")) + + def test_iter(self): + self.assertEquals( + [ + "Bool", + "Char", + "Short", + "Int", + "Long", + "Float", + "Double", + "String", + "Instant", + ], + list(iter(self.test_definition)), + ) + + def test_keys(self): + self.assertEquals( + [ + "Bool", + "Char", + "Short", + "Int", + "Long", + "Float", + "Double", + "String", + "Instant", + ], + list(self.test_definition.keys()), + ) + + def test_values(self): + self.assertEquals( + [ + col_def("Bool", dtypes.bool_), + col_def("Char", dtypes.char), + col_def("Short", dtypes.short), + col_def("Int", dtypes.int32), + col_def("Long", dtypes.int64), + col_def("Float", dtypes.float32), + col_def("Double", dtypes.float64), + col_def("String", dtypes.string), + col_def("Instant", dtypes.Instant), + ], + list(self.test_definition.values()), + ) + + def test_items(self): + self.assertEquals( + [ + ("Bool", col_def("Bool", dtypes.bool_)), + ("Char", col_def("Char", dtypes.char)), + ("Short", col_def("Short", dtypes.short)), + ("Int", col_def("Int", dtypes.int32)), + ("Long", col_def("Long", dtypes.int64)), + ("Float", col_def("Float", dtypes.float32)), + ("Double", col_def("Double", dtypes.float64)), + ("String", col_def("String", dtypes.string)), + ("Instant", col_def("Instant", dtypes.Instant)), + ], + list(self.test_definition.items()), + ) + + def test_equals_hash_and_from_columns(self): + expected_hash = hash(self.test_definition) + for actual in [ + # should be equal to the same exact object + self.test_definition, + # should be equal to a new python object, but same underlying java object + TableDefinition(self.test_definition), + # should be equal to a new python object and new underlying java object + TableDefinition(self.test_definition.values()), + ]: + self.assertEquals(actual, self.test_definition) + self.assertEquals(hash(actual), expected_hash) + + def test_meta_table(self): + expected = new_table( + [ + string_col( + "Name", + [ + "Bool", + "Char", + "Short", + "Int", + "Long", + "Float", + "Double", + "String", + "Instant", + ], + ), + string_col( + "DataType", + [ + "java.lang.Boolean", + "char", + "short", + "int", + "long", + "float", + "double", + "java.lang.String", + "java.time.Instant", + ], + ), + string_col("ColumnType", ["Normal"] * 9), + bool_col("IsPartitioning", [False] * 9), + ] + ) + + self.assert_table_equals(self.test_definition.table, expected) + + def test_from_TableDefinition(self): + self.assertEquals(TableDefinition(self.test_definition), self.test_definition) + + def test_from_JpyJType(self): + self.assertEquals( + TableDefinition(self.test_definition.j_table_definition), + self.test_definition, + ) + + def test_from_Mapping(self): + # This case is already tested, it's how self.test_definition is created + pass + + def test_from_Iterable(self): + self.assertEquals( + TableDefinition(self.test_definition.values()), self.test_definition + ) + self.assertEquals( + TableDefinition(list(self.test_definition.values())), self.test_definition + ) + + def test_from_unexpected_type(self): + with self.assertRaises(DHError): + TableDefinition(42) + + def test_bad_Mapping_key(self): + with self.assertRaises(DHError): + TableDefinition( + { + "Foo": dtypes.int32, + 42: dtypes.string, + } + ) + + def test_bad_Mapping_value(self): + with self.assertRaises(DHError): + TableDefinition( + { + "Foo": dtypes.int32, + "Bar": 42, + } + ) + + def test_bad_Iterable(self): + with self.assertRaises(DHError): + TableDefinition([col_def("Foo", dtypes.int32), 42]) + + +if __name__ == "__main__": + unittest.main() diff --git a/py/server/tests/test_table_factory.py b/py/server/tests/test_table_factory.py index 3b1cfe55062..c4fb1cbea36 100644 --- a/py/server/tests/test_table_factory.py +++ b/py/server/tests/test_table_factory.py @@ -41,7 +41,7 @@ def tearDown(self) -> None: def test_empty_table(self): t = empty_table(10) - self.assertEqual(0, len(t.columns)) + self.assertEqual(0, len(t.definition)) def test_empty_table_error(self): with self.assertRaises(DHError) as cm: @@ -52,22 +52,22 @@ def test_empty_table_error(self): def test_time_table(self): t = time_table("PT00:00:01") - self.assertEqual(1, len(t.columns)) + self.assertEqual(1, len(t.definition)) self.assertTrue(t.is_refreshing) t = time_table("PT00:00:01", start_time="2021-11-06T13:21:00 ET") - self.assertEqual(1, len(t.columns)) + self.assertEqual(1, len(t.definition)) self.assertTrue(t.is_refreshing) self.assertEqual("2021-11-06T13:21:00.000000000 ET", _JDateTimeUtils.formatDateTime(t.j_table.getColumnSource("Timestamp").get(0), time.to_j_time_zone('ET'))) t = time_table(1000_000_000) - self.assertEqual(1, len(t.columns)) + self.assertEqual(1, len(t.definition)) self.assertTrue(t.is_refreshing) t = time_table(1000_1000_1000, start_time="2021-11-06T13:21:00 ET") - self.assertEqual(1, len(t.columns)) + self.assertEqual(1, len(t.definition)) self.assertTrue(t.is_refreshing) self.assertEqual("2021-11-06T13:21:00.000000000 ET", _JDateTimeUtils.formatDateTime(t.j_table.getColumnSource("Timestamp").get(0), @@ -75,12 +75,12 @@ def test_time_table(self): p = time.to_timedelta(time.to_j_duration("PT1s")) t = time_table(p) - self.assertEqual(1, len(t.columns)) + self.assertEqual(1, len(t.definition)) self.assertTrue(t.is_refreshing) st = time.to_datetime(time.to_j_instant("2021-11-06T13:21:00 ET")) t = time_table(p, start_time=st) - self.assertEqual(1, len(t.columns)) + self.assertEqual(1, len(t.definition)) self.assertTrue(t.is_refreshing) self.assertEqual("2021-11-06T13:21:00.000000000 ET", _JDateTimeUtils.formatDateTime(t.j_table.getColumnSource("Timestamp").get(0), @@ -88,15 +88,18 @@ def test_time_table(self): def test_time_table_blink(self): t = time_table("PT1s", blink_table=True) - self.assertEqual(1, len(t.columns)) + self.assertEqual(1, len(t.definition)) self.assertTrue(t.is_blink) def test_time_table_error(self): with self.assertRaises(DHError) as cm: - t = time_table("PT00:0a:01") + time_table("PT00:0a:01") self.assertIn("DateTimeParseException", cm.exception.root_cause) + with self.assertRaises(DHError): + time_table(None) + def test_merge(self): t1 = self.test_table.update(formulas=["Timestamp=epochNanosToInstant(0L)"]) t2 = self.test_table.update(formulas=["Timestamp=nowSystem()"]) @@ -325,19 +328,18 @@ def test_input_table(self): ] t = new_table(cols=cols) self.assertEqual(t.size, 2) - col_defs = {c.name: c.data_type for c in t.columns} with self.subTest("from table definition"): - append_only_input_table = input_table(col_defs=col_defs) + append_only_input_table = input_table(col_defs=t.definition) self.assertEqual(append_only_input_table.key_names, []) - self.assertEqual(append_only_input_table.value_names, [col.name for col in cols]) + self.assertEqual(append_only_input_table.value_names, [col._column_definition.name for col in cols]) append_only_input_table.add(t) self.assertEqual(append_only_input_table.size, 2) append_only_input_table.add(t) self.assertEqual(append_only_input_table.size, 4) - keyed_input_table = input_table(col_defs=col_defs, key_cols="String") + keyed_input_table = input_table(col_defs=t.definition, key_cols="String") self.assertEqual(keyed_input_table.key_names, ["String"]) - self.assertEqual(keyed_input_table.value_names, [col.name for col in cols if col.name != "String"]) + self.assertEqual(keyed_input_table.value_names, [col._column_definition.name for col in cols if col._column_definition.name != "String"]) keyed_input_table.add(t) self.assertEqual(keyed_input_table.size, 2) keyed_input_table.add(t) @@ -346,14 +348,14 @@ def test_input_table(self): with self.subTest("from init table"): append_only_input_table = input_table(init_table=t) self.assertEqual(append_only_input_table.key_names, []) - self.assertEqual(append_only_input_table.value_names, [col.name for col in cols]) + self.assertEqual(append_only_input_table.value_names, [col._column_definition.name for col in cols]) self.assertEqual(append_only_input_table.size, 2) append_only_input_table.add(t) self.assertEqual(append_only_input_table.size, 4) keyed_input_table = input_table(init_table=t, key_cols="String") self.assertEqual(keyed_input_table.key_names, ["String"]) - self.assertEqual(keyed_input_table.value_names, [col.name for col in cols if col.name != "String"]) + self.assertEqual(keyed_input_table.value_names, [col._column_definition.name for col in cols if col._column_definition.name != "String"]) self.assertEqual(keyed_input_table.size, 2) keyed_input_table.add(t) self.assertEqual(keyed_input_table.size, 2) @@ -368,7 +370,7 @@ def test_input_table(self): keyed_input_table = input_table(init_table=t, key_cols=["String", "Double"]) self.assertEqual(keyed_input_table.key_names, ["String", "Double"]) - self.assertEqual(keyed_input_table.value_names, [col.name for col in cols if col.name != "String" and col.name != "Double"]) + self.assertEqual(keyed_input_table.value_names, [col._column_definition.name for col in cols if col._column_definition.name != "String" and col._column_definition.name != "Double"]) self.assertEqual(keyed_input_table.size, 2) keyed_input_table.delete(t.select(["String", "Double"])) self.assertEqual(keyed_input_table.size, 0) @@ -449,7 +451,7 @@ def test_input_table_empty_data(self): with cm: t = time_table("PT1s", blink_table=True) - it = input_table({c.name: c.data_type for c in t.columns}, key_cols="Timestamp") + it = input_table(t.definition, key_cols="Timestamp") it.add(t) self.assertEqual(it.size, 0) it.delete(t) @@ -467,8 +469,7 @@ def test_j_input_wrapping(self): string_col(name="String", data=["foo", "bar"]), ] t = new_table(cols=cols) - col_defs = {c.name: c.data_type for c in t.columns} - append_only_input_table = input_table(col_defs=col_defs) + append_only_input_table = input_table(col_defs=t.definition) it = _wrapper.wrap_j_object(append_only_input_table.j_table) self.assertTrue(isinstance(it, InputTable)) diff --git a/py/server/tests/test_table_iterator.py b/py/server/tests/test_table_iterator.py index 465ba913453..0bb617fa6f5 100644 --- a/py/server/tests/test_table_iterator.py +++ b/py/server/tests/test_table_iterator.py @@ -22,7 +22,7 @@ def test_iteration_in_chunks(self): test_table = read_csv("tests/data/test_table.csv") total_read_size = 0 for d in test_table.iter_chunk_dict(chunk_size=10): - self.assertEqual(len(d), len(test_table.columns)) + self.assertEqual(len(d), len(test_table.definition)) for col in test_table.columns: self.assertIn(col.name, d) self.assertEqual(d[col.name].dtype, col.data_type.np_type) @@ -36,7 +36,7 @@ def test_iteration_in_chunks(self): test_table.await_update() total_read_size = 0 for d in test_table.iter_chunk_dict(chunk_size=100): - self.assertEqual(len(d), len(test_table.columns)) + self.assertEqual(len(d), len(test_table.definition)) for col in test_table.columns: self.assertIn(col.name, d) self.assertEqual(d[col.name].dtype, col.data_type.np_type) @@ -65,7 +65,7 @@ def test_iteration_in_rows(self): test_table = read_csv("tests/data/test_table.csv") total_read_size = 0 for d in test_table.iter_dict(): - self.assertEqual(len(d), len(test_table.columns)) + self.assertEqual(len(d), len(test_table.definition)) for col in test_table.columns: self.assertIn(col.name, d) self.assertTrue(np.can_cast(col.data_type.np_type, np.dtype(type(d[col.name])))) @@ -77,7 +77,7 @@ def test_iteration_in_rows(self): test_table.await_update() total_read_size = 0 for d in test_table.iter_dict(): - self.assertEqual(len(d), len(test_table.columns)) + self.assertEqual(len(d), len(test_table.definition)) for col in test_table.columns: self.assertIn(col.name, d) v_type = type(d[col.name]) @@ -108,7 +108,7 @@ def test_direct_call_chunks(self): test_table = read_csv("tests/data/test_table.csv") t_iter = test_table.iter_chunk_dict(chunk_size=10) for d in t_iter: - self.assertEqual(len(d), len(test_table.columns)) + self.assertEqual(len(d), len(test_table.definition)) for col in test_table.columns: self.assertIn(col.name, d) self.assertEqual(d[col.name].dtype, col.data_type.np_type) @@ -159,7 +159,7 @@ def test_direct_call_rows(self): test_table = read_csv("tests/data/test_table.csv") t_iter = test_table.iter_dict() for d in t_iter: - self.assertEqual(len(d), len(test_table.columns)) + self.assertEqual(len(d), len(test_table.definition)) for col in test_table.columns: self.assertIn(col.name, d) self.assertTrue(np.can_cast(col.data_type.np_type, np.dtype(type(d[col.name])))) @@ -232,7 +232,7 @@ class CustomClass: with self.subTest("Chunks"): for d in test_table.iter_chunk_dict(chunk_size=10): - self.assertEqual(len(d), len(test_table.columns)) + self.assertEqual(len(d), len(test_table.definition)) for col in test_table.columns: self.assertIn(col.name, d) self.assertEqual(dtypes.from_np_dtype(d[col.name].dtype).np_type, col.data_type.np_type) @@ -240,7 +240,7 @@ class CustomClass: with self.subTest("Rows"): for d in test_table.iter_dict(): - self.assertEqual(len(d), len(test_table.columns)) + self.assertEqual(len(d), len(test_table.definition)) for col in test_table.columns: self.assertIn(col.name, d) v_type = type(d[col.name]) @@ -258,7 +258,7 @@ def test_iteration_in_chunks_tuple(self): test_table = read_csv("tests/data/test_table.csv") total_read_size = 0 for d in test_table.iter_chunk_tuple(chunk_size=10): - self.assertEqual(len(d), len(test_table.columns)) + self.assertEqual(len(d), len(test_table.definition)) for i, col in enumerate(test_table.columns): self.assertEqual(col.name, d._fields[i]) self.assertEqual(d[i].dtype, col.data_type.np_type) @@ -272,7 +272,7 @@ def test_iteration_in_chunks_tuple(self): test_table.await_update() total_read_size = 0 for d in test_table.iter_chunk_tuple(chunk_size=100): - self.assertEqual(len(d), len(test_table.columns)) + self.assertEqual(len(d), len(test_table.definition)) for i, col in enumerate(test_table.columns): self.assertEqual(col.name, d._fields[i]) self.assertEqual(d[i].dtype, col.data_type.np_type) @@ -301,7 +301,7 @@ def test_iteration_in_rows_tuple(self): test_table = read_csv("tests/data/test_table.csv") total_read_size = 0 for d in test_table.iter_tuple(): - self.assertEqual(len(d), len(test_table.columns)) + self.assertEqual(len(d), len(test_table.definition)) for i, col in enumerate(test_table.columns): self.assertEqual(col.name, d._fields[i]) self.assertTrue(np.can_cast(col.data_type.np_type, np.dtype(type(d[i])))) @@ -313,7 +313,7 @@ def test_iteration_in_rows_tuple(self): test_table.await_update() total_read_size = 0 for d in test_table.iter_tuple(): - self.assertEqual(len(d), len(test_table.columns)) + self.assertEqual(len(d), len(test_table.definition)) for i, col in enumerate(test_table.columns): self.assertEqual(col.name, d._fields[i]) v_type = type(d[i]) diff --git a/py/server/tests/test_table_listener.py b/py/server/tests/test_table_listener.py index db570b77414..e0e6cadb64e 100644 --- a/py/server/tests/test_table_listener.py +++ b/py/server/tests/test_table_listener.py @@ -22,6 +22,7 @@ _JColumnVectors = jpy.get_type("io.deephaven.engine.table.vectors.ColumnVectors") + class TableUpdateRecorder: def __init__(self, table: Optional[Table] = None, chunk_size: int = None, cols: Union[str, List[str]] = None): self.table = table @@ -104,7 +105,7 @@ def verify_data_changes(self, changes, cols: Union[str, List[str]]): for change in changes: self.assertTrue(isinstance(change, dict)) if not cols: - cols = [col.name for col in self.test_table.columns] + cols = self.test_table.column_names for col in cols: self.assertIn(col, change.keys()) self.assertTrue(isinstance(change[col], numpy.ndarray)) @@ -266,6 +267,8 @@ def on_update(self, update, is_replay): has_added=True, has_removed=True, has_modified=False) self.assertTrue(all([len(ja) > 0 for ja in j_arrays])) + dep_table = dep_table_2 = None + def test_listener_func_with_deps(self): cols = [ @@ -274,8 +277,7 @@ def test_listener_func_with_deps(self): ] t = new_table(cols=cols) self.assertEqual(t.size, 2) - col_defs = {c.name: c.data_type for c in t.columns} - dep_table = input_table(col_defs=col_defs) + dep_table = input_table(col_defs=t.definition) def listener_func(update, is_replay): table_update_recorder.record(update, is_replay) @@ -363,6 +365,8 @@ def on_update(self, updates: Dict[Table, TableUpdate], is_replay: bool) -> None: mlh.stop() self.assertGreaterEqual(len(tur.replays), 6) + t1 = t2 = t3 = None + def test_merged_listener_func(self): t1 = time_table("PT1s").update(["X=i % 11"]) t2 = time_table("PT2s").update(["Y=i % 8"]) @@ -393,6 +397,8 @@ def test_ml_func(updates: Dict[Table, TableUpdate], is_replay: bool) -> None: mlh.stop() self.assertGreaterEqual(len(tur.replays), 6) + t1 = t2 = t3 = None + def test_merged_listener_with_deps(self): t1 = time_table("PT1s").update(["X=i % 11"]) t2 = time_table("PT2s").update(["Y=i % 8"]) @@ -423,6 +429,8 @@ def on_update(self, updates: Dict[Table, TableUpdate], is_replay: bool) -> None: self.assertGreaterEqual(len(tur.replays), 6) self.assertTrue(len(j_arrays) > 0 and all([len(ja) > 0 for ja in j_arrays])) + t1 = t2 = t3 = None + def test_merged_listener_error(self): t1 = time_table("PT1s").update(["X=i % 11"]) @@ -438,6 +446,8 @@ def test_ml_func(updates: Dict[Table, TableUpdate]) -> None: mlh = merged_listen([t1, et], test_ml_func) self.assertIn("must be a refreshing table", str(cm.exception)) + t1 = et = None + def test_merged_listener_replay(self): t1 = time_table("PT1s").update(["X=i % 11"]) t2 = time_table("PT2s").update(["Y=i % 8"]) @@ -478,6 +488,277 @@ def test_ml_func(updates: Dict[Table, TableUpdate], is_replay: bool) -> None: self.assertGreaterEqual(len(tur.replays), 6) self.assertEqual(tur.replays.count(True), 2) + t1 = t2 = t3 = None + + def test_on_error_listener_func(self): + t = time_table("PT1S").update("X = i") + with self.subTest("Bad Listener Good Error Callback"): + def bad_listner_func(table_udpate, is_replay: bool) -> None: + raise ValueError("invalid value") + + def on_error(e: Exception) -> None: + nonlocal error_caught + error_caught = True + self.assertIn("invalid value", str(e)) + + error_caught = False + tlh = listen(t, bad_listner_func, on_error=on_error) + t.await_update() + self.assertTrue(error_caught) + self.assertTrue(tlh.j_object.isFailed()) + + with self.subTest("Good Listener Good Error Callback"): + def good_listner_func(table_udpate, is_replay: bool) -> None: + pass + + error_caught = False + tlh = listen(t, good_listner_func, on_error=on_error) + t.await_update() + self.assertFalse(error_caught) + self.assertFalse(tlh.j_object.isFailed()) + + with self.subTest("Bad Listener Bad Error Callback"): + error_caught: bool = False + + def bad_listner_func(table_udpate, is_replay: bool) -> None: + raise ValueError("invalid value") + + def on_error(e: Exception) -> None: + nonlocal error_caught + error_caught = True + self.assertIn("invalid value", str(e)) + raise ValueError("reraise the exception") from e + + tlh = listen(t, bad_listner_func, on_error=on_error) + t.await_update() + self.assertTrue(error_caught) + self.assertTrue(tlh.j_object.isFailed()) + + t = None + + def test_on_error_listener_obj(self): + test_self = self + t = time_table("PT1S").update("X = i") + + with self.subTest("Bad Listener Good Error Callback"): + class BadListener(TableListener): + def on_update(self, update: TableUpdate, is_replay: bool) -> None: + raise ValueError("invalid value") + + def on_error(self, e: Exception) -> None: + nonlocal error_caught + error_caught = True + test_self.assertIn("invalid value", str(e)) + + error_caught = False + bad_listener_obj = BadListener() + tlh = listen(t, bad_listener_obj) + t.await_update() + self.assertTrue(error_caught) + self.assertTrue(tlh.j_object.isFailed()) + + with self.assertRaises(DHError): + def on_error(e: Exception) -> None: + ... + tlh = listen(t, bad_listener_obj, on_error=on_error) + + with self.subTest("Good Listener Good Error Callback"): + class GoodListener(TableListener): + def on_update(self, update: TableUpdate, is_replay: bool) -> None: + ... + + def on_error(self, e: Exception) -> None: + nonlocal error_caught + error_caught = True + test_self.assertIn("invalid value", str(e)) + + error_caught = False + good_listener_obj = GoodListener() + tlh = listen(t, good_listener_obj) + t.await_update() + self.assertFalse(error_caught) + self.assertFalse(tlh.j_object.isFailed()) + + with self.subTest("Bad Listener Bad Error Callback"): + class GoodListener(TableListener): + def on_update(self, update: TableUpdate, is_replay: bool) -> None: + raise ValueError("invalid value") + + def on_error(self, e: Exception) -> None: + nonlocal error_caught + error_caught = True + test_self.assertIn("invalid value", str(e)) + raise ValueError("reraise the exception") from e + + error_caught = False + + good_listener_obj = GoodListener() + tlh = listen(t, good_listener_obj) + t.await_update() + self.assertTrue(error_caught) + self.assertTrue(tlh.j_object.isFailed()) + + t = None + + def test_on_error_merged_listener_func(self): + t1 = time_table("PT1s").update(["X=i % 11"]) + t2 = time_table("PT2s").update(["Y=i % 8"]) + t3 = time_table("PT3s").update(["Z=i % 5"]) + + with self.subTest("Bad Listener Good Error Callback"): + def bad_listner_func(updates: Dict[Table, TableUpdate], is_replay: bool) -> None: + raise ValueError("invalid value") + + def on_error(e: Exception) -> None: + nonlocal error_caught + error_caught = True + self.assertIn("invalid value", str(e)) + + error_caught = False + mlh = merged_listen([t1, t2, t3], bad_listner_func, on_error=on_error) + t1.await_update() + self.assertTrue(error_caught) + self.assertTrue(mlh.j_object.isFailed()) + + with self.subTest("Good Listener Good Error Callback"): + def good_listner_func(updates: Dict[Table, TableUpdate], is_replay: bool) -> None: + pass + + error_caught = False + mlh = merged_listen([t1, t2, t3], good_listner_func, on_error=on_error) + t1.await_update() + self.assertFalse(error_caught) + self.assertFalse(mlh.j_object.isFailed()) + + with self.subTest("Bad Listener Bad Error Callback"): + def bad_listner_func(updates: Dict[Table, TableUpdate], is_replay: bool) -> None: + raise ValueError("invalid value") + + def bad_on_error(e: Exception) -> None: + nonlocal error_caught + error_caught = True + self.assertIn("invalid value", str(e)) + raise ValueError("reraise the exception") from e + + error_caught = False + mlh = merged_listen([t1, t2, t3], bad_listner_func, on_error=bad_on_error) + t1.await_update() + self.assertTrue(error_caught) + self.assertTrue(mlh.j_object.isFailed()) + + t1 = t2 = t3 = None + + def test_on_error_merged_listener_obj(self): + test_self = self + t1 = time_table("PT1s").update(["X=i % 11"]) + t2 = time_table("PT2s").update(["Y=i % 8"]) + t3 = time_table("PT3s").update(["Z=i % 5"]) + + with self.subTest("Bad Listener Good Error Callback"): + class BadListener(MergedListener): + def on_update(self, updates: Dict[Table, TableUpdate], is_replay: bool) -> None: + raise ValueError("invalid value") + + def on_error(self, e: Exception) -> None: + nonlocal error_caught + error_caught = True + test_self.assertIn("invalid value", str(e)) + + error_caught = False + bad_listener_obj = BadListener() + mlh = merged_listen([t1, t2, t3], bad_listener_obj) + t1.await_update() + self.assertTrue(error_caught) + self.assertTrue(mlh.j_object.isFailed()) + + with self.assertRaises(DHError): + def on_error(e: Exception) -> None: + ... + tlh = merged_listen([t1, t2, t3], bad_listener_obj, on_error=on_error) + + + with self.subTest("Good Listener Good Error Callback"): + class GoodListener(MergedListener): + def on_update(self, updates: Dict[Table, TableUpdate], is_replay: bool) -> None: + ... + + def on_error(self, e: Exception) -> None: + nonlocal error_caught + error_caught = True + test_self.assertIn("invalid value", str(e)) + + error_caught = False + good_listener_obj = GoodListener() + mlh = merged_listen([t1, t2, t3], good_listener_obj) + t1.await_update() + self.assertFalse(error_caught) + self.assertFalse(mlh.j_object.isFailed()) + + with self.subTest("Bad Listener Bad Error Callback"): + class BadListener(MergedListener): + def on_update(self, updates: Dict[Table, TableUpdate], is_replay: bool) -> None: + raise ValueError("invalid value") + + def on_error(self, e: Exception) -> None: + nonlocal error_caught + error_caught = True + test_self.assertIn("invalid value", str(e)) + raise ValueError("reraise the exception") from e + + error_caught = False + bad_listener_obj = BadListener() + mlh = merged_listen([t1, t2, t3], bad_listener_obj) + t1.await_update() + self.assertTrue(error_caught) + self.assertTrue(mlh.j_object.isFailed()) + + t1 = t2 = t3 = None + + def test_default_on_error(self): + t = time_table("PT1S").update("X = i") + + def bad_listner_func(table_udpate, is_replay: bool) -> None: + raise ValueError("invalid value") + + error_caught = False + tlh = listen(t, bad_listner_func) + t.await_update() + # the default on_error only logs the error + self.assertFalse(error_caught) + self.assertTrue(tlh.j_object.isFailed()) + + class BadListener(TableListener): + def on_update(self, update, is_replay): + raise ValueError("invalid value") + + tlh = listen(t, BadListener()) + t.await_update() + # the default on_error only logs the error + self.assertFalse(error_caught) + self.assertTrue(tlh.j_object.isFailed()) + + t2 = time_table("PT1S").update("X = i") + def bad_listner_func(updates: Dict[Table, TableUpdate], is_replay: bool) -> None: + raise ValueError("invalid value") + + mlh = merged_listen([t, t2], bad_listner_func) + t.await_update() + # the default on_error only logs the error + self.assertFalse(error_caught) + self.assertTrue(mlh.j_object.isFailed()) + + class BadListener(MergedListener): + def on_update(self, updates: Dict[Table, TableUpdate], is_replay: bool) -> None: + raise ValueError("invalid value") + + mlh = merged_listen([t, t2], BadListener()) + t.await_update() + # the default on_error only logs the error + self.assertFalse(error_caught) + self.assertTrue(mlh.j_object.isFailed()) + + t = t2 = None + if __name__ == "__main__": unittest.main() diff --git a/py/server/tests/test_updateby.py b/py/server/tests/test_updateby.py index e4ecbc2aae2..e58ce542539 100644 --- a/py/server/tests/test_updateby.py +++ b/py/server/tests/test_updateby.py @@ -177,7 +177,7 @@ def test_em(self): for t in (self.static_table, self.ticking_table): rt = t.update_by(ops=op, by="b") self.assertTrue(rt.is_refreshing is t.is_refreshing) - self.assertEqual(len(rt.columns), 1 + len(t.columns)) + self.assertEqual(len(rt.definition), 1 + len(t.definition)) with update_graph.exclusive_lock(self.test_update_graph): self.assertEqual(rt.size, t.size) @@ -192,7 +192,7 @@ def test_em_proxy(self): rt_proxy = pt_proxy.update_by(op, by="e") for ct, rct in zip(pt_proxy.target.constituent_tables, rt_proxy.target.constituent_tables): self.assertTrue(rct.is_refreshing is ct.is_refreshing) - self.assertEqual(len(rct.columns), 1 + len(ct.columns)) + self.assertEqual(len(rct.definition), 1 + len(ct.definition)) with update_graph.exclusive_lock(self.test_update_graph): self.assertEqual(ct.size, rct.size) @@ -202,7 +202,7 @@ def test_simple_ops(self): for t in (self.static_table, self.ticking_table): rt = t.update_by(ops=op, by="e") self.assertTrue(rt.is_refreshing is t.is_refreshing) - self.assertEqual(len(rt.columns), 2 + len(t.columns)) + self.assertEqual(len(rt.definition), 2 + len(t.definition)) with update_graph.exclusive_lock(self.test_update_graph): self.assertEqual(rt.size, t.size) @@ -230,7 +230,7 @@ def test_rolling_ops(self): for t in (self.static_table, self.ticking_table): rt = t.update_by(ops=op, by="c") self.assertTrue(rt.is_refreshing is t.is_refreshing) - self.assertEqual(len(rt.columns), 2 + len(t.columns)) + self.assertEqual(len(rt.definition), 2 + len(t.definition)) with update_graph.exclusive_lock(self.test_update_graph): self.assertEqual(rt.size, t.size) @@ -245,7 +245,7 @@ def test_rolling_ops_proxy(self): rt_proxy = pt_proxy.update_by(op, by="c") for ct, rct in zip(pt_proxy.target.constituent_tables, rt_proxy.target.constituent_tables): self.assertTrue(rct.is_refreshing is ct.is_refreshing) - self.assertEqual(len(rct.columns), 2 + len(ct.columns)) + self.assertEqual(len(rct.definition), 2 + len(ct.definition)) with update_graph.exclusive_lock(self.test_update_graph): self.assertEqual(ct.size, rct.size) @@ -260,7 +260,7 @@ def test_multiple_ops(self): for t in (self.static_table, self.ticking_table): rt = t.update_by(ops=multiple_ops, by="c") self.assertTrue(rt.is_refreshing is t.is_refreshing) - self.assertEqual(len(rt.columns), 10 + len(t.columns)) + self.assertEqual(len(rt.definition), 10 + len(t.definition)) with update_graph.exclusive_lock(self.test_update_graph): self.assertEqual(rt.size, t.size) diff --git a/py/server/tests/test_vectorization.py b/py/server/tests/test_vectorization.py index ebac32aff93..ab227d02cc5 100644 --- a/py/server/tests/test_vectorization.py +++ b/py/server/tests/test_vectorization.py @@ -234,7 +234,7 @@ def my_sum(*args): source = new_table([int_col(c, [0, 1, 2, 3, 4, 5, 6]) for c in cols]) result = source.update(f"X = my_sum({','.join(cols)})") - self.assertEqual(len(cols) + 1, len(result.columns)) + self.assertEqual(len(cols) + 1, len(result.definition)) self.assertEqual(_udf.vectorized_count, 0) def test_enclosed_by_parentheses(self): diff --git a/replication/static/src/main/java/io/deephaven/replicators/ReplicateBarrageUtils.java b/replication/static/src/main/java/io/deephaven/replicators/ReplicateBarrageUtils.java index 6824f8d91f9..35dadff92d0 100644 --- a/replication/static/src/main/java/io/deephaven/replicators/ReplicateBarrageUtils.java +++ b/replication/static/src/main/java/io/deephaven/replicators/ReplicateBarrageUtils.java @@ -36,6 +36,9 @@ public static void main(final String[] args) throws IOException { fixupVectorExpansionKernel(CHUNK_PACKAGE + "/vector/IntVectorExpansionKernel.java", "Int"); fixupVectorExpansionKernel(CHUNK_PACKAGE + "/vector/LongVectorExpansionKernel.java", "Long"); fixupVectorExpansionKernel(CHUNK_PACKAGE + "/vector/DoubleVectorExpansionKernel.java", "Double"); + + ReplicatePrimitiveCode.charToAllButBoolean("replicateBarrageUtils", + "web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebCharColumnData.java"); } private static void fixupVectorExpansionKernel(final @NotNull String path, final @NotNull String type) diff --git a/replication/static/src/main/java/io/deephaven/replicators/ReplicatePageMaterializers.java b/replication/static/src/main/java/io/deephaven/replicators/ReplicatePageMaterializers.java index 13909dd32b3..e4e96e356d7 100644 --- a/replication/static/src/main/java/io/deephaven/replicators/ReplicatePageMaterializers.java +++ b/replication/static/src/main/java/io/deephaven/replicators/ReplicatePageMaterializers.java @@ -5,8 +5,7 @@ import java.io.IOException; -import static io.deephaven.replication.ReplicatePrimitiveCode.charToShortAndByte; -import static io.deephaven.replication.ReplicatePrimitiveCode.floatToAllFloatingPoints; +import static io.deephaven.replication.ReplicatePrimitiveCode.charToByte; import static io.deephaven.replication.ReplicatePrimitiveCode.replaceAll; /** @@ -20,7 +19,14 @@ public class ReplicatePageMaterializers { "extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/"; private static final String CHAR_MATERIALIZER_PATH = MATERIALIZER_DIR + "CharMaterializer.java"; - private static final String FLOAT_MATERIALIZER_PATH = MATERIALIZER_DIR + "FloatMaterializer.java"; + private static final String SHORT_MATERIALIZER_PATH = MATERIALIZER_DIR + "ShortMaterializer.java"; + private static final String INT_MATERIALIZER_PATH = MATERIALIZER_DIR + "IntMaterializer.java"; + private static final String LONG_MATERIALIZER_BASE_PATH = MATERIALIZER_DIR + "LongMaterializerBase.java"; + private static final String LONG_MATERIALIZER_PATH = MATERIALIZER_DIR + "LongMaterializer.java"; + private static final String LONG_FROM_INT_MATERIALIZER_PATH = MATERIALIZER_DIR + "LongFromIntMaterializer.java"; + private static final String LONG_FROM_UNSIGNED_SHORT_MATERIALIZER_PATH = + MATERIALIZER_DIR + "LongFromUnsignedShortMaterializer.java"; + private static final String DOUBLE_MATERIALIZER_PATH = MATERIALIZER_DIR + "DoubleMaterializer.java"; private static final String LOCAL_TIME_FROM_MICROS_MATERIALIZER_PATH = MATERIALIZER_DIR + "LocalTimeFromMicrosMaterializer.java"; private static final String LOCAL_DATE_TIME_FROM_MILLIS_MATERIALIZER_PATH = @@ -34,18 +40,121 @@ public class ReplicatePageMaterializers { private static final String BIG_INTEGER_MATERIALIZER_PATH = MATERIALIZER_DIR + "BigIntegerMaterializer.java"; public static void main(String... args) throws IOException { - charToShortAndByte(TASK, CHAR_MATERIALIZER_PATH, NO_EXCEPTIONS); + charToByte(TASK, CHAR_MATERIALIZER_PATH, NO_EXCEPTIONS); - // Float -> Double - floatToAllFloatingPoints(TASK, FLOAT_MATERIALIZER_PATH, NO_EXCEPTIONS); - - // Float -> Int + // LongBase -> IntBase String[][] pairs = new String[][] { - {"readFloat", "readInteger"}, - {"Float", "Int"}, - {"float", "int"} + {"Long", "Int"}, + {"long", "int"}, + }; + replaceAll(TASK, LONG_MATERIALIZER_BASE_PATH, null, NO_EXCEPTIONS, pairs); + + // Long -> Int + pairs = new String[][] { + {"readLong", "readInteger"}, + {"Long", "Int"}, + {"long", "int"} + }; + replaceAll(TASK, LONG_MATERIALIZER_PATH, null, NO_EXCEPTIONS, pairs); + + // Int -> IntFromBoolean + pairs = new String[][] { + {"IntMaterializer", "IntFromBooleanMaterializer"}, + {"readInteger\\(\\)", "readBoolean() ? 1 : 0"} + }; + replaceAll(TASK, INT_MATERIALIZER_PATH, null, new String[] {"IntMaterializerBase"}, pairs); + + // LongBase -> ShortBase + pairs = new String[][] { + {"Long", "Short"}, + {"long", "short"}, + }; + replaceAll(TASK, LONG_MATERIALIZER_BASE_PATH, null, NO_EXCEPTIONS, pairs); + + // Long -> Short + pairs = new String[][] { + {"dataReader.readLong", "(short) dataReader.readInteger"}, + {"dataReader, 0, numValues", "dataReader, (short) 0, numValues"}, + {"Long", "Short"}, + {"long", "short"} + }; + replaceAll(TASK, LONG_MATERIALIZER_PATH, null, NO_EXCEPTIONS, pairs); + + // Long -> LongFromInt + pairs = new String[][] { + {"LongMaterializer", "LongFromIntMaterializer"}, + {"readLong", "readInteger"}, + }; + replaceAll(TASK, LONG_MATERIALIZER_PATH, LONG_FROM_INT_MATERIALIZER_PATH, null, + new String[] {"LongMaterializerBase"}, pairs); + + // Long -> LongFromBoolean + pairs = new String[][] { + {"LongMaterializer", "LongFromBooleanMaterializer"}, + {"readLong\\(\\)", "readBoolean() ? 1 : 0"}, + }; + replaceAll(TASK, LONG_MATERIALIZER_PATH, null, new String[] {"LongMaterializerBase"}, pairs); + + // LongFromUnsignedShort -> LongFromUnsignedByte + pairs = new String[][] { + {"Short", "Byte"}, + {"short", "byte"} + }; + replaceAll(TASK, LONG_FROM_UNSIGNED_SHORT_MATERIALIZER_PATH, null, NO_EXCEPTIONS, pairs); + + // LongFromUnsignedShort -> LongFromUnsignedInt + pairs = new String[][] { + {"Short.toUnsignedLong", "Integer.toUnsignedLong"}, + {"Short", "Int"}, + {"short", "int"} + }; + replaceAll(TASK, LONG_FROM_UNSIGNED_SHORT_MATERIALIZER_PATH, null, NO_EXCEPTIONS, pairs); + + // LongFromUnsignedShort -> IntFromUnsignedShort + pairs = new String[][] { + {"LongFromUnsignedShort", "IntFromUnsignedShort"}, + {"Long", "Int"}, + {"long", "int"} + }; + replaceAll(TASK, LONG_FROM_UNSIGNED_SHORT_MATERIALIZER_PATH, null, NO_EXCEPTIONS, pairs); + + // LongFromUnsignedShort -> IntFromUnsignedByte + pairs = new String[][] { + {"Short", "Byte"}, + {"short", "byte"}, + {"Long", "Int"}, + {"long", "int"} + }; + replaceAll(TASK, LONG_FROM_UNSIGNED_SHORT_MATERIALIZER_PATH, null, NO_EXCEPTIONS, pairs); + + // LongBase -> DoubleBase + pairs = new String[][] { + {"Long", "Double"}, + {"long", "double"}, + }; + replaceAll(TASK, LONG_MATERIALIZER_BASE_PATH, null, NO_EXCEPTIONS, pairs); + + // Long -> Double + pairs = new String[][] { + {"dataReader.readLong", "dataReader.readDouble"}, + {"Long", "Double"}, + {"long", "double"} + }; + replaceAll(TASK, LONG_MATERIALIZER_PATH, null, NO_EXCEPTIONS, pairs); + + // Double -> DoubleFromFloat + pairs = new String[][] { + {"DoubleMaterializer", "DoubleFromFloatMaterializer"}, + {"Double", "Float"} + }; + replaceAll(TASK, DOUBLE_MATERIALIZER_PATH, null, new String[] {"DoubleMaterializerBase"}, pairs); + + // Short -> ShortFromBoolean + pairs = new String[][] { + {"ShortMaterializer", "ShortFromBooleanMaterializer"}, + {"dataReader.readInteger\\(\\)", "(dataReader.readBoolean() ? 1 : 0)"} }; - replaceAll(TASK, FLOAT_MATERIALIZER_PATH, null, NO_EXCEPTIONS, pairs); + replaceAll(TASK, SHORT_MATERIALIZER_PATH, null, new String[] {"ShortMaterializerBase"}, pairs); // LocalTimeFromMicros -> LocalTimeFromMillis // We change from Micros to Millis and not the other way since converting from Long to Integer has fewer diff --git a/replication/static/src/main/java/io/deephaven/replicators/ReplicateToPage.java b/replication/static/src/main/java/io/deephaven/replicators/ReplicateToPage.java index bd76f91cf1a..42e4cf0f6cd 100644 --- a/replication/static/src/main/java/io/deephaven/replicators/ReplicateToPage.java +++ b/replication/static/src/main/java/io/deephaven/replicators/ReplicateToPage.java @@ -4,8 +4,10 @@ package io.deephaven.replicators; import java.io.IOException; +import java.util.Map; -import static io.deephaven.replication.ReplicatePrimitiveCode.intToAllButBooleanAndLong; +import static io.deephaven.replication.ReplicatePrimitiveCode.charToByte; +import static io.deephaven.replication.ReplicatePrimitiveCode.charToFloat; import static io.deephaven.replication.ReplicatePrimitiveCode.replaceAll; /** @@ -18,12 +20,12 @@ public class ReplicateToPage { private static final String TO_PAGE_DIR = "extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/"; - private static final String TO_INT_PAGE_PATH = TO_PAGE_DIR + "ToIntPage.java"; + private static final String TO_CHAR_PAGE_PATH = TO_PAGE_DIR + "ToCharPage.java"; private static final String TO_LOCAL_DATE_TIME_PAGE_PATH = TO_PAGE_DIR + "ToLocalDateTimePage.java"; private static final String TO_BIG_INTEGER_PAGE_PATH = TO_PAGE_DIR + "ToBigIntegerPage.java"; public static void main(String... args) throws IOException { - intToAllButBooleanAndLong(TASK, TO_INT_PAGE_PATH, "interface"); + charToFloat(TASK, TO_CHAR_PAGE_PATH, null, "interface"); // LocalDateTime -> LocalTime String[][] pairs = new String[][] { diff --git a/server/jetty/src/main/java/io/deephaven/server/jetty/JettyBackedGrpcServer.java b/server/jetty/src/main/java/io/deephaven/server/jetty/JettyBackedGrpcServer.java index 6122b287bcc..641948d9489 100644 --- a/server/jetty/src/main/java/io/deephaven/server/jetty/JettyBackedGrpcServer.java +++ b/server/jetty/src/main/java/io/deephaven/server/jetty/JettyBackedGrpcServer.java @@ -325,6 +325,7 @@ private static ServerConnector createConnector(Server server, JettyConfig config httpConfig.addCustomizer(new SecureRequestCustomizer(config.sniHostCheck())); final HTTP2ServerConnectionFactory h2 = new HTTP2ServerConnectionFactory(httpConfig); h2.setRateControlFactory(new RateControl.Factory() {}); + config.maxConcurrentStreams().ifPresent(h2::setMaxConcurrentStreams); final ALPNServerConnectionFactory alpn = new ALPNServerConnectionFactory(); alpn.setDefaultProtocol(http11 != null ? http11.getProtocol() : h2.getProtocol()); @@ -345,6 +346,8 @@ private static ServerConnector createConnector(Server server, JettyConfig config } else { final HTTP2CServerConnectionFactory h2c = new HTTP2CServerConnectionFactory(httpConfig); h2c.setRateControlFactory(new RateControl.Factory() {}); + config.maxConcurrentStreams().ifPresent(h2c::setMaxConcurrentStreams); + if (http11 != null) { serverConnector = new ServerConnector(server, http11, h2c); } else { @@ -353,6 +356,7 @@ private static ServerConnector createConnector(Server server, JettyConfig config } config.host().ifPresent(serverConnector::setHost); serverConnector.setPort(config.port()); + config.maxHeaderRequestSize().ifPresent(httpConfig::setRequestHeaderSize); // Give connections extra time to shutdown, since we have an explicit server shutdown serverConnector.setShutdownIdleTimeout(serverConnector.getIdleTimeout()); diff --git a/server/jetty/src/main/java/io/deephaven/server/jetty/JettyConfig.java b/server/jetty/src/main/java/io/deephaven/server/jetty/JettyConfig.java index 109cd2371d3..5dc058c8e16 100644 --- a/server/jetty/src/main/java/io/deephaven/server/jetty/JettyConfig.java +++ b/server/jetty/src/main/java/io/deephaven/server/jetty/JettyConfig.java @@ -11,6 +11,8 @@ import org.immutables.value.Value.Style; import javax.annotation.Nullable; +import java.util.Optional; +import java.util.OptionalInt; import java.util.OptionalLong; /** @@ -29,6 +31,8 @@ public abstract class JettyConfig implements ServerConfig { public static final String HTTP_STREAM_TIMEOUT = "http2.stream.idleTimeoutMs"; public static final String HTTP_COMPRESSION = "http.compression"; public static final String SNI_HOST_CHECK = "https.sniHostCheck"; + public static final String MAX_CONCURRENT_STREAMS = "http2.maxConcurrentStreams"; + public static final String MAX_HEADER_REQUEST_SIZE = "http.maxHeaderRequestSize"; /** * Values to indicate what kind of websocket support should be offered. @@ -93,6 +97,8 @@ public static Builder buildFromConfig(Configuration config) { String httpCompression = config.getStringWithDefault(HTTP_COMPRESSION, null); String sniHostCheck = config.getStringWithDefault(SNI_HOST_CHECK, null); String h2StreamIdleTimeout = config.getStringWithDefault(HTTP_STREAM_TIMEOUT, null); + String h2MaxConcurrentStreams = config.getStringWithDefault(MAX_CONCURRENT_STREAMS, null); + String maxHeaderRequestSize = config.getStringWithDefault(MAX_HEADER_REQUEST_SIZE, null); if (httpWebsockets != null) { switch (httpWebsockets.toLowerCase()) { case "true":// backwards compatible @@ -122,6 +128,12 @@ public static Builder buildFromConfig(Configuration config) { if (sniHostCheck != null) { builder.sniHostCheck(Boolean.parseBoolean(sniHostCheck)); } + if (h2MaxConcurrentStreams != null) { + builder.maxConcurrentStreams(Integer.parseInt(h2MaxConcurrentStreams)); + } + if (maxHeaderRequestSize != null) { + builder.maxHeaderRequestSize(Integer.parseInt(maxHeaderRequestSize)); + } return builder; } @@ -212,6 +224,16 @@ public final boolean httpCompressionOrDefault() { return httpCompression == null || httpCompression; } + /** + * Value is in bytes. If unset, uses Jetty's default (presently 8192). + */ + public abstract OptionalInt maxHeaderRequestSize(); + + /** + * If unset, uses Jetty's default (presently 128). Only applies to http2 connections. + */ + public abstract OptionalInt maxConcurrentStreams(); + public interface Builder extends ServerConfig.Builder { Builder websockets(WebsocketsSupport websockets); @@ -223,5 +245,9 @@ public interface Builder extends ServerConfig.Builder { Builder http2StreamIdleTimeout(long timeoutInMillis); Builder sniHostCheck(boolean sniHostCheck); + + Builder maxHeaderRequestSize(int maxHeaderRequestSize); + + Builder maxConcurrentStreams(int maxConcurrentStreams); } } diff --git a/web/client-api/client-api.gradle b/web/client-api/client-api.gradle index f9f9067a65b..93f8930f32f 100644 --- a/web/client-api/client-api.gradle +++ b/web/client-api/client-api.gradle @@ -15,15 +15,32 @@ configurations { dts typescriptDoclet } - dependencies { + implementation platform(libs.grpc.bom) + implementation variantOf(libs.grpc.api) { classifier('sources') } + + implementation project(':engine-chunk') + implementation project(':extensions-barrage') + implementation project(':DataStructures') implementation project(':web-shared-beans') implementation project(':web-client-backplane') + implementation libs.jetbrains.annotations + implementation variantOf(libs.jetbrains.annotations) { classifier("sources") } + implementation libs.immutables.value.annotations + implementation variantOf(libs.immutables.value.annotations) { classifier("sources") } + + implementation libs.flatbuffers.java + implementation variantOf(libs.flatbuffers.java) { classifier("sources") } + implementation libs.arrow.format + implementation variantOf(libs.arrow.format) { classifier('sources') } + implementation libs.deephaven.barrage.format + implementation variantOf(libs.deephaven.barrage.format) { classifier('sources') } implementation libs.vertispan.ts.defs.annotations typescriptDoclet libs.vertispan.ts.defs.doclet implementation libs.vertispan.nio.gwt + implementation libs.vertispan.flatbuffers.gwt js project(path: ':proto:raw-js-openapi', configuration: 'js') @@ -76,10 +93,14 @@ def gwtUnitTest = tasks.register('gwtUnitTest', Test) { t -> '-runStyle HtmlUnit', '-ea', '-style PRETTY', + '-generateJsInteropExports', + '-includeJsInteropExports io.deephaven.*', + '-excludeJsInteropExports io.deephaven.web.client.api.widget.plot.*', "-war ${layout.buildDirectory.dir('unitTest-war').get().asFile.absolutePath}" ].join(' '), 'gwt.persistentunitcachedir': layout.buildDirectory.dir('unitTest-unitCache').get().asFile.absolutePath, ] + t.classpath += tasks.getByName('gwtCompile').src t.include '**/ClientUnitTestSuite.class' t.useJUnit() t.scanForTestClasses = false @@ -148,6 +169,9 @@ def gwtIntegrationTest = tasks.register('gwtIntegrationTest', Test) { t -> "-runStyle io.deephaven.web.junit.RunStyleRemoteWebDriver:${webdriverUrl}?firefox", '-ea', '-style PRETTY', + '-generateJsInteropExports', + '-includeJsInteropExports io.deephaven.*', + '-excludeJsInteropExports io.deephaven.web.client.api.widget.plot.*', "-setProperty dh.server=http://${deephavenDocker.containerName.get()}:10000", "-war ${layout.buildDirectory.dir('integrationTest-war').get().asFile.absolutePath}" ].join(' ')) @@ -181,6 +205,9 @@ Click the URL that is printed out to run the test in your browser, or refresh an '-runStyle Manual:1', '-ea', '-style PRETTY', + '-generateJsInteropExports', + '-includeJsInteropExports io.deephaven.*', + '-excludeJsInteropExports io.deephaven.web.client.api.widget.plot.*', '-setProperty dh.server=http://localhost:10000', '-setProperty compiler.useSourceMaps=true', "-war ${layout.buildDirectory.dir('manualTest-war').get().asFile.absolutePath}" diff --git a/web/client-api/src/main/java/io/deephaven/web/DeephavenApi.gwt.xml b/web/client-api/src/main/java/io/deephaven/web/DeephavenApi.gwt.xml index 4f6177621b8..9649d1d487c 100644 --- a/web/client-api/src/main/java/io/deephaven/web/DeephavenApi.gwt.xml +++ b/web/client-api/src/main/java/io/deephaven/web/DeephavenApi.gwt.xml @@ -2,8 +2,20 @@ + + + + + + + + + + + + diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/Column.java b/web/client-api/src/main/java/io/deephaven/web/client/api/Column.java index 4e4840754dd..00e97223b86 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/Column.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/Column.java @@ -10,6 +10,7 @@ import jsinterop.annotations.JsProperty; import jsinterop.base.Any; +import java.util.Objects; import java.util.stream.IntStream; import java.util.stream.IntStream.Builder; @@ -21,7 +22,6 @@ public class Column { private final int index; - private final Integer formatColumnIndex; private final Integer styleColumnIndex; private final Integer formatStringColumnIndex; @@ -75,7 +75,7 @@ public Column(int jsIndex, int index, Integer formatColumnIndex, Integer styleCo boolean inputTableKeyColumn, boolean isSortable) { this.jsIndex = jsIndex; this.index = index; - this.formatColumnIndex = formatColumnIndex; + assert Objects.equals(formatColumnIndex, styleColumnIndex); this.styleColumnIndex = styleColumnIndex; this.type = type; this.name = name; @@ -170,14 +170,6 @@ public void setConstituentType(final String constituentType) { this.constituentType = constituentType; } - /** - * @deprecated Prefer {@link #getFormatStringColumnIndex()}. - */ - @Deprecated - public Integer getFormatColumnIndex() { - return formatColumnIndex; - } - public Integer getFormatStringColumnIndex() { return formatStringColumnIndex; } @@ -266,7 +258,6 @@ public CustomColumn formatDate(String expression) { public String toString() { return "Column{" + "index=" + index + - ", formatColumnIndex=" + formatColumnIndex + ", styleColumnIndex=" + styleColumnIndex + ", formatStringColumnIndex=" + formatStringColumnIndex + ", type='" + type + '\'' + @@ -285,9 +276,6 @@ public boolean equals(Object o) { if (index != column.index) return false; - if (formatColumnIndex != null ? !formatColumnIndex.equals(column.formatColumnIndex) - : column.formatColumnIndex != null) - return false; if (styleColumnIndex != null ? !styleColumnIndex.equals(column.styleColumnIndex) : column.styleColumnIndex != null) return false; @@ -302,7 +290,6 @@ public boolean equals(Object o) { @Override public int hashCode() { int result = index; - result = 31 * result + (formatColumnIndex != null ? formatColumnIndex.hashCode() : 0); result = 31 * result + (styleColumnIndex != null ? styleColumnIndex.hashCode() : 0); result = 31 * result + (formatStringColumnIndex != null ? formatStringColumnIndex.hashCode() : 0); result = 31 * result + type.hashCode(); @@ -311,12 +298,12 @@ public int hashCode() { } public Column withFormatStringColumnIndex(int formatStringColumnIndex) { - return new Column(jsIndex, index, formatColumnIndex, styleColumnIndex, type, name, isPartitionColumn, + return new Column(jsIndex, index, styleColumnIndex, styleColumnIndex, type, name, isPartitionColumn, formatStringColumnIndex, description, isInputTableKeyColumn, isSortable); } public Column withStyleColumnIndex(int styleColumnIndex) { - return new Column(jsIndex, index, formatColumnIndex, styleColumnIndex, type, name, isPartitionColumn, + return new Column(jsIndex, index, styleColumnIndex, styleColumnIndex, type, name, isPartitionColumn, formatStringColumnIndex, description, isInputTableKeyColumn, isSortable); } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java index 8687dedcd18..6c6c2d2d556 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java @@ -131,19 +131,6 @@ public Promise login(@TsTypeRef(LoginCredentials.class) JsPropertyMap login = loginPromise.asPromise(); - // fetch configs and check session timeout - login.then(ignore -> getServerConfigValues()).then(configs -> { - for (String[] config : configs) { - if (config[0].equals("http.session.durationMs")) { - workerConnection.setSessionTimeoutMs(Double.parseDouble(config[1])); - } - } - return null; - }).catch_(ignore -> { - // Ignore this failure and suppress browser logging, we have a safe fallback - return Promise.resolve((Object) null); - }); - if (alreadyRunning) { ideConnection.connection.get().forceReconnect(); } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/DateWrapper.java b/web/client-api/src/main/java/io/deephaven/web/client/api/DateWrapper.java index 4e6580b57c0..f53d2a35928 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/DateWrapper.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/DateWrapper.java @@ -4,6 +4,7 @@ package io.deephaven.web.client.api; import elemental2.core.JsDate; +import io.deephaven.util.QueryConstants; import io.deephaven.web.client.api.i18n.JsDateTimeFormat; import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsType; @@ -17,6 +18,9 @@ public DateWrapper(long valueInNanos) { @JsIgnore public static DateWrapper of(long dateInNanos) { + if (dateInNanos == QueryConstants.NULL_LONG) { + return null; + } return new DateWrapper(dateInNanos); } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/Format.java b/web/client-api/src/main/java/io/deephaven/web/client/api/Format.java index 747a3402b39..2aea9065a9c 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/Format.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/Format.java @@ -10,12 +10,16 @@ import jsinterop.annotations.JsNullable; import jsinterop.annotations.JsProperty; +import java.util.Objects; + /** * This object may be pooled internally or discarded and not updated. Do not retain references to it. */ @TsInterface @TsName(namespace = "dh") -public class Format { +public final class Format { + public static final Format EMPTY = new Format(0, 0, null, null); + private final long cellColors; private final long rowColors; @@ -23,8 +27,8 @@ public class Format { private final String formatString; public Format(long cellColors, long rowColors, String numberFormat, String formatString) { - this.cellColors = cellColors; - this.rowColors = rowColors; + this.cellColors = cellColors == Long.MIN_VALUE ? 0 : cellColors; + this.rowColors = rowColors == Long.MIN_VALUE ? 0 : rowColors; this.numberFormat = numberFormat; this.formatString = formatString; } @@ -103,4 +107,30 @@ public String getFormatString() { return formatString; } + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + Format format = (Format) o; + return cellColors == format.cellColors && rowColors == format.rowColors + && Objects.equals(numberFormat, format.numberFormat) + && Objects.equals(formatString, format.formatString); + } + + @Override + public int hashCode() { + return Objects.hash(cellColors, rowColors, numberFormat, formatString); + } + + @Override + public String toString() { + return "Format{" + + "cellColors=" + cellColors + + ", rowColors=" + rowColors + + ", numberFormat='" + numberFormat + '\'' + + ", formatString='" + formatString + '\'' + + '}'; + } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/JsColumnStatistics.java b/web/client-api/src/main/java/io/deephaven/web/client/api/JsColumnStatistics.java index 7b117047e80..d66034f67d4 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/JsColumnStatistics.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/JsColumnStatistics.java @@ -8,7 +8,6 @@ import com.vertispan.tsdefs.annotations.TsName; import elemental2.core.JsArray; import elemental2.core.JsMap; -import io.deephaven.web.shared.data.ColumnStatistics; import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsMethod; import jsinterop.annotations.JsProperty; @@ -18,8 +17,7 @@ import java.util.Map; /** - * Javascript wrapper for {@link ColumnStatistics} This class holds the results of a call to generate statistics on a - * table column. + * Represents statistics for a given table column. */ @TsInterface @TsName(name = "ColumnStatistics", namespace = "dh") diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/JsPartitionedTable.java b/web/client-api/src/main/java/io/deephaven/web/client/api/JsPartitionedTable.java index acc260335e0..aa7aca03f00 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/JsPartitionedTable.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/JsPartitionedTable.java @@ -26,7 +26,6 @@ import io.deephaven.web.client.state.ClientTableState; import io.deephaven.web.shared.data.RangeSet; import jsinterop.annotations.JsIgnore; -import jsinterop.annotations.JsMethod; import jsinterop.annotations.JsNullable; import jsinterop.annotations.JsProperty; import jsinterop.annotations.JsType; @@ -143,11 +142,10 @@ private Promise subscribeToBaseTable() { private void handleKeys(Event update) { // noinspection unchecked - CustomEvent event = - (CustomEvent) update; + CustomEvent event = (CustomEvent) update; // We're only interested in added rows, send an event indicating the new keys that are available - SubscriptionTableData.UpdateEventData eventData = event.detail; + SubscriptionTableData eventData = event.detail; RangeSet added = eventData.getAdded().getRange(); added.indexIterator().forEachRemaining((long index) -> { // extract the key to use diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/JsRangeSet.java b/web/client-api/src/main/java/io/deephaven/web/client/api/JsRangeSet.java index 50e09604c73..85872589483 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/JsRangeSet.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/JsRangeSet.java @@ -24,6 +24,9 @@ public class JsRangeSet { private final RangeSet range; public static JsRangeSet ofRange(double first, double last) { + if (first > last) { + throw new IllegalStateException(first + " > " + last); + } return new JsRangeSet(RangeSet.ofRange((long) first, (long) last)); } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/JsTable.java b/web/client-api/src/main/java/io/deephaven/web/client/api/JsTable.java index 4e380631ad7..21fd3ca9eff 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/JsTable.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/JsTable.java @@ -9,7 +9,6 @@ import com.vertispan.tsdefs.annotations.TsUnionMember; import elemental2.core.JsArray; import elemental2.dom.CustomEventInit; -import elemental2.dom.DomGlobal; import elemental2.promise.IThenable.ThenOnFulfilledCallbackFn; import elemental2.promise.Promise; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.hierarchicaltable_pb.RollupRequest; @@ -38,27 +37,22 @@ import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.table_pb.runchartdownsamplerequest.ZoomRange; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.ticket_pb.Ticket; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.ticket_pb.TypedTicket; -import io.deephaven.web.client.api.barrage.def.ColumnDefinition; import io.deephaven.web.client.api.barrage.def.TableAttributesDefinition; import io.deephaven.web.client.api.barrage.stream.ResponseStreamWrapper; import io.deephaven.web.client.api.batch.RequestBatcher; -import io.deephaven.web.client.api.batch.TableConfig; import io.deephaven.web.client.api.console.JsVariableType; import io.deephaven.web.client.api.filter.FilterCondition; -import io.deephaven.web.client.api.filter.FilterValue; import io.deephaven.web.client.api.input.JsInputTable; import io.deephaven.web.client.api.lifecycle.HasLifecycle; import io.deephaven.web.client.api.state.StateCache; +import io.deephaven.web.client.api.subscription.AbstractTableSubscription; import io.deephaven.web.client.api.subscription.TableSubscription; import io.deephaven.web.client.api.subscription.TableViewportSubscription; import io.deephaven.web.client.api.subscription.ViewportData; -import io.deephaven.web.client.api.subscription.ViewportData.MergeResults; -import io.deephaven.web.client.api.subscription.ViewportRow; import io.deephaven.web.client.api.tree.JsRollupConfig; import io.deephaven.web.client.api.tree.JsTreeTable; import io.deephaven.web.client.api.tree.JsTreeTableConfig; import io.deephaven.web.client.api.widget.JsWidget; -import io.deephaven.web.client.fu.JsData; import io.deephaven.web.client.fu.JsItr; import io.deephaven.web.client.fu.JsLog; import io.deephaven.web.client.fu.LazyPromise; @@ -66,14 +60,11 @@ import io.deephaven.web.client.state.ClientTableState; import io.deephaven.web.client.state.HasTableBinding; import io.deephaven.web.shared.data.*; -import io.deephaven.web.shared.data.TableSnapshot.SnapshotType; -import io.deephaven.web.shared.data.columns.ColumnData; import io.deephaven.web.shared.fu.JsConsumer; import io.deephaven.web.shared.fu.JsProvider; import io.deephaven.web.shared.fu.JsRunnable; import io.deephaven.web.shared.fu.RemoverFn; import javaemul.internal.annotations.DoNotAutobox; -import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsMethod; import jsinterop.annotations.JsNullable; import jsinterop.annotations.JsOptional; @@ -88,7 +79,6 @@ import java.util.*; import java.util.stream.Stream; -import static io.deephaven.web.client.api.subscription.ViewportData.NO_ROW_FORMAT_COLUMN; import static io.deephaven.web.client.fu.LazyPromise.logError; /** @@ -155,15 +145,11 @@ public class JsTable extends HasLifecycle implements HasTableBinding, JoinableTa // change in some table data INTERNAL_EVENT_SIZELISTENER = "sizelistener-internal"; - // Amount of debounce to use when eating snapshot events. - public static final int DEBOUNCE_TIME = 20; public static final int MAX_BATCH_TIME = 600_000; private final WorkerConnection workerConnection; - private Map subscriptions = new HashMap<>(); - @Deprecated // TODO refactor this inside of the viewportSubscription type - private ViewportData currentViewportData; + private final Map subscriptions = new HashMap<>(); private ClientTableState lastVisibleState; @@ -181,7 +167,6 @@ public class JsTable extends HasLifecycle implements HasTableBinding, JoinableTa private final int subscriptionId; private static int nextSubscriptionId; - private TableSubscription nonViewportSub; /** * Creates a new Table directly from an existing ClientTableState. The CTS manages all fetch operations, so this is @@ -312,7 +297,9 @@ public boolean isAlive() { @Override public ClientTableState state() { - assert currentState != null : "Table already closed, cannot be used again"; + if (currentState == null) { + throw new IllegalStateException("Table already closed, cannot be used again"); + } return currentState; } @@ -467,6 +454,7 @@ public double getSize() { if (isUncoalesced()) { return JsTable.SIZE_UNCOALESCED; } + // Only return the size from ETUM if we have no other choice return size; } @@ -486,10 +474,9 @@ public String getDescription() { */ @JsProperty public double getTotalSize() { - TableViewportSubscription subscription = subscriptions.get(getHandle()); - if (subscription != null && subscription.getStatus() == TableViewportSubscription.Status.ACTIVE) { - // only ask the viewport for the size if it is alive and ticking - return subscription.totalSize(); + if (state().getFilters().isEmpty()) { + // If there are no filters, use the subscription size (if any) + return getSize(); } return getHeadState().getSize(); } @@ -683,14 +670,14 @@ public JsArray getCustomColumns() { * Overload for Java (since JS just omits the optional params) */ public TableViewportSubscription setViewport(double firstRow, double lastRow) { - return setViewport(firstRow, lastRow, null, null); + return setViewport(firstRow, lastRow, null, null, null); } /** * Overload for Java (since JS just omits the optional param) */ public TableViewportSubscription setViewport(double firstRow, double lastRow, JsArray columns) { - return setViewport(firstRow, lastRow, columns, null); + return setViewport(firstRow, lastRow, columns, null, null); } /** @@ -709,13 +696,14 @@ public TableViewportSubscription setViewport(double firstRow, double lastRow, Js @JsMethod public TableViewportSubscription setViewport(double firstRow, double lastRow, @JsOptional @JsNullable JsArray columns, - @JsOptional @JsNullable Double updateIntervalMs) { - Column[] columnsCopy = columns != null ? Js.uncheckedCast(columns.slice()) : null; + @JsOptional @JsNullable Double updateIntervalMs, + @JsOptional @JsNullable Boolean isReverseViewport) { + Column[] columnsCopy = columns != null ? Js.uncheckedCast(columns.slice()) : state().getColumns(); ClientTableState currentState = state(); TableViewportSubscription activeSubscription = subscriptions.get(getHandle()); if (activeSubscription != null && activeSubscription.getStatus() != TableViewportSubscription.Status.DONE) { // hasn't finished, lets reuse it - activeSubscription.setInternalViewport(firstRow, lastRow, columnsCopy, updateIntervalMs); + activeSubscription.setInternalViewport(firstRow, lastRow, columnsCopy, updateIntervalMs, isReverseViewport); return activeSubscription; } else { // In the past, we left the old sub going until the new one was ready, then started the new one. But now, @@ -727,35 +715,23 @@ public TableViewportSubscription setViewport(double firstRow, double lastRow, // rewrap current state in a new one, when ready the viewport will be applied TableViewportSubscription replacement = - new TableViewportSubscription(firstRow, lastRow, columnsCopy, updateIntervalMs, this); + TableViewportSubscription.make(firstRow, lastRow, columnsCopy, updateIntervalMs, this); subscriptions.put(currentState.getHandle(), replacement); return replacement; } } - public void setInternalViewport(double firstRow, double lastRow, Column[] columns) { - if (firstRow > lastRow) { - throw new IllegalArgumentException(firstRow + " > " + lastRow); - } - if (firstRow < 0) { - throw new IllegalArgumentException(firstRow + " < " + 0); - } - currentViewportData = null; - // we must wait for the latest stack entry that can add columns (so we get an appropriate BitSet) - state().setDesiredViewport(this, (long) firstRow, (long) lastRow, columns); - } - /** * Gets the currently visible viewport. If the current set of operations has not yet resulted in data, it will not * resolve until that data is ready. If this table is closed before the promise resolves, it will be rejected - to * separate the lifespan of this promise from the table itself, call * {@link TableViewportSubscription#getViewportData()} on the result from {@link #setViewport(double, double)}. - * + * * @return Promise of {@link TableData} */ @JsMethod - public Promise getViewportData() { + public Promise getViewportData() { TableViewportSubscription subscription = subscriptions.get(getHandle()); if (subscription == null) { return Promise.reject("No viewport currently set"); @@ -763,20 +739,6 @@ public Promise getViewportData() { return subscription.getInternalViewportData(); } - public Promise getInternalViewportData() { - final LazyPromise promise = new LazyPromise<>(); - final ClientTableState active = state(); - active.onRunning(state -> { - if (currentViewportData == null) { - // no viewport data received yet; let's set up a one-shot UPDATED event listener - addEventListenerOneShot(EVENT_UPDATED, ignored -> promise.succeed(currentViewportData)); - } else { - promise.succeed(currentViewportData); - } - }, promise::fail, () -> promise.fail("Table closed before viewport data was read")); - return promise.asPromise(MAX_BATCH_TIME); - } - /** * Overload for java (since js just omits the optional var) */ @@ -798,20 +760,9 @@ public TableSubscription subscribe(JsArray columns) { */ @JsMethod public TableSubscription subscribe(JsArray columns, @JsOptional Double updateIntervalMs) { - assert nonViewportSub == null : "Can't directly subscribe to the 'private' table instance"; - // make a new table with a pUT call, listen to the subscription there return new TableSubscription(columns, this, updateIntervalMs); } - public void internalSubscribe(JsArray columns, TableSubscription sub) { - if (columns == null) { - columns = getColumns(); - } - this.nonViewportSub = sub; - - state().subscribe(this, Js.uncheckedCast(columns)); - } - /** * a new table containing the distinct tuples of values from the given columns that are present in the original * table. This table can be manipulated as any other table. Sorting is often desired as the default sort is the @@ -1512,7 +1463,6 @@ public void revive(ClientTableState state) { unsuppressEvents(); LazyPromise.runLater(() -> { fireEvent(EVENT_RECONNECT); - getBinding().maybeReviveSubscription(); }); } } @@ -1550,205 +1500,6 @@ public Promise downsample(LongWrapper[] zoomRange, int pixelCount, Stri .then(state -> Promise.resolve(new JsTable(workerConnection, state))); } - private final class Debounce { - private final ClientTableState state; - private final TableTicket handle; - private final SnapshotType type; - private final RangeSet includedRows; - private final BitSet columns; - private final Object[] dataColumns; - private final double timestamp; - private final long maxRows; - - public Debounce( - TableTicket table, - SnapshotType snapshotType, - RangeSet includedRows, - BitSet columns, - Object[] dataColumns, - long maxRows) { - this.handle = table; - this.type = snapshotType; - this.includedRows = includedRows; - this.columns = columns; - this.dataColumns = dataColumns; - this.state = currentState; - this.maxRows = maxRows; - timestamp = System.currentTimeMillis(); - } - - public boolean isEqual(Debounce o) { - if (type == o.type) { - // this is intentionally weird. We only want to debounce when one instance is column snapshot and the - // other is row snapshot, - // so we consider two events of the same type to be incompatible with debouncing. - return false; - } - if (handle != o.handle) { - assert !handle.equals(o.handle); - return false; - } - if (state != o.state) { - assert state.getHandle() != o.state.getHandle(); - return false; - } - if (!includedRows.equals(o.includedRows)) { - return false; - } - if (!columns.equals(o.columns)) { - return false; - } - if (maxRows != o.maxRows) { - return false; - } - assert Arrays.deepEquals(dataColumns, o.dataColumns) : "Debounce is broken, remove it."; - return true; - } - } - - private Debounce debounce; - - private void handleSnapshot(TableTicket table, SnapshotType snapshotType, RangeSet includedRows, - Object[] dataColumns, BitSet columns, long maxRows) { - assert table.equals(state().getHandle()) : "Table received incorrect snapshot"; - // if the type is initial_snapshot, we've already recorded the size, so only watch for the other two updates. - // note that this will sometimes result in multiple updates on startup, so we do this ugly debounce-dance. - // When IDS-2113 is fixed, we can likely remove this code. - JsLog.debug("Received snapshot for ", table, snapshotType, includedRows, dataColumns, columns); - Debounce operation = new Debounce(table, snapshotType, includedRows, columns, dataColumns, maxRows); - if (debounce == null) { - debounce = operation; - DomGlobal.setTimeout(ignored -> processSnapshot(), DEBOUNCE_TIME); - } else if (debounce.isEqual(operation)) { - // If we think the problem is fixed, we can put `assert false` here for a while before deleting Debounce - // class - JsLog.debug("Eating duplicated operation", debounce, operation); - } else { - processSnapshot(); - debounce = operation; - DomGlobal.setTimeout(ignored -> processSnapshot(), DEBOUNCE_TIME); - } - } - - public void handleSnapshot(TableTicket handle, TableSnapshot snapshot) { - if (!handle.equals(state().getHandle())) { - return; - } - Viewport viewport = getBinding().getSubscription(); - if (viewport == null || viewport.getRows() == null || viewport.getRows().size() == 0) { - // check out if we have a non-viewport sub attached - if (nonViewportSub != null) { - nonViewportSub.handleSnapshot(snapshot); - } - return; - } - - RangeSet viewportRows = viewport.getRows(); - JsLog.debug("handleSnapshot on " + viewportRows, handle, snapshot, viewport); - - RangeSet includedRows = snapshot.getIncludedRows(); - ColumnData[] dataColumns = snapshot.getDataColumns(); - JsArray[] remappedData = new JsArray[dataColumns.length]; - // remap dataColumns to the expected range for that table's viewport - long lastRow = -1; - for (int col = viewport.getColumns().nextSetBit(0); col >= 0; col = viewport.getColumns().nextSetBit(col + 1)) { - ColumnData dataColumn = dataColumns[col]; - if (dataColumn == null) { - // skip this, at least one column requested by that table isn't present, waiting on a later update - // TODO when IDS-2138 is fixed stop throwing this data away - return; - } - Object columnData = dataColumn.getData(); - - final ColumnDefinition def = state().getTableDef().getColumns()[col]; - remappedData[col] = JsData.newArray(def.getType()); - - PrimitiveIterator.OfLong viewportIterator = viewportRows.indexIterator(); - PrimitiveIterator.OfLong includedRowsIterator = includedRows.indexIterator(); - int dataIndex = 0; - while (viewportIterator.hasNext()) { - long viewportIndex = viewportIterator.nextLong(); - if (viewportIndex >= snapshot.getTableSize()) { - // reached or passed the end of the table, we'll still make a snapshot - break; - } - if (!includedRowsIterator.hasNext()) { - // we've reached the end, the viewport apparently goes past the end of what the server sent, - // so there is another snapshot on its way - // TODO when IDS-2138 is fixed stop throwing this data away - return; - } - - long possibleMatch = includedRowsIterator.nextLong(); - while (includedRowsIterator.hasNext() && possibleMatch < viewportIndex) { - dataIndex++;// skip, still seeking to the next item - - possibleMatch = includedRowsIterator.nextLong(); - } - if (!includedRowsIterator.hasNext() && possibleMatch < viewportIndex) { - // we didn't find any items which match, just give up - return; - } - - if (possibleMatch > viewportIndex) { - // if we hit a gap (more data coming, doesn't match viewport), skip the - // rest of this table entirely, a later update will get us caught up - return; - } - Object data = Js.>uncheckedCast(columnData).getAt(dataIndex); - remappedData[col].push(data); - dataIndex++;// increment for the next row - - // Track how many rows were actually present, allowing the snapshot to stop before the viewport's end - lastRow = Math.max(lastRow, possibleMatch); - } - } - - // TODO correct this - assumes max one range per table viewport, and nothing skipped - RangeSet actualViewport = - lastRow == -1 ? RangeSet.empty() : RangeSet.ofRange(viewportRows.indexIterator().nextLong(), lastRow); - - handleSnapshot(handle, snapshot.getSnapshotType(), actualViewport, remappedData, viewport.getColumns(), - viewportRows.size()); - } - - @JsIgnore - public void processSnapshot() { - try { - if (debounce == null) { - JsLog.debug("Skipping snapshot b/c debounce is null"); - return; - } - if (debounce.state != currentState) { - JsLog.debug("Skipping snapshot because state has changed ", debounce.state, " != ", currentState); - return; - } - if (isClosed()) { - JsLog.debug("Skipping snapshot because table is closed", this); - return; - } - JsArray viewportColumns = - getColumns().filter((item, index) -> debounce.columns.get(item.getIndex())); - ViewportData data = new ViewportData(debounce.includedRows, debounce.dataColumns, viewportColumns, - currentState.getRowFormatColumn() == null ? NO_ROW_FORMAT_COLUMN - : currentState.getRowFormatColumn().getIndex(), - debounce.maxRows); - this.currentViewportData = data; - CustomEventInit updatedEvent = CustomEventInit.create(); - updatedEvent.setDetail(data); - fireEvent(EVENT_UPDATED, updatedEvent); - - // also fire rowadded events - TODO also fire some kind of remove event for now-missing rows? - for (int i = 0; i < data.getRows().length; i++) { - CustomEventInit addedEvent = CustomEventInit.create(); - addedEvent.setDetail(wrap(data.getRows().getAt(i), i)); - fireEvent(EVENT_ROWADDED, addedEvent); - } - } finally { - debounce = null; - } - } - /** * True if this table has been closed. * @@ -1788,59 +1539,6 @@ public String getPluginName() { return lastVisibleState().getTableDef().getAttributes().getPluginName(); } - // Factored out so that we always apply the same format - private Object wrap(ViewportRow at, int index) { - return JsPropertyMap.of("row", at, "index", (double) index); - } - - public void handleDelta(ClientTableState current, DeltaUpdates updates) { - current.onRunning(s -> { - if (current != state()) { - return; - } - if (nonViewportSub != null) { - nonViewportSub.handleDelta(updates); - return; - } - final ViewportData vpd = currentViewportData; - if (vpd == null) { - // if the current viewport data is null, we're waiting on an initial snapshot to arrive for a different - // part of the viewport - JsLog.debug("Received delta while waiting for reinitialization"); - return; - } - MergeResults mergeResults = vpd.merge(updates); - if (mergeResults.added.size() == 0 && mergeResults.modified.size() == 0 - && mergeResults.removed.size() == 0) { - return; - } - CustomEventInit event = CustomEventInit.create(); - event.setDetail(vpd); - // user might call setViewport, and wind up nulling our currentViewportData - fireEvent(EVENT_UPDATED, event); - - // fire rowadded/rowupdated/rowremoved - // TODO when we keep more rows loaded than the user is aware of, check if a given row is actually in the - // viewport - // here - for (Integer index : mergeResults.added) { - CustomEventInit addedEvent = CustomEventInit.create(); - addedEvent.setDetail(wrap(vpd.getRows().getAt(index), index)); - fireEvent(EVENT_ROWADDED, addedEvent); - } - for (Integer index : mergeResults.modified) { - CustomEventInit addedEvent = CustomEventInit.create(); - addedEvent.setDetail(wrap(vpd.getRows().getAt(index), index)); - fireEvent(EVENT_ROWUPDATED, addedEvent); - } - for (Integer index : mergeResults.removed) { - CustomEventInit addedEvent = CustomEventInit.create(); - addedEvent.setDetail(wrap(vpd.getRows().getAt(index), index)); - fireEvent(EVENT_ROWREMOVED, addedEvent); - } - }, JsRunnable.doNothing()); - } - @Override public TableTicket getHandle() { return state().getHandle(); @@ -1875,62 +1573,6 @@ public WorkerConnection getConnection() { return workerConnection; } - public void refreshViewport(ClientTableState state, Viewport vp) { - assert state() == state : "Called refreshViewport with wrong state (" + state + " instead of " + state() + ")"; - assert state.getResolution() == ClientTableState.ResolutionState.RUNNING - : "Do not call refreshViewport for a state that is not running! (" + state + ")"; - - currentViewportData = null; // ignore any deltas for past viewports - workerConnection.scheduleCheck(state); - // now that we've made sure the server knows, if we already know that the viewport is beyond what exists, we - // can go ahead and fire an update event. We're in the onResolved call, so we know the handle has resolved - // and if size is not -1, then we've already at least gotten the initial snapshot (otherwise, that snapshot - // will be here soon, and will fire its own event) - if (state.getSize() != ClientTableState.SIZE_UNINITIALIZED && state.getSize() <= vp.getRows().getFirstRow()) { - JsLog.debug("Preparing to send a 'fake' update event since " + state.getSize() + "<=" - + vp.getRows().getFirstRow(), state); - LazyPromise.runLater(() -> { - if (state != state()) { - return; - } - - // get the column expected to be in the snapshot - JsArray columns = Js.uncheckedCast(getBinding().getColumns()); - Column[] allColumns = state.getColumns(); - if (columns == null) { - columns = Js.uncheckedCast(allColumns); - } - // build an array of empty column data for this snapshot - Object[] dataColumns = new Object[allColumns.length]; - - for (int i = 0; i < columns.length; i++) { - Column c = columns.getAt(i); - dataColumns[c.getIndex()] = JsData.newArray(c.getType()); - if (c.getFormatStringColumnIndex() != null) { - dataColumns[c.getFormatStringColumnIndex()] = JsData.newArray("java.lang.String"); - } - if (c.getStyleColumnIndex() != null) { - dataColumns[c.getStyleColumnIndex()] = JsData.newArray("long"); - } - } - if (currentState.getRowFormatColumn() != null) { - dataColumns[currentState.getRowFormatColumn().getIndex()] = JsData.newArray("long"); - } - - ViewportData data = new ViewportData(RangeSet.empty(), dataColumns, columns, - currentState.getRowFormatColumn() == null ? NO_ROW_FORMAT_COLUMN - : currentState.getRowFormatColumn().getIndex(), - 0); - this.currentViewportData = data; - CustomEventInit updatedEvent = CustomEventInit.create(); - updatedEvent.setDetail(data); - JsLog.debug("Sending 'fake' update event since " + state.getSize() + "<=" + vp.getRows().getFirstRow(), - vp, state); - fireEvent(EVENT_UPDATED, updatedEvent); - }); - } - } - public boolean isActive(ClientTableState state) { return currentState == state; } @@ -2110,7 +1752,10 @@ public int getSubscriptionId() { @Override public void maybeReviveSubscription() { - getBinding().maybeReviveSubscription(); + TableViewportSubscription viewportSubscription = subscriptions.get(getHandle()); + if (viewportSubscription != null) { + viewportSubscription.revive(); + } } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/JsTotalsTable.java b/web/client-api/src/main/java/io/deephaven/web/client/api/JsTotalsTable.java index 652bfd4f55f..0bab10ea1d5 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/JsTotalsTable.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/JsTotalsTable.java @@ -5,14 +5,16 @@ import com.vertispan.tsdefs.annotations.TsInterface; import com.vertispan.tsdefs.annotations.TsName; +import com.vertispan.tsdefs.annotations.TsTypeRef; import elemental2.core.JsArray; import elemental2.core.JsString; import elemental2.dom.CustomEvent; -import elemental2.dom.Event; import elemental2.promise.Promise; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.ticket_pb.TypedTicket; import io.deephaven.web.client.api.console.JsVariableType; import io.deephaven.web.client.api.filter.FilterCondition; +import io.deephaven.web.client.api.subscription.AbstractTableSubscription; +import io.deephaven.web.client.api.subscription.ViewportData; import io.deephaven.web.client.state.ClientTableState; import io.deephaven.web.shared.fu.RemoverFn; import jsinterop.annotations.JsIgnore; @@ -66,7 +68,7 @@ public JsTotalsTable(JsTable wrappedTable, String directive, JsArray gro public void refreshViewport() { if (firstRow != null && lastRow != null) { - setViewport(firstRow, lastRow, Js.uncheckedCast(columns), updateIntervalMs); + setViewport(firstRow, lastRow, Js.uncheckedCast(columns), updateIntervalMs, null); } } @@ -109,12 +111,12 @@ public JsTotalsTableConfig getTotalsTableConfig() { */ @JsMethod public void setViewport(double firstRow, double lastRow, @JsOptional JsArray columns, - @JsOptional Double updateIntervalMs) { + @JsOptional Double updateIntervalMs, @JsOptional @JsNullable Boolean isReverseViewport) { this.firstRow = firstRow; this.lastRow = lastRow; this.columns = columns != null ? Js.uncheckedCast(columns.slice()) : null; this.updateIntervalMs = updateIntervalMs; - wrappedTable.setViewport(firstRow, lastRow, columns, updateIntervalMs); + wrappedTable.setViewport(firstRow, lastRow, columns, updateIntervalMs, isReverseViewport); } /** @@ -124,7 +126,7 @@ public void setViewport(double firstRow, double lastRow, @JsOptional JsArray getViewportData() { + public Promise getViewportData() { return wrappedTable.getViewportData(); } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/LocalDateWrapper.java b/web/client-api/src/main/java/io/deephaven/web/client/api/LocalDateWrapper.java index a750b0f0310..5e3bb30eca8 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/LocalDateWrapper.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/LocalDateWrapper.java @@ -6,29 +6,20 @@ import com.google.gwt.i18n.client.NumberFormat; import com.vertispan.tsdefs.annotations.TsInterface; import com.vertispan.tsdefs.annotations.TsName; -import io.deephaven.web.shared.data.LocalDate; import jsinterop.annotations.JsMethod; -import javax.annotation.Nonnull; - /** * Wrap LocalDate values for use in JS. Provides text formatting for display and access to the underlying value. */ @TsInterface @TsName(namespace = "dh") public class LocalDateWrapper { - private final static NumberFormat YEAR_FORMAT = NumberFormat.getFormat("0000"); - private final static NumberFormat MONTH_DAY_FORMAT = NumberFormat.getFormat("00"); + private static final NumberFormat YEAR_FORMAT = NumberFormat.getFormat("0000"); + private static final NumberFormat MONTH_DAY_FORMAT = NumberFormat.getFormat("00"); private final int year; private final int monthValue, dayOfMonth; - public LocalDateWrapper(@Nonnull LocalDate localDate) { - year = localDate.getYear(); - monthValue = localDate.getMonthValue(); - dayOfMonth = localDate.getDayOfMonth(); - } - public LocalDateWrapper(int year, int monthValue, int dayOfMonth) { this.year = year; this.monthValue = monthValue; @@ -55,15 +46,6 @@ public int getDayOfMonth() { return dayOfMonth; } - @Deprecated - public LocalDate getWrapped() { - LocalDate localDate = new LocalDate(); - localDate.setYear(year); - localDate.setMonthValue((byte) monthValue); - localDate.setDayOfMonth((byte) dayOfMonth); - return localDate; - } - @JsMethod @Override public String toString() { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/LocalTimeWrapper.java b/web/client-api/src/main/java/io/deephaven/web/client/api/LocalTimeWrapper.java index 10d64798f0d..078884e88fb 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/LocalTimeWrapper.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/LocalTimeWrapper.java @@ -6,10 +6,11 @@ import com.google.gwt.i18n.client.NumberFormat; import com.vertispan.tsdefs.annotations.TsInterface; import com.vertispan.tsdefs.annotations.TsName; -import io.deephaven.web.shared.data.LocalTime; +import io.deephaven.util.QueryConstants; import jsinterop.annotations.JsMethod; -import javax.annotation.Nonnull; +import java.util.function.IntFunction; +import java.util.function.LongFunction; /** * Wrap LocalTime values for use in JS. Provides text formatting for display and access to the underlying value. @@ -17,13 +18,51 @@ @TsInterface @TsName(namespace = "dh") public class LocalTimeWrapper { - private final static NumberFormat TWO_DIGIT_FORMAT = NumberFormat.getFormat("00"); - private final static NumberFormat NANOS_FORMAT = NumberFormat.getFormat("000000000"); + private static final NumberFormat TWO_DIGIT_FORMAT = NumberFormat.getFormat("00"); + private static final NumberFormat NANOS_FORMAT = NumberFormat.getFormat("000000000"); - private final LocalTime localTime; + private final int hour; + private final int minute; + private final int second; + private final int nano; - public LocalTimeWrapper(@Nonnull LocalTime localTime) { - this.localTime = localTime; + public static IntFunction intCreator(int unitPerMicro) { + int nanoPerUnit = 1_000_000_000 / unitPerMicro; + return val -> { + if (val == QueryConstants.NULL_INT) { + return null; + } + int nano = (val % unitPerMicro) * nanoPerUnit; + int secVal = val / unitPerMicro; + int second = (secVal % 60); + secVal /= 60; + int minute = (secVal % 60); + int hour = (secVal / 60); + return new LocalTimeWrapper(hour, minute, second, nano); + }; + } + + public static LongFunction longCreator(int unitPerMicro) { + int nanoPerUnit = 1_000_000_000 / unitPerMicro; + return val -> { + if (val == QueryConstants.NULL_LONG) { + return null; + } + int nano = (int) (val % unitPerMicro) * nanoPerUnit; + int secVal = (int) (val / unitPerMicro); + byte second = (byte) (secVal % 60); + secVal /= 60; + byte minute = (byte) (secVal % 60); + byte hour = (byte) (secVal / 60); + return new LocalTimeWrapper(hour, minute, second, nano); + }; + } + + public LocalTimeWrapper(int hour, int minute, int second, int nano) { + this.hour = hour; + this.minute = minute; + this.second = second; + this.nano = nano; } @JsMethod @@ -33,34 +72,30 @@ public String valueOf() { @JsMethod public int getHour() { - return localTime.getHour(); + return hour; } @JsMethod public int getMinute() { - return localTime.getMinute(); + return minute; } @JsMethod public int getSecond() { - return localTime.getSecond(); + return second; } @JsMethod public int getNano() { - return localTime.getNano(); - } - - public LocalTime getWrapped() { - return localTime; + return nano; } @JsMethod @Override public String toString() { - return TWO_DIGIT_FORMAT.format(localTime.getHour()) - + ":" + TWO_DIGIT_FORMAT.format(localTime.getMinute()) - + ":" + TWO_DIGIT_FORMAT.format(localTime.getSecond()) - + "." + NANOS_FORMAT.format(localTime.getNano()); + return TWO_DIGIT_FORMAT.format(hour) + + ":" + TWO_DIGIT_FORMAT.format(minute) + + ":" + TWO_DIGIT_FORMAT.format(second) + + "." + NANOS_FORMAT.format(nano); } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/LongWrapper.java b/web/client-api/src/main/java/io/deephaven/web/client/api/LongWrapper.java index ec679c0442a..ea2647184b0 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/LongWrapper.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/LongWrapper.java @@ -3,6 +3,7 @@ // package io.deephaven.web.client.api; +import io.deephaven.util.QueryConstants; import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsType; @@ -12,6 +13,9 @@ public class LongWrapper { @JsIgnore public static LongWrapper of(long value) { + if (value == QueryConstants.NULL_LONG) { + return null; + } return new LongWrapper(value); } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/TableData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/TableData.java index 5018db59ab7..19df98c4221 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/TableData.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/TableData.java @@ -3,11 +3,11 @@ // package io.deephaven.web.client.api; -import com.vertispan.tsdefs.annotations.TsName; import com.vertispan.tsdefs.annotations.TsTypeRef; import com.vertispan.tsdefs.annotations.TsUnion; import com.vertispan.tsdefs.annotations.TsUnionMember; import elemental2.core.JsArray; +import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsMethod; import jsinterop.annotations.JsOverlay; import jsinterop.annotations.JsPackage; @@ -17,13 +17,25 @@ import jsinterop.base.Js; /** - * Common interface for various ways of accessing table data and formatting. - * + * Common interface for various ways of accessing table data and formatting for viewport or non-viewport subscriptions + * on tables, data in trees, and snapshots. + *

+ * Generally speaking, it is more efficient to access data in column-major order, rather than iterating through each Row + * and accessing all columns that it holds. The {@link #getRows()} accessor can be useful to read row data, but may + * incur other costs - it is likely faster to access data by columns using {@link #getData(RowPositionUnion, Column)}. + */ +/* * Java note: this interface contains some extra overloads that aren't available in JS. Implementations are expected to * implement only abstract methods, and default methods present in this interface will dispatch accordingly. */ -@TsName(namespace = "dh") +@JsType(namespace = "dh") public interface TableData { + @JsIgnore + int NO_ROW_FORMAT_COLUMN = -1; + + /** + * TS type union to allow either "int" or "LongWrapper" to be passed as an argument for various methods. + */ @TsUnion @JsType(name = "?", namespace = JsPackage.GLOBAL, isNative = true) interface RowPositionUnion { @@ -53,9 +65,18 @@ default int asInt() { @JsProperty JsArray getColumns(); + /** + * A lazily computed array of all rows available on the client. + */ @JsProperty JsArray<@TsTypeRef(Row.class) ? extends Row> getRows(); + /** + * Reads a row object from the table, from which any subscribed column can be read. + * + * @param index the position or key to access + * @return the row at the given location + */ @JsMethod default Row get(RowPositionUnion index) { if (index.isLongWrapper()) { @@ -64,10 +85,20 @@ default Row get(RowPositionUnion index) { return get(Js.coerceToInt(index)); } + @JsIgnore Row get(long index); + @JsIgnore Row get(int index); + /** + * Reads a specific cell from the table, by row key and column. + * + * @param index the row in the table to get data from + * @param column the column to read + * @return the value in the table + */ + // TODO (deephaven-core#5927) Consider a get/fillChunk API as an efficient alternative @JsMethod default Any getData(RowPositionUnion index, Column column) { if (index.isLongWrapper()) { @@ -76,10 +107,19 @@ default Any getData(RowPositionUnion index, Column column) { return getData(index.asInt(), column); } + @JsIgnore Any getData(int index, Column column); + @JsIgnore Any getData(long index, Column column); + /** + * The server-specified Format to use for the cell at the given position. + * + * @param index the row to read + * @param column the column to read + * @return a Format instance with any server-specified details + */ @JsMethod default Format getFormat(RowPositionUnion index, Column column) { if (index.isLongWrapper()) { @@ -88,12 +128,19 @@ default Format getFormat(RowPositionUnion index, Column column) { return getFormat(index.asInt(), column); } + @JsIgnore Format getFormat(int index, Column column); + @JsIgnore Format getFormat(long index, Column column); - @TsName(namespace = "dh") - public interface Row { + /** + * Represents a row available in a subscription/snapshot on the client. Do not retain references to rows - they will + * not function properly when the event isn't actively going off (or promise resolving). Instead, wait for the next + * event, or re-request the viewport data. + */ + @JsType(namespace = "dh") + interface Row { @JsProperty LongWrapper getIndex(); diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/WorkerConnection.java b/web/client-api/src/main/java/io/deephaven/web/client/api/WorkerConnection.java index 625b80443df..239dd2efbd0 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/WorkerConnection.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/WorkerConnection.java @@ -3,6 +3,7 @@ // package io.deephaven.web.client.api; +import com.google.flatbuffers.FlatBufferBuilder; import com.vertispan.tsdefs.annotations.TsIgnore; import elemental2.core.JsArray; import elemental2.core.JsObject; @@ -12,34 +13,20 @@ import elemental2.dom.CustomEventInit; import elemental2.dom.DomGlobal; import elemental2.promise.Promise; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf.FieldNode; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf.Message; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf.MessageHeader; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf.RecordBatch; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Buffer; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Field; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.KeyValue; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.MetadataVersion; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Schema; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.browserflight_pb_service.BrowserFlightServiceClient; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightData; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.HandshakeRequest; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.HandshakeResponse; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb_service.FlightServiceClient; import io.deephaven.javascript.proto.dhinternal.browserheaders.BrowserHeaders; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Long; import io.deephaven.javascript.proto.dhinternal.grpcweb.grpc.Code; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageMessageType; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageMessageWrapper; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageSubscriptionOptions; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageSubscriptionRequest; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageUpdateMetadata; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.ColumnConversionMode; +import io.deephaven.javascript.proto.dhinternal.grpcweb.grpc.UnaryOutput; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.application_pb.FieldInfo; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.application_pb.FieldsChangeUpdate; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.application_pb.ListFieldsRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.application_pb_service.ApplicationServiceClient; +import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.config_pb.ConfigValue; +import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.config_pb.ConfigurationConstantsRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.config_pb.ConfigurationConstantsResponse; +import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.config_pb_service.ConfigService; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.config_pb_service.ConfigServiceClient; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.console_pb.LogSubscriptionData; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.console_pb.LogSubscriptionRequest; @@ -69,16 +56,15 @@ import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.ticket_pb.Ticket; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.ticket_pb.TypedTicket; import io.deephaven.web.client.api.barrage.WebBarrageUtils; -import io.deephaven.web.client.api.barrage.def.ColumnDefinition; import io.deephaven.web.client.api.barrage.def.InitialTableDefinition; import io.deephaven.web.client.api.barrage.stream.BiDiStream; -import io.deephaven.web.client.api.barrage.stream.HandshakeStreamFactory; import io.deephaven.web.client.api.barrage.stream.ResponseStreamWrapper; import io.deephaven.web.client.api.batch.RequestBatcher; import io.deephaven.web.client.api.batch.TableConfig; import io.deephaven.web.client.api.console.JsVariableChanges; import io.deephaven.web.client.api.console.JsVariableDefinition; import io.deephaven.web.client.api.console.JsVariableType; +import io.deephaven.web.client.api.grpc.UnaryWithHeaders; import io.deephaven.web.client.api.i18n.JsTimeZone; import io.deephaven.web.client.api.impl.TicketAndPromise; import io.deephaven.web.client.api.lifecycle.HasLifecycle; @@ -94,22 +80,25 @@ import io.deephaven.web.client.state.ClientTableState; import io.deephaven.web.client.state.HasTableBinding; import io.deephaven.web.client.state.TableReviver; -import io.deephaven.web.shared.data.DeltaUpdates; -import io.deephaven.web.shared.data.RangeSet; -import io.deephaven.web.shared.data.TableSnapshot; -import io.deephaven.web.shared.data.TableSubscriptionRequest; import io.deephaven.web.shared.fu.JsConsumer; import io.deephaven.web.shared.fu.JsRunnable; import jsinterop.annotations.JsMethod; import jsinterop.annotations.JsOptional; import jsinterop.base.Js; import jsinterop.base.JsPropertyMap; +import org.apache.arrow.flatbuf.Buffer; +import org.apache.arrow.flatbuf.Field; +import org.apache.arrow.flatbuf.FieldNode; +import org.apache.arrow.flatbuf.KeyValue; +import org.apache.arrow.flatbuf.Message; +import org.apache.arrow.flatbuf.MessageHeader; +import org.apache.arrow.flatbuf.MetadataVersion; +import org.apache.arrow.flatbuf.RecordBatch; +import org.apache.arrow.flatbuf.Schema; import javax.annotation.Nullable; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; -import java.util.BitSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -121,12 +110,6 @@ import java.util.stream.Collectors; import static io.deephaven.web.client.api.CoreClient.EVENT_REFRESH_TOKEN_UPDATED; -import static io.deephaven.web.client.api.barrage.WebBarrageUtils.DeltaUpdatesBuilder; -import static io.deephaven.web.client.api.barrage.WebBarrageUtils.createSnapshot; -import static io.deephaven.web.client.api.barrage.WebBarrageUtils.deltaUpdates; -import static io.deephaven.web.client.api.barrage.WebBarrageUtils.makeUint8ArrayFromBitset; -import static io.deephaven.web.client.api.barrage.WebBarrageUtils.serializeRanges; -import static io.deephaven.web.client.api.barrage.WebBarrageUtils.typedArrayToLittleEndianByteBuffer; import static io.deephaven.web.client.api.barrage.WebGrpcUtils.CLIENT_OPTIONS; /** @@ -203,7 +186,6 @@ private enum State { private final Set flushable = new HashSet<>(); private final JsSet> logCallbacks = new JsSet<>(); - private final Map> subscriptionStreams = new HashMap<>(); private ResponseStreamWrapper exportNotifications; private JsSet simpleReconnectableInstances = new JsSet<>(); @@ -218,6 +200,8 @@ private enum State { private Map knownFields = new HashMap<>(); private ResponseStreamWrapper fieldsChangeUpdateStream; + private ConfigurationConstantsResponse constants; + public WorkerConnection(QueryConnectable info) { this.info = info; this.config = new ClientConfiguration(); @@ -304,7 +288,6 @@ private void connectToWorker() { ClientTableState[] hasActiveSubs = cache.getAllStates().stream() .peek(cts -> { cts.getHandle().setConnected(false); - cts.setSubscribed(false); cts.forActiveLifecycles(item -> { assert !(item instanceof JsTable) || ((JsTable) item).state() == cts @@ -479,54 +462,58 @@ private Promise authUpdate() { DomGlobal.clearTimeout(scheduledAuthUpdate); scheduledAuthUpdate = null; } - return new Promise<>((resolve, reject) -> { - // the streamfactory will automatically reference our existing metadata, but we can listen to update it - BiDiStream handshake = HandshakeStreamFactory.create(this); - handshake.onHeaders(headers -> { - // unchecked cast is required here due to "aliasing" in ts/webpack resulting in BrowserHeaders != - // Metadata - JsArray authorization = Js.uncheckedCast(headers).get(FLIGHT_AUTH_HEADER_NAME); - if (authorization.length > 0) { - JsArray existing = metadata().get(FLIGHT_AUTH_HEADER_NAME); - if (!existing.getAt(0).equals(authorization.getAt(0))) { - // use this new token - metadata().set(FLIGHT_AUTH_HEADER_NAME, authorization); - CustomEventInit init = CustomEventInit.create(); - init.setDetail(new JsRefreshToken(authorization.getAt(0), sessionTimeoutMs)); - info.fireEvent(EVENT_REFRESH_TOKEN_UPDATED, init); + return UnaryWithHeaders.call( + this, ConfigService.GetConfigurationConstants, new ConfigurationConstantsRequest()) + .then(result -> { + BrowserHeaders headers = result.getHeaders(); + // unchecked cast is required here due to "aliasing" in ts/webpack resulting in BrowserHeaders != + // Metadata + JsArray authorization = + Js.uncheckedCast(headers).get(FLIGHT_AUTH_HEADER_NAME); + if (authorization.length > 0) { + JsArray existing = metadata().get(FLIGHT_AUTH_HEADER_NAME); + if (!existing.getAt(0).equals(authorization.getAt(0))) { + // use this new token + metadata().set(FLIGHT_AUTH_HEADER_NAME, authorization); + CustomEventInit init = CustomEventInit.create(); + init.setDetail(new JsRefreshToken(authorization.getAt(0), sessionTimeoutMs)); + info.fireEvent(EVENT_REFRESH_TOKEN_UPDATED, init); + } } - } - handshake.end(); - }); - handshake.onStatus(status -> { - if (status.isOk()) { + + // Read the timeout from the server, we'll refresh at less than that + constants = result.getMessage(); + ConfigValue sessionDuration = constants.getConfigValuesMap().get("http.session.durationMs"); + if (sessionDuration != null && sessionDuration.hasStringValue()) { + sessionTimeoutMs = Double.parseDouble(sessionDuration.getStringValue()); + } + // schedule an update based on our currently configured delay scheduledAuthUpdate = DomGlobal.setTimeout(ignore -> { authUpdate(); }, sessionTimeoutMs / 2); - resolve.onInvoke((Void) null); - } else { - if (status.getCode() == Code.Unauthenticated) { + return Promise.resolve((Void) null); + }).catch_(err -> { + UnaryOutput result = (UnaryOutput) err; + if (result.getStatus() == Code.Unauthenticated) { // explicitly clear out any metadata for authentication, and signal that auth failed metadata.delete(FLIGHT_AUTH_HEADER_NAME); // Fire an event for the UI to attempt to re-auth info.fireEvent(CoreClient.EVENT_RECONNECT_AUTH_FAILED); - return; + + // We return here rather than continue and call checkStatus() + return Promise.reject("Authentication failed, please reconnect"); } - // TODO deephaven-core#2564 fire an event for the UI to re-auth - checkStatus(status); - if (status.getDetails() == null || status.getDetails().isEmpty()) { - reject.onInvoke("Error occurred while authenticating, gRPC status " + status.getCode()); + checkStatus(ResponseStreamWrapper.Status.of(result.getStatus(), result.getMessage().toString(), + result.getTrailers())); + if (result.getMessage() == null || result.getMessage().toString().isEmpty()) { + return Promise.reject(result.getMessage()); } else { - reject.onInvoke(status.getDetails()); + return Promise.reject("Error occurred while authenticating, gRPC status " + result.getStatus()); } - } - }); - - handshake.send(new HandshakeRequest()); - }); + }); } private void subscribeToTerminationNotification() { @@ -557,52 +544,10 @@ private void subscribeToTerminationNotification() { }); } - // @Override - public void initialSnapshot(TableTicket handle, TableSnapshot snapshot) { - LazyPromise.runLater(() -> { - // notify table that it has a snapshot available to replace viewport rows - // TODO looping in this way is not ideal, means that we're roughly O(n*m), where - // n is the number of rows, and m the number of tables with viewports. - // Instead, we should track all rows here in WorkerConnection, and then - // tell every table who might be interested about the rows it is interested in. - if (!cache.get(handle).isPresent()) { - JsLog.debug("Discarding snapshot for ", handle, " : ", snapshot); - } - cache.get(handle).ifPresent(s -> { - s.setSize(snapshot.getTableSize()); - s.forActiveTables(table -> { - table.handleSnapshot(handle, snapshot); - }); - }); - }); - } - - // @Override - public void incrementalUpdates(TableTicket tableHandle, DeltaUpdates updates) { - LazyPromise.runLater(() -> { - // notify table that it has individual row updates - final Optional cts = cache.get(tableHandle); - if (!cts.isPresent()) { - JsLog.debug("Discarding delta for disconnected state ", tableHandle, " : ", updates); - } - JsLog.debug("Delta received", tableHandle, updates); - cts.ifPresent(s -> { - if (!s.isSubscribed()) { - JsLog.debug("Discarding delta for unsubscribed table", tableHandle, updates); - return; - } - s.handleDelta(updates); - }); - }); - } - // @Override public void exportedTableUpdateMessage(TableTicket clientId, long size) { cache.get(clientId).ifPresent(state -> { - if (!state.isSubscribed()) { - // not presently subscribed so this is the only way to be informed of size changes - state.setSize(size); - } + state.setSize(size); }); } @@ -1074,12 +1019,12 @@ public Promise newTable(String[] columnNames, String[] types, Object[][ dataRef[0] = null; // make a schema that we can embed in the first DoPut message - Builder schema = new Builder(1024); + FlatBufferBuilder schema = new FlatBufferBuilder(1024); // while we're examining columns, build the copiers for data List columns = new ArrayList<>(); - double[] fields = new double[columnNames.length]; + int[] fields = new int[columnNames.length]; for (int i = 0; i < columnNames.length; i++) { String columnName = columnNames[i]; String columnType = types[i]; @@ -1087,9 +1032,9 @@ public Promise newTable(String[] columnNames, String[] types, Object[][ JsDataHandler writer = JsDataHandler.getHandler(columnType); columns.add(writer); - double nameOffset = schema.createString(columnName); - double typeOffset = writer.writeType(schema); - double metadataOffset = Field.createCustomMetadataVector(schema, new double[] { + int nameOffset = schema.createString(columnName); + int typeOffset = writer.writeType(schema); + int metadataOffset = Field.createCustomMetadataVector(schema, new int[] { KeyValue.createKeyValue(schema, schema.createString("deephaven:type"), schema.createString(writer.deephavenType())) }); @@ -1104,7 +1049,7 @@ public Promise newTable(String[] columnNames, String[] types, Object[][ fields[i] = Field.endField(schema); } - double fieldsOffset = Schema.createFieldsVector(schema, fields); + int fieldsOffset = Schema.createFieldsVector(schema, fields); Schema.startSchema(schema); Schema.addFields(schema, fieldsOffset); @@ -1146,7 +1091,7 @@ public Promise newTable(String[] columnNames, String[] types, Object[][ FlightData bodyMessage = new FlightData(); bodyMessage.setAppMetadata(WebBarrageUtils.emptyMessage()); - Builder bodyData = new Builder(1024); + FlatBufferBuilder bodyData = new FlatBufferBuilder(1024); // iterate each column, building buffers and fieldnodes, as well as building the actual payload List buffers = new ArrayList<>(); @@ -1170,25 +1115,25 @@ public Promise newTable(String[] columnNames, String[] types, Object[][ for (int i = buffers.size() - 1; i >= 0; i--) { Uint8Array buffer = buffers.get(i); cumulativeOffset -= buffer.byteLength; - Buffer.createBuffer(bodyData, Long.create(cumulativeOffset, 0), Long.create(buffer.byteLength, 0)); + Buffer.createBuffer(bodyData, cumulativeOffset, buffer.byteLength); } assert cumulativeOffset == 0; - double buffersOffset = bodyData.endVector(); + int buffersOffset = bodyData.endVector(); RecordBatch.startNodesVector(bodyData, nodes.size()); for (int i = nodes.size() - 1; i >= 0; i--) { JsDataHandler.Node node = nodes.get(i); - FieldNode.createFieldNode(bodyData, Long.create(node.length(), 0), Long.create(node.nullCount(), 0)); + FieldNode.createFieldNode(bodyData, node.length(), node.nullCount()); } - double nodesOffset = bodyData.endVector(); + int nodesOffset = bodyData.endVector(); RecordBatch.startRecordBatch(bodyData); RecordBatch.addBuffers(bodyData, buffersOffset); RecordBatch.addNodes(bodyData, nodesOffset); - RecordBatch.addLength(bodyData, Long.create(data[0].length, 0)); + RecordBatch.addLength(bodyData, data[0].length); - double recordBatchOffset = RecordBatch.endRecordBatch(bodyData); + int recordBatchOffset = RecordBatch.endRecordBatch(bodyData); bodyMessage.setDataHeader(createMessage(bodyData, MessageHeader.RecordBatch, recordBatchOffset, length, 0)); bodyMessage.setDataBody(padAndConcat(buffers, length)); @@ -1209,11 +1154,11 @@ private Uint8Array padAndConcat(List buffers, int length) { return all; } - private static Uint8Array createMessage(Builder payload, int messageHeaderType, double messageHeaderOffset, - int bodyLength, double customMetadataOffset) { + private static Uint8Array createMessage(FlatBufferBuilder payload, byte messageHeaderType, int messageHeaderOffset, + int bodyLength, int customMetadataOffset) { payload.finish(Message.createMessage(payload, MetadataVersion.V5, messageHeaderType, messageHeaderOffset, - Long.create(bodyLength, 0), customMetadataOffset)); - return payload.asUint8Array(); + bodyLength, customMetadataOffset)); + return WebBarrageUtils.bbToUint8ArrayView(payload.dataBuffer()); } public Promise mergeTables(JsTable[] tables, HasEventHandling failHandler) { @@ -1317,7 +1262,7 @@ public StateCache getCache() { } /** - * Schedules a deferred command to check the given state for active tables and adjust viewports accordingly. + * Schedules a deferred command to check the given state for active tables. */ public void scheduleCheck(ClientTableState state) { if (flushable.isEmpty()) { @@ -1342,210 +1287,24 @@ public void releaseTicket(Ticket ticket) { sessionServiceClient.release(releaseRequest, metadata, null); } - - /** - * For those calls where we don't really care what happens - */ - private static final Callback DONOTHING_CALLBACK = new Callback() { - @Override - public void onSuccess(Void value) { - // Do nothing. - } - - @Override - public void onFailure(String error) { - JsLog.error("Callback failed: " + error); - } - }; - private void flush() { - // LATER: instead of running a bunch of serial operations, - // condense these all into a single batch operation. - // All three server calls made by this method are _only_ called by this method, - // so we can reasonably merge all three into a single batched operation. ArrayList statesToFlush = new ArrayList<>(flushable); flushable.clear(); - for (ClientTableState state : statesToFlush) { - if (state.hasNoSubscriptions()) { - // state may be retained if it is held by at least one paused binding; - // it is either an unsubscribed active table, an interim state for an - // active table, or a pending rollback for an operation that has not - // yet completed (we leave orphaned nodes paused until a request completes). - if (state.isSubscribed()) { - state.setSubscribed(false); + if (state.isEmpty()) { + // completely empty; perform release + final ClientTableState.ResolutionState previousState = state.getResolution(); + state.setResolution(ClientTableState.ResolutionState.RELEASED); + if (previousState != ClientTableState.ResolutionState.RELEASED) { + cache.release(state); + + JsLog.debug("Releasing state", state, LazyString.of(state.getHandle())); + // don't send a release message to the server if the table isn't really there if (state.getHandle().isConnected()) { - BiDiStream stream = subscriptionStreams.remove(state); - if (stream != null) { - stream.end(); - stream.cancel(); - } + releaseHandle(state.getHandle()); } } - - if (state.isEmpty()) { - // completely empty; perform release - final ClientTableState.ResolutionState previousState = state.getResolution(); - state.setResolution(ClientTableState.ResolutionState.RELEASED); - state.setSubscribed(false); - if (previousState != ClientTableState.ResolutionState.RELEASED) { - cache.release(state); - - JsLog.debug("Releasing state", state, LazyString.of(state.getHandle())); - // don't send a release message to the server if the table isn't really there - if (state.getHandle().isConnected()) { - releaseHandle(state.getHandle()); - } - } - } - } else if (state.isRunning()) { - List vps = new ArrayList<>(); - state.forActiveSubscriptions((table, subscription) -> { - assert table.isActive(state) : "Inactive table has a viewport still attached"; - vps.add(new TableSubscriptionRequest(table.getSubscriptionId(), subscription.getRows(), - subscription.getColumns())); - }); - - boolean isViewport = vps.stream().allMatch(req -> req.getRows() != null); - assert isViewport || vps.stream().noneMatch(req -> req.getRows() != null) - : "All subscriptions to a given handle must be consistently viewport or non-viewport"; - - - BitSet includedColumns = vps.stream().map(TableSubscriptionRequest::getColumns).reduce((bs1, bs2) -> { - BitSet result = new BitSet(); - result.or(bs1); - result.or(bs2); - return result; - }).orElseThrow(() -> new IllegalStateException("Cannot call subscribe with zero subscriptions")); - String[] columnTypes = Arrays.stream(state.getTableDef().getColumns()) - .map(ColumnDefinition::getType) - .toArray(String[]::new); - - state.setSubscribed(true); - - Builder subscriptionReq = new Builder(1024); - - double columnsOffset = BarrageSubscriptionRequest.createColumnsVector(subscriptionReq, - makeUint8ArrayFromBitset(includedColumns)); - double viewportOffset = 0; - if (isViewport) { - viewportOffset = BarrageSubscriptionRequest.createViewportVector(subscriptionReq, serializeRanges( - vps.stream().map(TableSubscriptionRequest::getRows).collect(Collectors.toSet()))); - } - // TODO #188 support minUpdateIntervalMs - double serializationOptionsOffset = BarrageSubscriptionOptions - .createBarrageSubscriptionOptions(subscriptionReq, ColumnConversionMode.Stringify, true, 1000, - 0, 0); - double tableTicketOffset = - BarrageSubscriptionRequest.createTicketVector(subscriptionReq, state.getHandle().getTicket()); - BarrageSubscriptionRequest.startBarrageSubscriptionRequest(subscriptionReq); - BarrageSubscriptionRequest.addColumns(subscriptionReq, columnsOffset); - BarrageSubscriptionRequest.addSubscriptionOptions(subscriptionReq, serializationOptionsOffset); - BarrageSubscriptionRequest.addViewport(subscriptionReq, viewportOffset); - BarrageSubscriptionRequest.addTicket(subscriptionReq, tableTicketOffset); - subscriptionReq.finish(BarrageSubscriptionRequest.endBarrageSubscriptionRequest(subscriptionReq)); - - FlightData request = new FlightData(); - request.setAppMetadata( - WebBarrageUtils.wrapMessage(subscriptionReq, BarrageMessageType.BarrageSubscriptionRequest)); - - BiDiStream stream = this.streamFactory().create( - headers -> flightServiceClient.doExchange(headers), - (first, headers) -> browserFlightServiceClient.openDoExchange(first, headers), - (next, headers, c) -> browserFlightServiceClient.nextDoExchange(next, headers, c::apply), - new FlightData()); - - stream.send(request); - stream.onData(new JsConsumer() { - @Override - public void apply(FlightData data) { - ByteBuffer body = typedArrayToLittleEndianByteBuffer(data.getDataBody_asU8()); - Message headerMessage = Message - .getRootAsMessage(new io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer( - data.getDataHeader_asU8())); - if (body.limit() == 0 && headerMessage.headerType() != MessageHeader.RecordBatch) { - // a subscription stream presently ignores schemas and other message types - // TODO hang on to the schema to better handle the now-Utf8 columns - return; - } - RecordBatch header = headerMessage.header(new RecordBatch()); - BarrageMessageWrapper barrageMessageWrapper = - BarrageMessageWrapper.getRootAsBarrageMessageWrapper( - new io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer( - data.getAppMetadata_asU8())); - if (barrageMessageWrapper.msgType() == BarrageMessageType.None) { - // continue previous message, just read RecordBatch - appendAndMaybeFlush(header, body); - } else { - assert barrageMessageWrapper.msgType() == BarrageMessageType.BarrageUpdateMetadata; - BarrageUpdateMetadata barrageUpdate = BarrageUpdateMetadata.getRootAsBarrageUpdateMetadata( - new io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer( - new Uint8Array(barrageMessageWrapper.msgPayloadArray()))); - startAndMaybeFlush(barrageUpdate.isSnapshot(), header, body, barrageUpdate, isViewport, - columnTypes); - } - } - - private DeltaUpdatesBuilder nextDeltaUpdates; - private DeltaUpdates deferredDeltaUpdates; - - private void appendAndMaybeFlush(RecordBatch header, ByteBuffer body) { - // using existing barrageUpdate, append to the current snapshot/delta - assert nextDeltaUpdates != null; - boolean shouldFlush = nextDeltaUpdates.appendRecordBatch(header, body); - if (shouldFlush) { - DeltaUpdates updates = nextDeltaUpdates.build(); - nextDeltaUpdates = null; - - if (state.getTableDef().getAttributes().isBlinkTable()) { - // blink tables remove all rows from the previous step, if there are no adds this step - // then defer removal until new data arrives -- this makes blink tables GUI friendly - if (updates.getAdded().isEmpty()) { - if (deferredDeltaUpdates != null) { - final RangeSet removed = deferredDeltaUpdates.getRemoved(); - updates.getRemoved().rangeIterator().forEachRemaining(removed::addRange); - } else { - deferredDeltaUpdates = updates; - } - return; - } else if (deferredDeltaUpdates != null) { - assert updates.getRemoved().isEmpty() - : "Blink table received two consecutive remove rowsets"; - updates.setRemoved(deferredDeltaUpdates.getRemoved()); - deferredDeltaUpdates = null; - } - } - incrementalUpdates(state.getHandle(), updates); - } - } - - private void startAndMaybeFlush(boolean isSnapshot, RecordBatch header, ByteBuffer body, - BarrageUpdateMetadata barrageUpdate, boolean isViewport, String[] columnTypes) { - if (isSnapshot) { - TableSnapshot snapshot = - createSnapshot(header, body, barrageUpdate, isViewport, columnTypes); - - // for now we always expect snapshots to arrive in a single payload - initialSnapshot(state.getHandle(), snapshot); - } else { - nextDeltaUpdates = deltaUpdates(barrageUpdate, isViewport, columnTypes); - appendAndMaybeFlush(header, body); - } - } - }); - stream.onStatus(err -> { - checkStatus(err); - if (!err.isOk() && !err.isTransportError()) { - state.setResolution(ClientTableState.ResolutionState.FAILED, err.getDetails()); - } - }); - BiDiStream oldStream = subscriptionStreams.put(state, stream); - if (oldStream != null) { - // cancel any old stream, we presently expect a fresh instance - oldStream.end(); - oldStream.cancel(); - } } } } @@ -1570,6 +1329,10 @@ public ClientConfiguration getConfig() { return config; } + public ConfigValue getServerConfigValue(String key) { + return constants.getConfigValuesMap().get(key); + } + public void onOpen(BiConsumer callback) { switch (state) { case Connected: diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/CompressedRangeSetReader.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/CompressedRangeSetReader.java index e4f14ef2fd4..1ee0c147621 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/CompressedRangeSetReader.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/CompressedRangeSetReader.java @@ -175,7 +175,7 @@ public RangeSet read(ByteBuffer data) { if (pending >= 0) { append(pending); } - return RangeSet.fromSortedRanges(sortedRanges.toArray(new Range[0])); + return RangeSet.fromSortedRanges(sortedRanges); default: throw new IllegalStateException("Bad command: " + command + " at position " + data.position()); } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/ShiftedRangeReader.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/ShiftedRangeReader.java index ac2b7524e6b..3c1c68fc3bb 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/ShiftedRangeReader.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/ShiftedRangeReader.java @@ -12,7 +12,7 @@ public class ShiftedRangeReader { - public ShiftedRange[] read(ByteBuffer data) { + public static ShiftedRange[] read(ByteBuffer data) { RangeSet start = new CompressedRangeSetReader().read(data); RangeSet end = new CompressedRangeSetReader().read(data); RangeSet postShiftStart = new CompressedRangeSetReader().read(data); @@ -30,4 +30,29 @@ public ShiftedRange[] read(ByteBuffer data) { return ranges; } + + public static ByteBuffer write(ShiftedRange[] shiftedRanges) { + RangeSet start = new RangeSet(); + RangeSet end = new RangeSet(); + RangeSet postShiftStart = new RangeSet(); + + for (int i = 0; i < shiftedRanges.length; i++) { + ShiftedRange range = shiftedRanges[i]; + long first = range.getRange().getFirst(); + long last = range.getRange().getLast(); + long delta = range.getDelta() + first; + start.addRange(new Range(first, first)); + end.addRange(new Range(last, last)); + postShiftStart.addRange(new Range(delta, delta)); + } + + ByteBuffer startBuf = CompressedRangeSetReader.writeRange(start); + ByteBuffer endBuf = CompressedRangeSetReader.writeRange(end); + ByteBuffer shiftBuf = CompressedRangeSetReader.writeRange(postShiftStart); + ByteBuffer all = ByteBuffer.allocateDirect(startBuf.remaining() + endBuf.remaining() + shiftBuf.remaining()); + all.put(startBuf); + all.put(endBuf); + all.put(shiftBuf); + return all; + } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageMessage.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageMessage.java new file mode 100644 index 00000000000..1b26f2ccadb --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageMessage.java @@ -0,0 +1,49 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.web.client.api.barrage; + +import io.deephaven.chunk.Chunk; +import io.deephaven.chunk.ChunkType; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.web.shared.data.RangeSet; +import io.deephaven.web.shared.data.ShiftedRange; + +import java.util.ArrayList; +import java.util.BitSet; + +public class WebBarrageMessage { + public static class ModColumnData { + public RangeSet rowsModified; + public Class type; + public Class componentType; + public ArrayList> data; + public ChunkType chunkType; + } + public static class AddColumnData { + public Class type; + public Class componentType; + public ArrayList> data; + public ChunkType chunkType; + } + + public long firstSeq = -1; + public long lastSeq = -1; + public long step = -1; + + public boolean isSnapshot; + public RangeSet snapshotRowSet; + public boolean snapshotRowSetIsReversed; + public BitSet snapshotColumns; + + public RangeSet rowsAdded; + public RangeSet rowsIncluded; + public RangeSet rowsRemoved; + public ShiftedRange[] shifted; + + public AddColumnData[] addColumnData; + public ModColumnData[] modColumnData; + + // Underlying RecordBatch.length, visible for reading snapshots + public long length; +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageStreamReader.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageStreamReader.java new file mode 100644 index 00000000000..6bf5196034e --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageStreamReader.java @@ -0,0 +1,291 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.web.client.api.barrage; + +import com.google.common.io.LittleEndianDataInputStream; +import io.deephaven.barrage.flatbuf.BarrageMessageType; +import io.deephaven.barrage.flatbuf.BarrageMessageWrapper; +import io.deephaven.barrage.flatbuf.BarrageModColumnMetadata; +import io.deephaven.barrage.flatbuf.BarrageUpdateMetadata; +import io.deephaven.chunk.ChunkType; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.extensions.barrage.chunk.ChunkInputStreamGenerator; +import io.deephaven.extensions.barrage.chunk.ChunkReader; +import io.deephaven.extensions.barrage.util.FlatBufferIteratorAdapter; +import io.deephaven.extensions.barrage.util.StreamReaderOptions; +import io.deephaven.io.streams.ByteBufferInputStream; +import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightData; +import io.deephaven.util.datastructures.LongSizedDataStructure; +import io.deephaven.web.client.fu.JsLog; +import io.deephaven.web.shared.data.RangeSet; +import io.deephaven.web.shared.data.ShiftedRange; +import org.apache.arrow.flatbuf.Field; +import org.apache.arrow.flatbuf.Message; +import org.apache.arrow.flatbuf.MessageHeader; +import org.apache.arrow.flatbuf.RecordBatch; +import org.apache.arrow.flatbuf.Schema; +import org.gwtproject.nio.TypedArrayHelper; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Iterator; +import java.util.List; +import java.util.PrimitiveIterator; + +/** + * Consumes FlightData fields from Flight/Barrage producers and builds browser-compatible WebBarrageMessage payloads + * that can be used to maintain table data. + */ +public class WebBarrageStreamReader { + private static final int MAX_CHUNK_SIZE = Integer.MAX_VALUE - 8; + + // record progress in reading + private long numAddRowsRead = 0; + private long numAddRowsTotal = 0; + private long numModRowsRead = 0; + private long numModRowsTotal = 0; + + // hold in-progress messages that aren't finished being built + private WebBarrageMessage msg; + + private final WebChunkReaderFactory chunkReaderFactory = new WebChunkReaderFactory(); + private final List readers = new ArrayList<>(); + + public WebBarrageMessage parseFrom( + final StreamReaderOptions options, + ChunkType[] columnChunkTypes, + Class[] columnTypes, + Class[] componentTypes, + FlightData flightData) throws IOException { + ByteBuffer headerAsBB = TypedArrayHelper.wrap(flightData.getDataHeader_asU8()); + Message header = headerAsBB.hasRemaining() ? Message.getRootAsMessage(headerAsBB) : null; + + ByteBuffer msgAsBB = TypedArrayHelper.wrap(flightData.getAppMetadata_asU8()); + if (msgAsBB.hasRemaining()) { + BarrageMessageWrapper wrapper = + BarrageMessageWrapper.getRootAsBarrageMessageWrapper(msgAsBB); + if (wrapper.magic() != WebBarrageUtils.FLATBUFFER_MAGIC) { + JsLog.warn( + "WebBarrageStreamReader: skipping app_metadata that does not look like BarrageMessageWrapper"); + } else if (wrapper.msgType() == BarrageMessageType.BarrageUpdateMetadata) { + if (msg != null) { + throw new IllegalStateException( + "Previous message was not complete; pending " + (numAddRowsTotal - numAddRowsRead) + + " add rows and " + (numModRowsTotal - numModRowsRead) + " mod rows"); + } + + final BarrageUpdateMetadata metadata = + BarrageUpdateMetadata.getRootAsBarrageUpdateMetadata(wrapper.msgPayloadAsByteBuffer()); + + msg = new WebBarrageMessage(); + + msg.isSnapshot = metadata.isSnapshot(); + msg.snapshotRowSetIsReversed = metadata.effectiveReverseViewport(); + + numAddRowsRead = 0; + numModRowsRead = 0; + + if (msg.isSnapshot) { + final ByteBuffer effectiveViewport = metadata.effectiveViewportAsByteBuffer(); + if (effectiveViewport != null) { + msg.snapshotRowSet = extractIndex(effectiveViewport); + } + final ByteBuffer effectiveSnapshotColumns = metadata.effectiveColumnSetAsByteBuffer(); + if (effectiveSnapshotColumns != null) { + msg.snapshotColumns = extractBitSet(effectiveSnapshotColumns); + } + } + + msg.firstSeq = metadata.firstSeq(); + msg.lastSeq = metadata.lastSeq(); + msg.rowsAdded = extractIndex(metadata.addedRowsAsByteBuffer()); + msg.rowsRemoved = extractIndex(metadata.removedRowsAsByteBuffer()); + msg.shifted = extractIndexShiftData(metadata.shiftDataAsByteBuffer()); + + final ByteBuffer rowsIncluded = metadata.addedRowsIncludedAsByteBuffer(); + msg.rowsIncluded = rowsIncluded != null ? extractIndex(rowsIncluded) : msg.rowsAdded; + msg.addColumnData = new WebBarrageMessage.AddColumnData[columnTypes.length]; + for (int ci = 0; ci < msg.addColumnData.length; ++ci) { + msg.addColumnData[ci] = new WebBarrageMessage.AddColumnData(); + // msg.addColumnData[ci].type = columnTypes[ci]; + // msg.addColumnData[ci].componentType = componentTypes[ci]; + msg.addColumnData[ci].data = new ArrayList<>(); + + // create an initial chunk of the correct size + final int chunkSize = (int) (Math.min(msg.rowsIncluded.size(), MAX_CHUNK_SIZE)); + final WritableChunk chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); + chunk.setSize(0); + msg.addColumnData[ci].data.add(chunk); + } + numAddRowsTotal = msg.rowsIncluded.size(); + + // if this message is a snapshot response (vs. subscription) then mod columns may be empty + numModRowsTotal = 0; + msg.modColumnData = new WebBarrageMessage.ModColumnData[metadata.modColumnNodesLength()]; + for (int ci = 0; ci < msg.modColumnData.length; ++ci) { + msg.modColumnData[ci] = new WebBarrageMessage.ModColumnData(); + // msg.modColumnData[ci].type = columnTypes[ci]; + // msg.modColumnData[ci].componentType = componentTypes[ci]; + msg.modColumnData[ci].data = new ArrayList<>(); + + final BarrageModColumnMetadata mcd = metadata.modColumnNodes(ci); + msg.modColumnData[ci].rowsModified = extractIndex(mcd.modifiedRowsAsByteBuffer()); + + // create an initial chunk of the correct size + final int chunkSize = (int) (Math.min(msg.modColumnData[ci].rowsModified.size(), + MAX_CHUNK_SIZE)); + final WritableChunk chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); + chunk.setSize(0); + msg.modColumnData[ci].data.add(chunk); + + numModRowsTotal = Math.max(numModRowsTotal, msg.modColumnData[ci].rowsModified.size()); + } + } + } + + byte headerType = header.headerType(); + if (headerType == MessageHeader.Schema) { + // there is no body and our clients do not want to see schema messages + Schema schema = new Schema(); + header.header(schema); + for (int i = 0; i < schema.fieldsLength(); i++) { + Field field = schema.fields(i); + ChunkReader chunkReader = chunkReaderFactory.getReader(options, + ChunkReader.typeInfo(columnChunkTypes[i], columnTypes[i], + componentTypes[i], field)); + readers.add(chunkReader); + } + return null; + } + if (headerType != MessageHeader.RecordBatch) { + throw new IllegalStateException("Only know how to decode Schema/RecordBatch messages"); + } + + + // throw an error when no app metadata (snapshots now provide by default) + if (msg == null) { + throw new IllegalStateException( + "Missing app metadata tag; cannot decode using BarrageStreamReader"); + } + + final RecordBatch batch = (RecordBatch) header.header(new RecordBatch()); + msg.length = batch.length(); + ByteBuffer body = TypedArrayHelper.wrap(flightData.getDataBody_asU8()); + final LittleEndianDataInputStream ois = + new LittleEndianDataInputStream(new ByteBufferInputStream(body)); + final Iterator fieldNodeIter = + new FlatBufferIteratorAdapter<>(batch.nodesLength(), + i -> new ChunkInputStreamGenerator.FieldNodeInfo(batch.nodes(i))); + + final long[] bufferInfo = new long[batch.buffersLength()]; + for (int bi = 0; bi < batch.buffersLength(); ++bi) { + int offset = LongSizedDataStructure.intSize("BufferInfo", batch.buffers(bi).offset()); + int length = LongSizedDataStructure.intSize("BufferInfo", batch.buffers(bi).length()); + if (bi < batch.buffersLength() - 1) { + final int nextOffset = + LongSizedDataStructure.intSize("BufferInfo", batch.buffers(bi + 1).offset()); + // our parsers handle overhanging buffers + length += Math.max(0, nextOffset - offset - length); + } + bufferInfo[bi] = length; + } + final PrimitiveIterator.OfLong bufferInfoIter = Arrays.stream(bufferInfo).iterator(); + + + // add and mod rows are never combined in a batch. all added rows must be received before the first + // mod rows will be received. + if (numAddRowsRead < numAddRowsTotal) { + for (int ci = 0; ci < msg.addColumnData.length; ++ci) { + final WebBarrageMessage.AddColumnData acd = msg.addColumnData[ci]; + + final long remaining = numAddRowsTotal - numAddRowsRead; + if (batch.length() > remaining) { + throw new IllegalStateException( + "Batch length exceeded the expected number of rows from app metadata"); + } + + // select the current chunk size and read the size + int lastChunkIndex = acd.data.size() - 1; + WritableChunk chunk = (WritableChunk) acd.data.get(lastChunkIndex); + + if (batch.length() > chunk.capacity() - chunk.size()) { + // reading the rows from this batch will overflow the existing chunk; create a new one + final int chunkSize = (int) (Math.min(remaining, MAX_CHUNK_SIZE)); + chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); + acd.data.add(chunk); + + chunk.setSize(0); + ++lastChunkIndex; + } + + // fill the chunk with data and assign back into the array + acd.data.set(lastChunkIndex, + readers.get(ci).readChunk(fieldNodeIter, bufferInfoIter, ois, chunk, chunk.size(), + (int) batch.length())); + chunk.setSize(chunk.size() + (int) batch.length()); + } + numAddRowsRead += batch.length(); + } else { + for (int ci = 0; ci < msg.modColumnData.length; ++ci) { + final WebBarrageMessage.ModColumnData mcd = msg.modColumnData[ci]; + + // another column may be larger than this column + long remaining = Math.max(0, mcd.rowsModified.size() - numModRowsRead); + + // need to add the batch row data to the column chunks + int lastChunkIndex = mcd.data.size() - 1; + WritableChunk chunk = (WritableChunk) mcd.data.get(lastChunkIndex); + + final int numRowsToRead = LongSizedDataStructure.intSize("BarrageStreamReader", + Math.min(remaining, batch.length())); + if (numRowsToRead > chunk.capacity() - chunk.size()) { + // reading the rows from this batch will overflow the existing chunk; create a new one + final int chunkSize = (int) (Math.min(remaining, MAX_CHUNK_SIZE)); + chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); + mcd.data.add(chunk); + + chunk.setSize(0); + ++lastChunkIndex; + } + + // fill the chunk with data and assign back into the array + mcd.data.set(lastChunkIndex, + readers.get(ci).readChunk(fieldNodeIter, bufferInfoIter, ois, chunk, chunk.size(), + numRowsToRead)); + chunk.setSize(chunk.size() + numRowsToRead); + } + numModRowsRead += batch.length(); + } + + if (numAddRowsRead == numAddRowsTotal && numModRowsRead == numModRowsTotal) { + final WebBarrageMessage retval = msg; + msg = null; + return retval; + } + + // otherwise, must wait for more data + return null; + } + + private static RangeSet extractIndex(final ByteBuffer bb) { + if (bb == null) { + return RangeSet.empty(); + } + return new CompressedRangeSetReader().read(bb); + } + + private static BitSet extractBitSet(final ByteBuffer bb) { + byte[] array = new byte[bb.remaining()]; + bb.get(array); + return BitSet.valueOf(array); + } + + private static ShiftedRange[] extractIndexShiftData(final ByteBuffer bb) { + return ShiftedRangeReader.read(bb); + } +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageUtils.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageUtils.java index 8f11e5ebe27..648025723a5 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageUtils.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageUtils.java @@ -3,64 +3,58 @@ // package io.deephaven.web.client.api.barrage; +import com.google.flatbuffers.FlatBufferBuilder; import elemental2.core.*; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf.FieldNode; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf.Message; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf.MessageHeader; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf.RecordBatch; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Buffer; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Field; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.KeyValue; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Schema; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageMessageType; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageMessageWrapper; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageModColumnMetadata; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageUpdateMetadata; +import io.deephaven.barrage.flatbuf.BarrageMessageType; +import io.deephaven.barrage.flatbuf.BarrageMessageWrapper; import io.deephaven.web.client.api.barrage.def.ColumnDefinition; import io.deephaven.web.client.api.barrage.def.InitialTableDefinition; import io.deephaven.web.client.api.barrage.def.TableAttributesDefinition; import io.deephaven.web.shared.data.*; -import io.deephaven.web.shared.data.columns.*; -import jsinterop.base.Js; +import org.apache.arrow.flatbuf.KeyValue; +import org.apache.arrow.flatbuf.Message; +import org.apache.arrow.flatbuf.MessageHeader; +import org.apache.arrow.flatbuf.Schema; import org.gwtproject.nio.TypedArrayHelper; -import java.math.BigDecimal; -import java.math.BigInteger; import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.IntBuffer; -import java.nio.charset.StandardCharsets; -import java.util.BitSet; import java.util.HashMap; -import java.util.Iterator; import java.util.Map; import java.util.Set; -import java.util.function.DoubleFunction; -import java.util.stream.IntStream; +import java.util.function.IntFunction; /** * Utility to read barrage record batches. */ public class WebBarrageUtils { - private static final int MAGIC = 0x6E687064; - - public static Uint8Array wrapMessage(Builder innerBuilder, int messageType) { - Builder outerBuilder = new Builder(1024); - // This deprecation is incorrect, tsickle didn't understand that only one overload is deprecated - // noinspection deprecation - double messageOffset = BarrageMessageWrapper.createMsgPayloadVector(outerBuilder, innerBuilder.asUint8Array()); - double offset = - BarrageMessageWrapper.createBarrageMessageWrapper(outerBuilder, MAGIC, messageType, messageOffset); + public static final int FLATBUFFER_MAGIC = 0x6E687064; + + public static Uint8Array wrapMessage(FlatBufferBuilder innerBuilder, byte messageType) { + FlatBufferBuilder outerBuilder = new FlatBufferBuilder(1024); + int messageOffset = BarrageMessageWrapper.createMsgPayloadVector(outerBuilder, innerBuilder.dataBuffer()); + int offset = + BarrageMessageWrapper.createBarrageMessageWrapper(outerBuilder, FLATBUFFER_MAGIC, messageType, + messageOffset); outerBuilder.finish(offset); - return outerBuilder.asUint8Array(); + ByteBuffer byteBuffer = outerBuilder.dataBuffer(); + return bbToUint8ArrayView(byteBuffer); + } + + public static Uint8Array bbToUint8ArrayView(ByteBuffer byteBuffer) { + ArrayBufferView view = TypedArrayHelper.unwrap(byteBuffer); + return new Uint8Array(view.buffer, byteBuffer.position() + view.byteOffset, byteBuffer.remaining()); } public static Uint8Array emptyMessage() { - Builder builder = new Builder(1024); - double offset = BarrageMessageWrapper.createBarrageMessageWrapper(builder, MAGIC, BarrageMessageType.None, 0); + FlatBufferBuilder builder = new FlatBufferBuilder(1024); + int offset = BarrageMessageWrapper.createBarrageMessageWrapper(builder, FLATBUFFER_MAGIC, + BarrageMessageType.None, 0); builder.finish(offset); - return builder.asUint8Array(); + return bbToUint8ArrayView(builder.dataBuffer()); + } + + public static InitialTableDefinition readTableDefinition(Uint8Array flightSchemaMessage) { + return readTableDefinition(readSchemaMessage(flightSchemaMessage)); } public static InitialTableDefinition readTableDefinition(Schema schema) { @@ -76,50 +70,10 @@ public static InitialTableDefinition readTableDefinition(Schema schema) { .setColumns(cols); } - public static ColumnDefinition[] readColumnDefinitions(Schema schema) { + private static ColumnDefinition[] readColumnDefinitions(Schema schema) { ColumnDefinition[] cols = new ColumnDefinition[(int) schema.fieldsLength()]; for (int i = 0; i < schema.fieldsLength(); i++) { - cols[i] = new ColumnDefinition(); - Field f = schema.fields(i); - Map fieldMetadata = - keyValuePairs("deephaven:", f.customMetadataLength(), f::customMetadata); - cols[i].setName(f.name().asString()); - cols[i].setColumnIndex(i); - cols[i].setType(fieldMetadata.get("type")); - cols[i].setIsSortable("true".equals(fieldMetadata.get("isSortable"))); - cols[i].setStyleColumn("true".equals(fieldMetadata.get("isStyle"))); - cols[i].setFormatColumn("true".equals(fieldMetadata.get("isDateFormat")) - || "true".equals(fieldMetadata.get("isNumberFormat"))); - cols[i].setForRow("true".equals(fieldMetadata.get("isRowStyle"))); - - String formatColumnName = fieldMetadata.get("dateFormatColumn"); - if (formatColumnName == null) { - formatColumnName = fieldMetadata.get("numberFormatColumn"); - } - cols[i].setFormatColumnName(formatColumnName); - - cols[i].setStyleColumnName(fieldMetadata.get("styleColumn")); - - if (fieldMetadata.containsKey("inputtable.isKey")) { - cols[i].setInputTableKeyColumn("true".equals(fieldMetadata.get("inputtable.isKey"))); - } - - cols[i].setDescription(fieldMetadata.get("description")); - - cols[i].setPartitionColumn("true".equals(fieldMetadata.get("isPartitioning"))); - - cols[i].setHierarchicalExpandByColumn( - "true".equals(fieldMetadata.get("hierarchicalTable.isExpandByColumn"))); - cols[i].setHierarchicalRowDepthColumn( - "true".equals(fieldMetadata.get("hierarchicalTable.isRowDepthColumn"))); - cols[i].setHierarchicalRowExpandedColumn( - "true".equals(fieldMetadata.get("hierarchicalTable.isRowExpandedColumn"))); - cols[i].setRollupAggregatedNodeColumn( - "true".equals(fieldMetadata.get("rollupTable.isAggregatedNodeColumn"))); - cols[i].setRollupConstituentNodeColumn( - "true".equals(fieldMetadata.get("rollupTable.isConstituentNodeColumn"))); - cols[i].setRollupGroupByColumn("true".equals(fieldMetadata.get("rollupTable.isGroupByColumn"))); - cols[i].setRollupAggregationInputColumn(fieldMetadata.get("rollupTable.aggregationInputColumnName")); + cols[i] = new ColumnDefinition(i, schema.fields(i)); } return cols; } @@ -129,505 +83,42 @@ public static Schema readSchemaMessage(Uint8Array flightSchemaMessage) { // - IPC_CONTINUATION_TOKEN (4-byte int of -1) // - message size (4-byte int) // - a Message wrapping the schema - io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer bb = - new io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer(flightSchemaMessage); - bb.setPosition(bb.position() + 8); + ByteBuffer bb = TypedArrayHelper.wrap(flightSchemaMessage); + bb.position(bb.position() + 8); Message headerMessage = Message.getRootAsMessage(bb); assert headerMessage.headerType() == MessageHeader.Schema; - return headerMessage.header(new Schema()); + return (Schema) headerMessage.header(new Schema()); } public static Map keyValuePairs(String filterPrefix, double count, - DoubleFunction accessor) { + IntFunction accessor) { Map map = new HashMap<>(); for (int i = 0; i < count; i++) { KeyValue pair = accessor.apply(i); - String key = pair.key().asString(); + String key = pair.key(); if (key.startsWith(filterPrefix)) { key = key.substring(filterPrefix.length()); - String oldValue = map.put(key, pair.value().asString()); + String oldValue = map.put(key, pair.value()); assert oldValue == null : key + " had " + oldValue + ", replaced with " + pair.value(); } } return map; } - /** - * Iterator wrapper that allows peeking at the next item, if any. - */ - private static class Iter implements Iterator { - private final Iterator wrapped; - private T next; - - private Iter(Iterator wrapped) { - this.wrapped = wrapped; - } - - public T peek() { - if (next != null) { - return next; - } - return next = next(); - } - - @Override - public boolean hasNext() { - return next != null || wrapped.hasNext(); - } - - @Override - public T next() { - if (next == null) { - return wrapped.next(); - } - T val = next; - next = null; - return val; - } - } - - public static Uint8Array makeUint8ArrayFromBitset(BitSet bitset) { - int length = (bitset.previousSetBit(Integer.MAX_VALUE - 1) + 8) / 8; - Uint8Array array = new Uint8Array(length); - byte[] bytes = bitset.toByteArray(); - for (int i = 0; i < bytes.length; i++) { - array.setAt(i, (double) bytes[i]); - } - - return array; - } - - public static Uint8Array serializeRanges(Set rangeSets) { + public static ByteBuffer serializeRanges(Set rangeSets) { final RangeSet s; - if (rangeSets.size() == 0) { - return new Uint8Array(0); + if (rangeSets.isEmpty()) { + return ByteBuffer.allocate(0); } else if (rangeSets.size() == 1) { s = rangeSets.iterator().next(); } else { s = new RangeSet(); for (RangeSet rangeSet : rangeSets) { - rangeSet.rangeIterator().forEachRemaining(s::addRange); - } - } - - ByteBuffer payload = CompressedRangeSetReader.writeRange(s); - ArrayBufferView buffer = TypedArrayHelper.unwrap(payload); - return new Uint8Array(buffer); - } - - public static ByteBuffer typedArrayToLittleEndianByteBuffer(Uint8Array data) { - ArrayBuffer slicedBuffer = data.slice().buffer; - ByteBuffer bb = TypedArrayHelper.wrap(slicedBuffer); - bb.order(ByteOrder.LITTLE_ENDIAN); - return bb; - } - - public static ByteBuffer typedArrayToLittleEndianByteBuffer(Int8Array data) { - ArrayBuffer slicedBuffer = data.slice().buffer; - ByteBuffer bb = TypedArrayHelper.wrap(slicedBuffer); - bb.order(ByteOrder.LITTLE_ENDIAN); - return bb; - } - - public static TableSnapshot createSnapshot(RecordBatch header, ByteBuffer body, BarrageUpdateMetadata barrageUpdate, - boolean isViewport, String[] columnTypes) { - RangeSet added; - - final RangeSet includedAdditions; - if (barrageUpdate == null) { - includedAdditions = added = RangeSet.ofRange(0, (long) (header.length().toFloat64() - 1)); - } else { - added = new CompressedRangeSetReader() - .read(typedArrayToLittleEndianByteBuffer(barrageUpdate.addedRowsArray())); - - Int8Array addedRowsIncluded = barrageUpdate.addedRowsIncludedArray(); - if (isViewport && addedRowsIncluded != null) { - includedAdditions = new CompressedRangeSetReader() - .read(typedArrayToLittleEndianByteBuffer(addedRowsIncluded)); - } else { - // if this isn't a viewport, then a second index isn't sent, because all rows are included - includedAdditions = added; - } - } - - // read the nodes and buffers into iterators so that we can descend into the data columns as necessary - Iter nodes = - new Iter<>(IntStream.range(0, (int) header.nodesLength()).mapToObj(header::nodes).iterator()); - Iter buffers = - new Iter<>(IntStream.range(0, (int) header.buffersLength()).mapToObj(header::buffers).iterator()); - ColumnData[] columnData = new ColumnData[columnTypes.length]; - for (int columnIndex = 0; columnIndex < columnTypes.length; ++columnIndex) { - columnData[columnIndex] = - readArrowBuffer(body, nodes, buffers, (int) includedAdditions.size(), columnTypes[columnIndex]); - } - - return new TableSnapshot(includedAdditions, columnData, added.size()); - } - - public static DeltaUpdatesBuilder deltaUpdates(BarrageUpdateMetadata barrageUpdate, boolean isViewport, - String[] columnTypes) { - return new DeltaUpdatesBuilder(barrageUpdate, isViewport, columnTypes); - } - - public static class DeltaUpdatesBuilder { - private final DeltaUpdates deltaUpdates = new DeltaUpdates(); - private final BarrageUpdateMetadata barrageUpdate; - private final String[] columnTypes; - private long numAddRowsRemaining = 0; - private long numModRowsRemaining = 0; - - public DeltaUpdatesBuilder(BarrageUpdateMetadata barrageUpdate, boolean isViewport, String[] columnTypes) { - this.barrageUpdate = barrageUpdate; - this.columnTypes = columnTypes; - - deltaUpdates.setAdded(new CompressedRangeSetReader() - .read(typedArrayToLittleEndianByteBuffer(barrageUpdate.addedRowsArray()))); - deltaUpdates.setRemoved(new CompressedRangeSetReader() - .read(typedArrayToLittleEndianByteBuffer(barrageUpdate.removedRowsArray()))); - - deltaUpdates.setShiftedRanges( - new ShiftedRangeReader().read(typedArrayToLittleEndianByteBuffer(barrageUpdate.shiftDataArray()))); - - RangeSet includedAdditions; - - Int8Array addedRowsIncluded = barrageUpdate.addedRowsIncludedArray(); - if (isViewport && addedRowsIncluded != null) { - includedAdditions = new CompressedRangeSetReader() - .read(typedArrayToLittleEndianByteBuffer(addedRowsIncluded)); - } else { - // if this isn't a viewport, then a second index isn't sent, because all rows are included - includedAdditions = deltaUpdates.getAdded(); + s.addRangeSet(rangeSet); } - numAddRowsRemaining = includedAdditions.size(); - deltaUpdates.setIncludedAdditions(includedAdditions); - deltaUpdates.setSerializedAdditions(new DeltaUpdates.ColumnAdditions[0]); - deltaUpdates.setSerializedModifications(new DeltaUpdates.ColumnModifications[0]); - - for (int columnIndex = 0; columnIndex < columnTypes.length; ++columnIndex) { - BarrageModColumnMetadata columnMetadata = barrageUpdate.modColumnNodes(columnIndex); - RangeSet modifiedRows = new CompressedRangeSetReader() - .read(typedArrayToLittleEndianByteBuffer(columnMetadata.modifiedRowsArray())); - numModRowsRemaining = Math.max(numModRowsRemaining, modifiedRows.size()); - } - } - - /** - * Appends a new record batch and payload. Returns true if this was the final record batch that was expected. - */ - public boolean appendRecordBatch(RecordBatch recordBatch, ByteBuffer body) { - if (numAddRowsRemaining > 0) { - handleAddBatch(recordBatch, body); - } else if (numModRowsRemaining > 0) { - handleModBatch(recordBatch, body); - } - // return true when complete - return numAddRowsRemaining == 0 && numModRowsRemaining == 0; } - private void handleAddBatch(RecordBatch recordBatch, ByteBuffer body) { - Iter nodes = new Iter<>( - IntStream.range(0, (int) recordBatch.nodesLength()).mapToObj(recordBatch::nodes).iterator()); - Iter buffers = new Iter<>( - IntStream.range(0, (int) recordBatch.buffersLength()).mapToObj(recordBatch::buffers).iterator()); - - DeltaUpdates.ColumnAdditions[] addedColumnData = new DeltaUpdates.ColumnAdditions[columnTypes.length]; - for (int columnIndex = 0; columnIndex < columnTypes.length; ++columnIndex) { - assert nodes.hasNext() && buffers.hasNext(); - ColumnData columnData = readArrowBuffer(body, nodes, buffers, (int) nodes.peek().length().toFloat64(), - columnTypes[columnIndex]); - - addedColumnData[columnIndex] = new DeltaUpdates.ColumnAdditions(columnIndex, columnData); - } - deltaUpdates.setSerializedAdditions(addedColumnData); - numAddRowsRemaining -= (long) recordBatch.length().toFloat64(); - } - - private void handleModBatch(RecordBatch recordBatch, ByteBuffer body) { - Iter nodes = new Iter<>( - IntStream.range(0, (int) recordBatch.nodesLength()).mapToObj(recordBatch::nodes).iterator()); - Iter buffers = new Iter<>( - IntStream.range(0, (int) recordBatch.buffersLength()).mapToObj(recordBatch::buffers).iterator()); - - DeltaUpdates.ColumnModifications[] modifiedColumnData = - new DeltaUpdates.ColumnModifications[columnTypes.length]; - for (int columnIndex = 0; columnIndex < columnTypes.length; ++columnIndex) { - assert nodes.hasNext() && buffers.hasNext(); - - BarrageModColumnMetadata columnMetadata = barrageUpdate.modColumnNodes(columnIndex); - RangeSet modifiedRows = new CompressedRangeSetReader() - .read(typedArrayToLittleEndianByteBuffer(columnMetadata.modifiedRowsArray())); - - ColumnData columnData = readArrowBuffer(body, nodes, buffers, (int) nodes.peek().length().toFloat64(), - columnTypes[columnIndex]); - modifiedColumnData[columnIndex] = - new DeltaUpdates.ColumnModifications(columnIndex, modifiedRows, columnData); - } - deltaUpdates.setSerializedModifications(modifiedColumnData); - numModRowsRemaining -= (long) recordBatch.length().toFloat64(); - } - - public DeltaUpdates build() { - return deltaUpdates; - } + return CompressedRangeSetReader.writeRange(s); } - - private static ColumnData readArrowBuffer(ByteBuffer data, Iter nodes, Iter buffers, int size, - String columnType) { - // explicit cast to be clear that we're rounding down - BitSet valid = readValidityBufferAsBitset(data, size, buffers.next()); - FieldNode thisNode = nodes.next(); - boolean hasNulls = thisNode.nullCount().toFloat64() != 0; - size = Math.min(size, (int) thisNode.length().toFloat64()); - - Buffer positions = buffers.next(); - switch (columnType) { - // for simple well-supported typedarray types, wrap and return - case "int": - assert positions.length().toFloat64() >= size * 4; - Int32Array intArray = new Int32Array(TypedArrayHelper.unwrap(data).buffer, - (int) positions.offset().toFloat64(), size); - return new IntArrayColumnData(Js.uncheckedCast(intArray)); - case "short": - assert positions.length().toFloat64() >= size * 2; - Int16Array shortArray = new Int16Array(TypedArrayHelper.unwrap(data).buffer, - (int) positions.offset().toFloat64(), size); - return new ShortArrayColumnData(Js.uncheckedCast(shortArray)); - case "boolean": - case "java.lang.Boolean": - // noinspection IntegerDivisionInFloatingPointContext - assert positions.length().toFloat64() >= ((size + 63) / 64); - // booleans are stored as a bitset, but internally we represent booleans as bytes - data.position((int) positions.offset().toFloat64()); - BitSet wireValues = readBitSetWithLength(data, (int) (positions.length().toFloat64())); - Boolean[] boolArray = new Boolean[size]; - for (int i = 0; i < size; ++i) { - if (!hasNulls || valid.get(i)) { - boolArray[i] = wireValues.get(i); - } else { - boolArray[i] = null; - } - } - return new BooleanArrayColumnData(boolArray); - case "byte": - assert positions.length().toFloat64() >= size; - Int8Array byteArray = - new Int8Array(TypedArrayHelper.unwrap(data).buffer, (int) positions.offset().toFloat64(), size); - return new ByteArrayColumnData(Js.uncheckedCast(byteArray)); - case "double": - assert positions.length().toFloat64() >= size * 8; - Float64Array doubleArray = new Float64Array(TypedArrayHelper.unwrap(data).buffer, - (int) positions.offset().toFloat64(), size); - return new DoubleArrayColumnData(Js.uncheckedCast(doubleArray)); - case "float": - assert positions.length().toFloat64() >= size * 4; - Float32Array floatArray = new Float32Array(TypedArrayHelper.unwrap(data).buffer, - (int) positions.offset().toFloat64(), size); - return new FloatArrayColumnData(Js.uncheckedCast(floatArray)); - case "char": - assert positions.length().toFloat64() >= size * 2; - Uint16Array charArray = new Uint16Array(TypedArrayHelper.unwrap(data).buffer, - (int) positions.offset().toFloat64(), size); - return new CharArrayColumnData(Js.uncheckedCast(charArray)); - // longs are a special case despite being java primitives - case "long": - case "java.time.Instant": - case "java.time.ZonedDateTime": - assert positions.length().toFloat64() >= size * 8; - long[] longArray = new long[size]; - - data.position((int) positions.offset().toFloat64()); - for (int i = 0; i < size; i++) { - longArray[i] = data.getLong(); - } - return new LongArrayColumnData(longArray); - // all other types are read out in some custom way - case "java.time.LocalTime":// LocalDateArrayColumnData - assert positions.length().toFloat64() >= size * 6; - data.position((int) positions.offset().toFloat64()); - LocalDate[] localDateArray = new LocalDate[size]; - for (int i = 0; i < size; i++) { - int year = data.getInt(); - byte month = data.get(); - byte day = data.get(); - localDateArray[i] = new LocalDate(year, month, day); - } - return new LocalDateArrayColumnData(localDateArray); - case "java.time.LocalDate":// LocalTimeArrayColumnData - assert positions.length().toFloat64() == size * 7; - LocalTime[] localTimeArray = new LocalTime[size]; - - data.position((int) positions.offset().toFloat64()); - for (int i = 0; i < size; i++) { - int nano = data.getInt(); - byte hour = data.get(); - byte minute = data.get(); - byte second = data.get(); - data.position(data.position() + 1);// aligned for next read - localTimeArray[i] = new LocalTime(hour, minute, second, nano); - } - return new LocalTimeArrayColumnData(localTimeArray); - default: - // remaining types have an offset buffer to read first - IntBuffer offsets = readOffsets(data, size, positions); - - if (columnType.endsWith("[]")) { - FieldNode arrayNode = nodes.next(); - int innerSize = (int) arrayNode.length().toFloat64(); - boolean innerHasNulls = arrayNode.nullCount().toFloat64() != 0; - - // array type, also read the inner valid buffer and inner offset buffer - BitSet innerValid = readValidityBufferAsBitset(data, innerSize, buffers.next()); - IntBuffer innerOffsets = readOffsets(data, innerSize, buffers.next()); - - Buffer payload = buffers.next(); - - switch (columnType) { - case "java.lang.String[]": - String[][] strArrArr = new String[size][]; - - for (int i = 0; i < size; i++) { - if (hasNulls && !valid.get(i)) { - strArrArr[i] = null; - continue; - } - int arrayStart = offsets.get(i); - int instanceSize = offsets.get(i + 1) - arrayStart; - String[] strArr = new String[instanceSize]; - for (int j = 0; j < instanceSize; j++) { - int inner = j + arrayStart; - assert innerOffsets != null; - if (innerHasNulls && !innerValid.get(inner)) { - assert innerOffsets.get(inner) == innerOffsets.get(inner + 1) - : innerOffsets.get(inner) + " == " + innerOffsets.get(inner + 1); - strArr[j] = null; - continue; - } - // might be cheaper to do views on the underlying bb (which will be copied anyway - // into the String) - data.position((int) (payload.offset().toFloat64()) + innerOffsets.get(inner)); - int stringSize = innerOffsets.get(inner + 1) - innerOffsets.get(inner); - byte[] stringBytes = new byte[stringSize]; - data.get(stringBytes); - strArr[j] = new String(stringBytes, StandardCharsets.UTF_8); - } - strArrArr[i] = strArr; - } - - return new StringArrayArrayColumnData(strArrArr); - default: - throw new IllegalStateException("Can't decode column of type " + columnType); - } - - } else { - // non-array, variable length stuff, just grab the buffer and read ranges specified by offsets - Buffer payload = buffers.next(); - - switch (columnType) { - case "java.lang.String": { - String[] stringArray = new String[size]; - byte[] buf = new byte[32]; - for (int i = 0; i < size; i++) { - if (hasNulls && !valid.get(i)) { - stringArray[i] = null; - continue; - } - int ioff = offsets.get(i); - int len = offsets.get(i + 1) - ioff; - data.position((int) (payload.offset().toFloat64()) + ioff); - if (buf.length < len) { - buf = new byte[len]; - } - data.get(buf, 0, len); - stringArray[i] = new String(buf, 0, len, StandardCharsets.UTF_8);// new - // String(Js.uncheckedCast(stringBytes)); - } - return new StringArrayColumnData(stringArray); - } - case "java.math.BigDecimal": { - BigDecimal[] bigDecArray = new BigDecimal[size]; - byte[] buf = null; - for (int i = 0; i < size; i++) { - if (hasNulls && !valid.get(i)) { - bigDecArray[i] = null; - continue; - } - int ioff = offsets.get(i); - int len = offsets.get(i + 1) - ioff; - data.position((int) (payload.offset().toFloat64()) + ioff); - int scale = data.getInt(); - len -= 4; - if (buf == null || buf.length != len) { - buf = new byte[len]; - } - bigDecArray[i] = new BigDecimal(readBigInt(data, buf), scale); - } - return new BigDecimalArrayColumnData(bigDecArray); - } - case "java.math.BigInteger": { - BigInteger[] bigIntArray = new BigInteger[size]; - byte[] buf = null; - for (int i = 0; i < size; i++) { - if (hasNulls && !valid.get(i)) { - bigIntArray[i] = null; - continue; - } - int ioff = offsets.get(i); - int len = offsets.get(i + 1) - ioff; - if (buf == null || buf.length != len) { - buf = new byte[len]; - } - data.position((int) (payload.offset().toFloat64()) + ioff); - bigIntArray[i] = readBigInt(data, buf); - } - return new BigIntegerArrayColumnData(bigIntArray); - } - default: - throw new IllegalStateException("Can't decode column of type " + columnType); - } - } - } - } - - private static BigInteger readBigInt(ByteBuffer data, byte[] buf) { - // TODO: Change to the code below when the Java 9 BigInteger(byte[], int, int) constructor is available. - // https://github.com/deephaven/deephaven-core/issues/1626 - // Make the call take an additional len parameter, and make the calling logic reallocate only when - // there is a need to grow, instead of the current need for an exact match. - // - // data.get(buf, 0, len); - // return new BigInteger(buf, 0, len); - data.get(buf); - return new BigInteger(buf); - } - - private static BitSet readValidityBufferAsBitset(ByteBuffer data, int size, Buffer buffer) { - if (size == 0 || buffer.length().toFloat64() == 0) { - // these buffers are optional (and empty) if the column is empty, or if it has primitives and we've allowed - // DH nulls - return new BitSet(0); - } - data.position((int) buffer.offset().toFloat64()); - BitSet valid = readBitSetWithLength(data, (int) (buffer.length().toFloat64())); - return valid; - } - - private static BitSet readBitSetWithLength(ByteBuffer data, int lenInBytes) { - byte[] array = new byte[lenInBytes]; - data.get(array); - - return BitSet.valueOf(array); - } - - private static IntBuffer readOffsets(ByteBuffer data, int size, Buffer buffer) { - if (size == 0) { - IntBuffer emptyOffsets = IntBuffer.allocate(1); - return emptyOffsets; - } - data.position((int) buffer.offset().toFloat64()); - IntBuffer offsets = data.slice().asIntBuffer(); - offsets.limit(size + 1); - return offsets; - } - } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebChunkReaderFactory.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebChunkReaderFactory.java new file mode 100644 index 00000000000..475adf7d686 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebChunkReaderFactory.java @@ -0,0 +1,372 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.web.client.api.barrage; + +import elemental2.core.JsDate; +import io.deephaven.base.verify.Assert; +import io.deephaven.chunk.WritableByteChunk; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.WritableIntChunk; +import io.deephaven.chunk.WritableLongChunk; +import io.deephaven.chunk.WritableObjectChunk; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.extensions.barrage.chunk.BooleanChunkReader; +import io.deephaven.extensions.barrage.chunk.ByteChunkReader; +import io.deephaven.extensions.barrage.chunk.CharChunkReader; +import io.deephaven.extensions.barrage.chunk.ChunkInputStreamGenerator; +import io.deephaven.extensions.barrage.chunk.ChunkReader; +import io.deephaven.extensions.barrage.chunk.DoubleChunkReader; +import io.deephaven.extensions.barrage.chunk.FloatChunkReader; +import io.deephaven.extensions.barrage.chunk.IntChunkReader; +import io.deephaven.extensions.barrage.chunk.LongChunkReader; +import io.deephaven.extensions.barrage.chunk.ShortChunkReader; +import io.deephaven.extensions.barrage.chunk.VarListChunkReader; +import io.deephaven.extensions.barrage.util.StreamReaderOptions; +import io.deephaven.util.BooleanUtils; +import io.deephaven.util.QueryConstants; +import io.deephaven.util.datastructures.LongSizedDataStructure; +import io.deephaven.web.client.api.BigDecimalWrapper; +import io.deephaven.web.client.api.BigIntegerWrapper; +import io.deephaven.web.client.api.DateWrapper; +import io.deephaven.web.client.api.LocalDateWrapper; +import io.deephaven.web.client.api.LocalTimeWrapper; +import io.deephaven.web.client.api.LongWrapper; +import org.apache.arrow.flatbuf.Date; +import org.apache.arrow.flatbuf.DateUnit; +import org.apache.arrow.flatbuf.FloatingPoint; +import org.apache.arrow.flatbuf.Int; +import org.apache.arrow.flatbuf.Precision; +import org.apache.arrow.flatbuf.Time; +import org.apache.arrow.flatbuf.TimeUnit; +import org.apache.arrow.flatbuf.Timestamp; +import org.apache.arrow.flatbuf.Type; + +import java.io.DataInput; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Iterator; +import java.util.PrimitiveIterator; + +/** + * Browser-compatible implementation of the ChunkReaderFactory, with a focus on reading from arrow types rather than + * successfully round-tripping to the Java server. + *

+ * Includes some specific workarounds to handle nullability that will make more sense for the browser. + */ +public class WebChunkReaderFactory implements ChunkReader.Factory { + @Override + public ChunkReader getReader(StreamReaderOptions options, int factor, ChunkReader.TypeInfo typeInfo) { + switch (typeInfo.arrowField().typeType()) { + case Type.Int: { + Int t = new Int(); + typeInfo.arrowField().type(t); + switch (t.bitWidth()) { + case 8: { + return new ByteChunkReader(options); + } + case 16: { + if (t.isSigned()) { + return new ShortChunkReader(options); + } + return new CharChunkReader(options); + } + case 32: { + return new IntChunkReader(options); + } + case 64: { + if (t.isSigned()) { + return new LongChunkReader(options).transform(LongWrapper::of); + } + throw new IllegalArgumentException("Unsigned 64bit integers not supported"); + } + default: + throw new IllegalArgumentException("Unsupported Int bitwidth: " + t.bitWidth()); + } + } + case Type.FloatingPoint: { + FloatingPoint t = new FloatingPoint(); + typeInfo.arrowField().type(t); + switch (t.precision()) { + case Precision.SINGLE: { + return new FloatChunkReader(options); + } + case Precision.DOUBLE: { + return new DoubleChunkReader(options); + } + default: + throw new IllegalArgumentException( + "Unsupported FloatingPoint precision " + Precision.name(t.precision())); + } + } + case Type.Binary: { + if (typeInfo.type() == BigIntegerWrapper.class) { + return (fieldNodeIter, bufferInfoIter, is, outChunk, outOffset, + totalRows) -> extractChunkFromInputStream( + is, + fieldNodeIter, + bufferInfoIter, + (val, off, len) -> new BigIntegerWrapper(new BigInteger(val, off, len)), + outChunk, outOffset, totalRows); + } + if (typeInfo.type() == BigDecimalWrapper.class) { + return (fieldNodeIter, bufferInfoIter, is, outChunk, outOffset, + totalRows) -> extractChunkFromInputStream( + is, + fieldNodeIter, + bufferInfoIter, + (final byte[] buf, final int offset, final int length) -> { + // read the int scale value as little endian, arrow's endianness. + final byte b1 = buf[offset]; + final byte b2 = buf[offset + 1]; + final byte b3 = buf[offset + 2]; + final byte b4 = buf[offset + 3]; + final int scale = b4 << 24 | (b3 & 0xFF) << 16 | (b2 & 0xFF) << 8 | (b1 & 0xFF); + BigDecimal bigDecimal = + new BigDecimal(new BigInteger(buf, offset + 4, length - 4), scale); + return new BigDecimalWrapper(bigDecimal); + }, + outChunk, outOffset, totalRows); + } + throw new IllegalArgumentException("Unsupported Binary type " + typeInfo.type()); + } + case Type.Utf8: { + return (fieldNodeIter, bufferInfoIter, is, outChunk, outOffset, + totalRows) -> extractChunkFromInputStream(is, fieldNodeIter, + bufferInfoIter, (buf, off, len) -> new String(buf, off, len, StandardCharsets.UTF_8), + outChunk, outOffset, totalRows); + } + case Type.Bool: { + BooleanChunkReader subReader = new BooleanChunkReader(); + return (fieldNodeIter, bufferInfoIter, is, outChunk, outOffset, totalRows) -> { + try (final WritableByteChunk inner = (WritableByteChunk) subReader.readChunk( + fieldNodeIter, bufferInfoIter, is, null, 0, 0)) { + + final WritableObjectChunk chunk; + if (outChunk != null) { + chunk = outChunk.asWritableObjectChunk(); + } else { + int numRows = Math.max(totalRows, inner.size()); + chunk = WritableObjectChunk.makeWritableChunk(numRows); + chunk.setSize(numRows); + } + + if (outChunk == null) { + // if we're not given an output chunk then we better be writing at the front of the new one + Assert.eqZero(outOffset, "outOffset"); + } + + for (int ii = 0; ii < inner.size(); ++ii) { + byte value = inner.get(ii); + chunk.set(outOffset + ii, BooleanUtils.byteAsBoolean(value)); + } + + return chunk; + } + + }; + } + case Type.Date: { + Date t = new Date(); + typeInfo.arrowField().type(t); + switch (t.unit()) { + case DateUnit.MILLISECOND: + return new LongChunkReader(options).transform(millis -> { + if (millis == QueryConstants.NULL_LONG) { + return null; + } + JsDate jsDate = new JsDate((double) (long) millis); + return new LocalDateWrapper(jsDate.getUTCFullYear(), 1 + jsDate.getUTCMonth(), + jsDate.getUTCDate()); + }); + case DateUnit.DAY: + return new IntChunkReader(options).transform(days -> { + if (days == QueryConstants.NULL_INT) { + return null; + } + JsDate jsDate = new JsDate(((double) (int) days) * 86400000); + return new LocalDateWrapper(jsDate.getUTCFullYear(), 1 + jsDate.getUTCMonth(), + jsDate.getUTCDate()); + }); + default: + throw new IllegalArgumentException("Unsupported Date unit: " + DateUnit.name(t.unit())); + } + } + case Type.Time: { + Time t = new Time(); + typeInfo.arrowField().type(t); + switch (t.bitWidth()) { + case 32: { + switch (t.unit()) { + case TimeUnit.SECOND: { + return new IntChunkReader(options) + .transform(LocalTimeWrapper.intCreator(1)::apply); + } + case TimeUnit.MILLISECOND: { + return new IntChunkReader(options) + .transform(LocalTimeWrapper.intCreator(1_000)::apply); + } + default: + throw new IllegalArgumentException("Unsupported Time unit: " + TimeUnit.name(t.unit())); + } + } + case 64: { + switch (t.unit()) { + case TimeUnit.NANOSECOND: { + return new LongChunkReader(options) + .transform(LocalTimeWrapper.longCreator(1_000_000_000)::apply); + } + case TimeUnit.MICROSECOND: { + return new LongChunkReader(options) + .transform(LocalTimeWrapper.longCreator(1_000_000)::apply); + } + default: + throw new IllegalArgumentException("Unsupported Time unit: " + TimeUnit.name(t.unit())); + } + } + default: + throw new IllegalArgumentException("Unsupported Time bitWidth: " + t.bitWidth()); + } + } + case Type.Timestamp: { + Timestamp t = new Timestamp(); + typeInfo.arrowField().type(t); + switch (t.unit()) { + case TimeUnit.NANOSECOND: { + if (!t.timezone().equals("UTC")) { + throw new IllegalArgumentException("Unsupported tz " + t.timezone()); + } + return new LongChunkReader(options).transform(DateWrapper::of); + } + default: + throw new IllegalArgumentException("Unsupported Timestamp unit: " + TimeUnit.name(t.unit())); + } + } + case Type.List: { + if (typeInfo.componentType() == byte.class) { + return (fieldNodeIter, bufferInfoIter, is, outChunk, outOffset, + totalRows) -> extractChunkFromInputStream( + is, + fieldNodeIter, + bufferInfoIter, + (buf, off, len) -> Arrays.copyOfRange(buf, off, off + len), + outChunk, outOffset, totalRows); + } + return new VarListChunkReader<>(options, typeInfo, this); + } + default: + throw new IllegalArgumentException("Unsupported type: " + Type.name(typeInfo.arrowField().typeType())); + } + } + + public interface Mapper { + T constructFrom(byte[] buf, int offset, int length) throws IOException; + } + + public static WritableObjectChunk extractChunkFromInputStream( + final DataInput is, + final Iterator fieldNodeIter, + final PrimitiveIterator.OfLong bufferInfoIter, + final Mapper mapper, + final WritableChunk outChunk, + final int outOffset, + final int totalRows) throws IOException { + final ChunkInputStreamGenerator.FieldNodeInfo nodeInfo = fieldNodeIter.next(); + final long validityBuffer = bufferInfoIter.nextLong(); + final long offsetsBuffer = bufferInfoIter.nextLong(); + final long payloadBuffer = bufferInfoIter.nextLong(); + + final int numElements = nodeInfo.numElements; + final WritableObjectChunk chunk; + if (outChunk != null) { + chunk = outChunk.asWritableObjectChunk(); + } else { + final int numRows = Math.max(totalRows, numElements); + chunk = WritableObjectChunk.makeWritableChunk(numRows); + chunk.setSize(numRows); + } + + if (numElements == 0) { + return chunk; + } + + final int numValidityWords = (numElements + 63) / 64; + try (final WritableLongChunk isValid = WritableLongChunk.makeWritableChunk(numValidityWords); + final WritableIntChunk offsets = WritableIntChunk.makeWritableChunk(numElements + 1)) { + // Read validity buffer: + int jj = 0; + for (; jj < Math.min(numValidityWords, validityBuffer / 8); ++jj) { + isValid.set(jj, is.readLong()); + } + final long valBufRead = jj * 8L; + if (valBufRead < validityBuffer) { + is.skipBytes(LongSizedDataStructure.intSize("VBCISG", validityBuffer - valBufRead)); + } + // we support short validity buffers + for (; jj < numValidityWords; ++jj) { + isValid.set(jj, -1); // -1 is bit-wise representation of all ones + } + + // Read offsets: + final long offBufRead = (numElements + 1L) * Integer.BYTES; + if (offsetsBuffer < offBufRead) { + throw new IllegalStateException("offset buffer is too short for the expected number of elements"); + } + for (int i = 0; i < numElements + 1; ++i) { + offsets.set(i, is.readInt()); + } + if (offBufRead < offsetsBuffer) { + is.skipBytes(LongSizedDataStructure.intSize("VBCISG", offsetsBuffer - offBufRead)); + } + + // Read data: + final int bytesRead = LongSizedDataStructure.intSize("VBCISG", payloadBuffer); + final byte[] serializedData = new byte[bytesRead]; + is.readFully(serializedData); + + // Deserialize: + int ei = 0; + int pendingSkips = 0; + + for (int vi = 0; vi < numValidityWords; ++vi) { + int bitsLeftInThisWord = Math.min(64, numElements - vi * 64); + long validityWord = isValid.get(vi); + do { + if ((validityWord & 1) == 1) { + if (pendingSkips > 0) { + chunk.fillWithNullValue(outOffset + ei, pendingSkips); + ei += pendingSkips; + pendingSkips = 0; + } + final int offset = offsets.get(ei); + final int length = offsets.get(ei + 1) - offset; + Assert.geq(length, "length", 0); + if (offset + length > serializedData.length) { + throw new IllegalStateException("not enough data was serialized to parse this element: " + + "elementIndex=" + ei + " offset=" + offset + " length=" + length + + " serializedLen=" + serializedData.length); + } + chunk.set(outOffset + ei++, mapper.constructFrom(serializedData, offset, length)); + validityWord >>= 1; + bitsLeftInThisWord--; + } else { + final int skips = Math.min(Long.numberOfTrailingZeros(validityWord), bitsLeftInThisWord); + pendingSkips += skips; + validityWord >>= skips; + bitsLeftInThisWord -= skips; + } + } while (bitsLeftInThisWord > 0); + } + + if (pendingSkips > 0) { + chunk.fillWithNullValue(outOffset + ei, pendingSkips); + } + } + + return chunk; + } + +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java new file mode 100644 index 00000000000..a46d8ec7f90 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java @@ -0,0 +1,515 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.web.client.api.barrage.data; + +import io.deephaven.chunk.Chunk; +import io.deephaven.chunk.ChunkType; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.web.client.api.barrage.WebBarrageMessage; +import io.deephaven.web.client.api.barrage.def.InitialTableDefinition; +import io.deephaven.web.client.state.ClientTableState; +import io.deephaven.web.shared.data.Range; +import io.deephaven.web.shared.data.RangeSet; +import io.deephaven.web.shared.data.ShiftedRange; +import jsinterop.base.Any; + +import java.util.Arrays; +import java.util.BitSet; +import java.util.NavigableSet; +import java.util.NoSuchElementException; +import java.util.PrimitiveIterator; +import java.util.TreeMap; + +/** + * In contrast to the server implementation, the JS API holds the "table" as distinct from the "subscription", so that + * developers are acutely aware of extra async costs in requesting data, and can clearly indicate how much data is + * requested. This class represents a barrage subscription for the JS API, and exposes access to the data presently + * available on the client. + *

+ * This is a rough analog to {@link io.deephaven.extensions.barrage.table.BarrageTable} and its subtypes, but isn't + * directly exposed to API consumers. Instead, the subscription types wrap this, and delegate their data storage and + * snapshot/delta handling here. + */ +public abstract class WebBarrageSubscription { + + public static final boolean COLUMNS_AS_LIST = false; + public static final int MAX_MESSAGE_SIZE = 10_000_000; + public static final int BATCH_SIZE = 100_000; + + public static WebBarrageSubscription subscribe(ClientTableState cts, ViewportChangedHandler viewportChangedHandler, + DataChangedHandler dataChangedHandler) { + + WebColumnData[] dataSinks = new WebColumnData[cts.columnTypes().length]; + ChunkType[] chunkTypes = cts.chunkTypes(); + for (int i = 0; i < dataSinks.length; i++) { + switch (chunkTypes[i]) { + case Boolean: + throw new IllegalStateException("Boolean unsupported here"); + case Char: + dataSinks[i] = new WebCharColumnData(); + break; + case Byte: + dataSinks[i] = new WebByteColumnData(); + break; + case Short: + dataSinks[i] = new WebShortColumnData(); + break; + case Int: + dataSinks[i] = new WebIntColumnData(); + break; + case Long: + dataSinks[i] = new WebLongColumnData(); + break; + case Float: + dataSinks[i] = new WebFloatColumnData(); + break; + case Double: + dataSinks[i] = new WebDoubleColumnData(); + break; + case Object: + dataSinks[i] = new WebObjectColumnData(); + break; + } + } + + if (cts.getTableDef().getAttributes().isBlinkTable()) { + return new BlinkImpl(cts, viewportChangedHandler, dataChangedHandler, dataSinks); + } + return new RedirectedImpl(cts, viewportChangedHandler, dataChangedHandler, dataSinks); + } + + public interface ViewportChangedHandler { + void onServerViewportChanged(RangeSet serverViewport, BitSet serverColumns, boolean serverReverseViewport); + } + public interface DataChangedHandler { + void onDataChanged(RangeSet rowsAdded, RangeSet rowsRemoved, RangeSet totalMods, ShiftedRange[] shifted, + BitSet modifiedColumnSet); + } + + protected final ClientTableState state; + protected final ViewportChangedHandler viewportChangedHandler; + protected final DataChangedHandler dataChangedHandler; + protected final RangeSet currentRowSet = RangeSet.empty(); + + protected long capacity = 0; + protected WebColumnData[] destSources; + + protected RangeSet serverViewport; + protected BitSet serverColumns; + protected boolean serverReverseViewport; + + protected WebBarrageSubscription(ClientTableState state, ViewportChangedHandler viewportChangedHandler, + DataChangedHandler dataChangedHandler, WebColumnData[] dataSinks) { + this.state = state; + destSources = dataSinks; + this.viewportChangedHandler = viewportChangedHandler; + this.dataChangedHandler = dataChangedHandler; + } + + public abstract void applyUpdates(WebBarrageMessage message); + + protected void updateServerViewport(RangeSet viewport, BitSet columns, boolean reverseViewport) { + serverViewport = viewport; + serverColumns = columns == null || columns.cardinality() == numColumns() ? null : columns; + serverReverseViewport = reverseViewport; + } + + protected int numColumns() { + return getDefinition().getColumns().length; + } + + private InitialTableDefinition getDefinition() { + return state.getTableDef(); + } + + public RangeSet getCurrentRowSet() { + return currentRowSet; + } + + public RangeSet getServerViewport() { + return serverViewport; + } + + public boolean isReversed() { + return serverReverseViewport; + } + + /** + * Reads a value from the table subscription. + * + * @param key the row to read in key-space + * @param col the index of the column to read + * @return the value read from the table + */ + public abstract Any getData(long key, int col); + + protected boolean isSubscribedColumn(int ii) { + return serverColumns == null || serverColumns.get(ii); + } + + public static class BlinkImpl extends WebBarrageSubscription { + enum Mode { + BLINK, APPEND + } + + private final Mode mode; + + public BlinkImpl(ClientTableState state, ViewportChangedHandler viewportChangedHandler, + DataChangedHandler dataChangedHandler, WebColumnData[] dataSinks) { + super(state, viewportChangedHandler, dataChangedHandler, dataSinks); + mode = Mode.BLINK; + } + + @Override + public void applyUpdates(WebBarrageMessage message) { + if (message.isSnapshot) { + updateServerViewport(message.snapshotRowSet, message.snapshotColumns, message.snapshotRowSetIsReversed); + viewportChangedHandler.onServerViewportChanged(serverViewport, serverColumns, serverReverseViewport); + } + + assert message.shifted.length == 0; + for (int i = 0; i < message.modColumnData.length; i++) { + assert message.modColumnData[i].rowsModified.isEmpty(); + } + + if (message.rowsIncluded.isEmpty()) { + return; + } + + long addedRows = message.rowsIncluded.size(); + RangeSet destinationRowSet; + if (mode == Mode.APPEND) { + destinationRowSet = RangeSet.ofRange(capacity, capacity + addedRows - 1); + capacity += addedRows; + } else { + destinationRowSet = RangeSet.ofRange(0, addedRows - 1); + capacity = addedRows; + } + Arrays.stream(destSources).forEach(s -> s.ensureCapacity(capacity)); + for (int ii = 0; ii < message.addColumnData.length; ii++) { + if (isSubscribedColumn(ii)) { + WebBarrageMessage.AddColumnData column = message.addColumnData[ii]; + PrimitiveIterator.OfLong destIterator = destinationRowSet.indexIterator(); + for (int j = 0; j < column.data.size(); j++) { + Chunk chunk = column.data.get(j); + destSources[ii].fillChunk(chunk, destIterator); + } + assert !destIterator.hasNext(); + } + } + + currentRowSet.clear(); + + currentRowSet.addRangeSet(message.rowsAdded); + state.setSize(message.rowsAdded.size()); + dataChangedHandler.onDataChanged(message.rowsAdded, message.rowsRemoved, RangeSet.empty(), message.shifted, + new BitSet(0)); + } + + @Override + public Any getData(long key, int col) { + if (!isSubscribedColumn(col)) { + throw new NoSuchElementException("No column at index " + col); + } + return destSources[col].get(key); + } + } + + public static class RedirectedImpl extends WebBarrageSubscription { + private RangeSet freeset = new RangeSet(); + private final TreeMap redirectedIndexes = new TreeMap<>(); + + public RedirectedImpl(ClientTableState state, ViewportChangedHandler viewportChangedHandler, + DataChangedHandler dataChangedHandler, WebColumnData[] dataSinks) { + super(state, viewportChangedHandler, dataChangedHandler, dataSinks); + } + + @Override + public void applyUpdates(WebBarrageMessage message) { + RangeSet populatedRows = serverViewport != null + ? currentRowSet.subsetForPositions(serverViewport, serverReverseViewport) + : null; + + if (message.isSnapshot) { + updateServerViewport(message.snapshotRowSet, message.snapshotColumns, message.snapshotRowSetIsReversed); + viewportChangedHandler.onServerViewportChanged(serverViewport, serverColumns, serverReverseViewport); + } + + final boolean mightBeInitialSnapshot = getCurrentRowSet().isEmpty() && message.isSnapshot; + + // Apply removes to our local rowset + currentRowSet.removeRangeSet(message.rowsRemoved); + + RangeSet removed = message.rowsRemoved; + if (populatedRows != null) { + // limit the removed rows to what intersect the viewport + removed = populatedRows.extract(message.rowsRemoved); + } + // free rows that are no longer needed + freeRows(removed); + + // Apply shifts + + // Shift moved rows in the redir index + boolean hasReverseShift = false; + final ShiftedRange[] shiftedRanges = message.shifted; + currentRowSet.applyShifts(shiftedRanges); + RangeSetBulkHelper populatedRowsetShifter = populatedRows == null ? null + : new RangeSetBulkHelper(populatedRows, RangeSetBulkHelper.Operation.APPEND); + for (int i = shiftedRanges.length - 1; i >= 0; --i) { + final ShiftedRange shiftedRange = shiftedRanges[i]; + final long offset = shiftedRange.getDelta(); + if (offset < 0) { + hasReverseShift = true; + continue; + } + + // test if shift is in populatedRows before continuing + if (populatedRows != null) { + if (!populatedRows.includesAnyOf(shiftedRange.getRange())) { + // no rows were included, we can skip updating populatedRows and redirectedIndexes + continue; + } + populatedRows.removeRange(shiftedRange.getRange()); + } + final NavigableSet toMove = redirectedIndexes.navigableKeySet() + .subSet(shiftedRange.getRange().getFirst(), true, shiftedRange.getRange().getLast(), true); + // iterate backward and move them forward + for (Long key : toMove.descendingSet()) { + long shiftedKey = key + offset; + Long oldValue = redirectedIndexes.put(shiftedKey, redirectedIndexes.remove(key)); + assert oldValue == null : shiftedKey + " already has a value, " + oldValue; + if (populatedRowsetShifter != null) { + populatedRowsetShifter.append(shiftedKey); + } + } + } + + if (hasReverseShift) { + for (int i = 0; i < shiftedRanges.length; ++i) { + final ShiftedRange shiftedRange = shiftedRanges[i]; + final long offset = shiftedRange.getDelta(); + if (offset > 0) { + continue; + } + + if (populatedRows != null) { + if (!populatedRows.includesAnyOf(shiftedRange.getRange())) { + // no rows were included, we can skip updating populatedRows and redirectedIndexes + continue; + } + populatedRows.removeRange(shiftedRange.getRange()); + } + final NavigableSet toMove = redirectedIndexes.navigableKeySet() + .subSet(shiftedRange.getRange().getFirst(), true, shiftedRange.getRange().getLast(), true); + // iterate forward and move them backward + for (Long key : toMove) { + long shiftedKey = key + offset; + Long oldValue = redirectedIndexes.put(shiftedKey, redirectedIndexes.remove(key)); + assert oldValue == null : shiftedKey + " already has a value, " + oldValue; + if (populatedRowsetShifter != null) { + populatedRowsetShifter.append(shiftedKey); + } + } + } + } + if (populatedRowsetShifter != null) { + populatedRowsetShifter.flush(); + } + + currentRowSet.addRangeSet(message.rowsAdded); + + RangeSet totalMods = new RangeSet(); + for (int i = 0; i < message.modColumnData.length; i++) { + WebBarrageMessage.ModColumnData column = message.modColumnData[i]; + totalMods.addRangeSet(column.rowsModified); + } + + if (!message.rowsIncluded.isEmpty()) { + if (mightBeInitialSnapshot) { + capacity = message.rowsIncluded.size(); + Arrays.stream(destSources).forEach(s -> s.ensureCapacity(capacity)); + freeset.addRange(new Range(0, capacity - 1)); + } + + RangeSet destinationRowSet = getFreeRows(message.rowsIncluded.size()); + + for (int ii = 0; ii < message.addColumnData.length; ii++) { + if (isSubscribedColumn(ii)) { + WebBarrageMessage.AddColumnData column = message.addColumnData[ii]; + PrimitiveIterator.OfLong destIterator = destinationRowSet.indexIterator(); + + for (int j = 0; j < column.data.size(); j++) { + Chunk chunk = column.data.get(j); + destSources[ii].fillChunk(chunk, destIterator); + } + assert !destIterator.hasNext(); + } + } + // Add redirection mappings + PrimitiveIterator.OfLong srcIter = message.rowsIncluded.indexIterator(); + PrimitiveIterator.OfLong destIter = destinationRowSet.indexIterator(); + while (srcIter.hasNext()) { + assert destIter.hasNext(); + redirectedIndexes.put(srcIter.next(), destIter.next()); + } + assert !destIter.hasNext(); + } + + BitSet modifiedColumnSet = new BitSet(numColumns()); + for (int ii = 0; ii < message.modColumnData.length; ii++) { + WebBarrageMessage.ModColumnData column = message.modColumnData[ii]; + if (column.rowsModified.isEmpty()) { + continue; + } + + modifiedColumnSet.set(ii); + + PrimitiveIterator.OfLong destIterator = column.rowsModified.indexIterator(); + for (int j = 0; j < column.data.size(); j++) { + Chunk chunk = column.data.get(j); + destSources[ii].fillChunk(chunk, destIterator); + } + assert !destIterator.hasNext(); + } + if (serverViewport != null && populatedRows != null) { + RangeSet newPopulated = currentRowSet.subsetForPositions(serverViewport, serverReverseViewport); + populatedRows.removeRangeSet(newPopulated); + freeRows(populatedRows); + } + + state.setSize(currentRowSet.size()); + dataChangedHandler.onDataChanged(message.rowsAdded, removed, totalMods, message.shifted, + modifiedColumnSet); + } + + @Override + public Any getData(long key, int col) { + if (!isSubscribedColumn(col)) { + throw new NoSuchElementException("No column at index " + col); + } + return this.destSources[col].get(redirectedIndexes.get(key)); + } + + private RangeSet getFreeRows(long size) { + if (size <= 0) { + return RangeSet.empty(); + } + boolean needsResizing = false; + final RangeSet result; + if (capacity == 0) { + capacity = Long.highestOneBit(Math.max(size * 2, 8)); + freeset.addRange(new Range(0, capacity - 1)); + needsResizing = true; + } else { + if (freeset.size() < size) { + long usedSlots = capacity - freeset.size(); + long prevCapacity = capacity; + + do { + capacity *= 2; + } while ((capacity - usedSlots) < size); + freeset.addRange(new Range(prevCapacity, capacity - 1)); + needsResizing = true; + } + } + if (needsResizing) { + Arrays.stream(destSources).forEach(s -> s.ensureCapacity(capacity)); + } + result = freeset.subsetForPositions(RangeSet.ofRange(0, size - 1), false); + freeset.removeRange(new Range(0, result.getLastRow())); + assert result.size() == size : result.size() + " == " + size; + return result; + } + + private void freeRows(RangeSet removed) { + RangeSetBulkHelper reusableHelper = new RangeSetBulkHelper(freeset, RangeSetBulkHelper.Operation.APPEND); + removed.indexIterator().forEachRemaining((long index) -> { + Long dest = redirectedIndexes.remove(index); + if (dest != null) { + reusableHelper.append(dest); + } + }); + reusableHelper.flush(); + } + } + + /** + * Helper to avoid appending many times when modifying indexes. The append() method should be called for each key + * in order to ensure that addRange/removeRange isn't called excessively. When no more items will be added, + * flush() must be called. + */ + private static class RangeSetBulkHelper { + enum Operation { + APPEND, REMOVE + } + + private final RangeSet rangeSet; + private final Operation operation; + + private long currentFirst = -1; + private long currentLast; + + public RangeSetBulkHelper(final RangeSet rangeSet, Operation operation) { + this.rangeSet = rangeSet; + this.operation = operation; + } + + public void append(long key) { + assert key >= 0; + + if (currentFirst == -1) { + // first key to be added, move both first and last + currentFirst = key; + currentLast = key; + } else if (key == currentLast + 1) { + // key appends to our current range + currentLast = key; + } else if (key == currentFirst - 1) { + // key appends to our current range + currentFirst = key; + } else { + // existing range doesn't match the new item, finish the old range and start a new one + if (operation == Operation.APPEND) { + rangeSet.addRange(new Range(currentFirst, currentLast)); + } else { + rangeSet.removeRange(new Range(currentFirst, currentLast)); + } + currentFirst = key; + currentLast = key; + } + } + + public void appendRange(Range range) { + if (currentFirst == -1) { + currentFirst = range.getFirst(); + currentLast = range.getLast(); + } else if (range.getFirst() == currentLast + 1) { + currentLast = range.getLast(); + } else if (range.getLast() == currentFirst - 1) { + currentFirst = range.getFirst(); + } else { + if (operation == Operation.APPEND) { + rangeSet.addRange(new Range(currentFirst, currentLast)); + } else { + rangeSet.removeRange(new Range(currentFirst, currentLast)); + } + currentFirst = range.getFirst(); + currentLast = range.getLast(); + } + } + + public void flush() { + if (currentFirst != -1) { + if (operation == Operation.APPEND) { + rangeSet.addRange(new Range(currentFirst, currentLast)); + } else { + rangeSet.removeRange(new Range(currentFirst, currentLast)); + } + currentFirst = -1; + } + } + } + +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebByteColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebByteColumnData.java new file mode 100644 index 00000000000..4f0593d05a9 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebByteColumnData.java @@ -0,0 +1,27 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit WebCharColumnData and run "./gradlew replicateBarrageUtils" to regenerate +// +// @formatter:off +package io.deephaven.web.client.api.barrage.data; + +import io.deephaven.chunk.ByteChunk; +import io.deephaven.chunk.Chunk; +import io.deephaven.util.QueryConstants; +import jsinterop.base.Js; + +import java.util.PrimitiveIterator; + +public class WebByteColumnData extends WebColumnData { + @Override + public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { + ByteChunk byteChunk = data.asByteChunk(); + int i = 0; + while (destIterator.hasNext()) { + byte value = byteChunk.get(i++); + arr.setAt((int) destIterator.nextLong(), value == QueryConstants.NULL_BYTE ? null : Js.asAny(value)); + } + } +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebCharColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebCharColumnData.java new file mode 100644 index 00000000000..dfe561d90df --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebCharColumnData.java @@ -0,0 +1,23 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.web.client.api.barrage.data; + +import io.deephaven.chunk.CharChunk; +import io.deephaven.chunk.Chunk; +import io.deephaven.util.QueryConstants; +import jsinterop.base.Js; + +import java.util.PrimitiveIterator; + +public class WebCharColumnData extends WebColumnData { + @Override + public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { + CharChunk charChunk = data.asCharChunk(); + int i = 0; + while (destIterator.hasNext()) { + char value = charChunk.get(i++); + arr.setAt((int) destIterator.nextLong(), value == QueryConstants.NULL_CHAR ? null : Js.asAny(value)); + } + } +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebColumnData.java new file mode 100644 index 00000000000..3c8fee323b4 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebColumnData.java @@ -0,0 +1,27 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.web.client.api.barrage.data; + +import elemental2.core.JsArray; +import io.deephaven.chunk.Chunk; +import jsinterop.base.Any; + +import java.util.PrimitiveIterator; + +/** + * Holds data from or intended for web clients, normalizing over nulls, with helpers to handle typed chunks. + */ +public abstract class WebColumnData { + protected final JsArray arr = new JsArray<>(); + + public abstract void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator); + + public void ensureCapacity(long size) { + // Current impl does nothing, js arrays don't behave better when told the size up front + } + + public Any get(long position) { + return arr.getAt((int) position); + } +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebDoubleColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebDoubleColumnData.java new file mode 100644 index 00000000000..d2c8e764ce7 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebDoubleColumnData.java @@ -0,0 +1,27 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit WebCharColumnData and run "./gradlew replicateBarrageUtils" to regenerate +// +// @formatter:off +package io.deephaven.web.client.api.barrage.data; + +import io.deephaven.chunk.DoubleChunk; +import io.deephaven.chunk.Chunk; +import io.deephaven.util.QueryConstants; +import jsinterop.base.Js; + +import java.util.PrimitiveIterator; + +public class WebDoubleColumnData extends WebColumnData { + @Override + public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { + DoubleChunk doubleChunk = data.asDoubleChunk(); + int i = 0; + while (destIterator.hasNext()) { + double value = doubleChunk.get(i++); + arr.setAt((int) destIterator.nextLong(), value == QueryConstants.NULL_DOUBLE ? null : Js.asAny(value)); + } + } +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebFloatColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebFloatColumnData.java new file mode 100644 index 00000000000..a624affbda6 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebFloatColumnData.java @@ -0,0 +1,27 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit WebCharColumnData and run "./gradlew replicateBarrageUtils" to regenerate +// +// @formatter:off +package io.deephaven.web.client.api.barrage.data; + +import io.deephaven.chunk.FloatChunk; +import io.deephaven.chunk.Chunk; +import io.deephaven.util.QueryConstants; +import jsinterop.base.Js; + +import java.util.PrimitiveIterator; + +public class WebFloatColumnData extends WebColumnData { + @Override + public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { + FloatChunk floatChunk = data.asFloatChunk(); + int i = 0; + while (destIterator.hasNext()) { + float value = floatChunk.get(i++); + arr.setAt((int) destIterator.nextLong(), value == QueryConstants.NULL_FLOAT ? null : Js.asAny(value)); + } + } +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebIntColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebIntColumnData.java new file mode 100644 index 00000000000..996cf43c6a8 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebIntColumnData.java @@ -0,0 +1,27 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit WebCharColumnData and run "./gradlew replicateBarrageUtils" to regenerate +// +// @formatter:off +package io.deephaven.web.client.api.barrage.data; + +import io.deephaven.chunk.IntChunk; +import io.deephaven.chunk.Chunk; +import io.deephaven.util.QueryConstants; +import jsinterop.base.Js; + +import java.util.PrimitiveIterator; + +public class WebIntColumnData extends WebColumnData { + @Override + public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { + IntChunk intChunk = data.asIntChunk(); + int i = 0; + while (destIterator.hasNext()) { + int value = intChunk.get(i++); + arr.setAt((int) destIterator.nextLong(), value == QueryConstants.NULL_INT ? null : Js.asAny(value)); + } + } +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebLongColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebLongColumnData.java new file mode 100644 index 00000000000..080c05e6034 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebLongColumnData.java @@ -0,0 +1,27 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit WebCharColumnData and run "./gradlew replicateBarrageUtils" to regenerate +// +// @formatter:off +package io.deephaven.web.client.api.barrage.data; + +import io.deephaven.chunk.LongChunk; +import io.deephaven.chunk.Chunk; +import io.deephaven.util.QueryConstants; +import jsinterop.base.Js; + +import java.util.PrimitiveIterator; + +public class WebLongColumnData extends WebColumnData { + @Override + public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { + LongChunk longChunk = data.asLongChunk(); + int i = 0; + while (destIterator.hasNext()) { + long value = longChunk.get(i++); + arr.setAt((int) destIterator.nextLong(), value == QueryConstants.NULL_LONG ? null : Js.asAny(value)); + } + } +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebObjectColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebObjectColumnData.java new file mode 100644 index 00000000000..251bca22e67 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebObjectColumnData.java @@ -0,0 +1,22 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.web.client.api.barrage.data; + +import io.deephaven.chunk.Chunk; +import io.deephaven.chunk.ObjectChunk; +import jsinterop.base.Js; + +import java.util.PrimitiveIterator; + +public class WebObjectColumnData extends WebColumnData { + @Override + public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { + ObjectChunk objectChunk = data.asObjectChunk(); + int i = 0; + while (destIterator.hasNext()) { + Object value = objectChunk.get(i++); + arr.setAt((int) destIterator.nextLong(), Js.asAny(value)); + } + } +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebShortColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebShortColumnData.java new file mode 100644 index 00000000000..328a0f654a4 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebShortColumnData.java @@ -0,0 +1,27 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit WebCharColumnData and run "./gradlew replicateBarrageUtils" to regenerate +// +// @formatter:off +package io.deephaven.web.client.api.barrage.data; + +import io.deephaven.chunk.ShortChunk; +import io.deephaven.chunk.Chunk; +import io.deephaven.util.QueryConstants; +import jsinterop.base.Js; + +import java.util.PrimitiveIterator; + +public class WebShortColumnData extends WebColumnData { + @Override + public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { + ShortChunk shortChunk = data.asShortChunk(); + int i = 0; + while (destIterator.hasNext()) { + short value = shortChunk.get(i++); + arr.setAt((int) destIterator.nextLong(), value == QueryConstants.NULL_SHORT ? null : Js.asAny(value)); + } + } +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/def/ColumnDefinition.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/def/ColumnDefinition.java index 604f80fcd77..377c09d840d 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/def/ColumnDefinition.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/def/ColumnDefinition.java @@ -4,108 +4,101 @@ package io.deephaven.web.client.api.barrage.def; import io.deephaven.web.client.api.Column; +import org.apache.arrow.flatbuf.Field; import java.util.Map; +import static io.deephaven.web.client.api.barrage.WebBarrageUtils.keyValuePairs; + public class ColumnDefinition { - private int columnIndex; - private String name; - private String type; - - private boolean isSortable; - - private String styleColumn; - private String formatColumn; - - private boolean isStyleColumn; - private boolean isFormatColumn; - private boolean isNumberFormatColumn; - private boolean isPartitionColumn; - private boolean isHierarchicalExpandByColumn; - private boolean isHierarchicalRowDepthColumn; - private boolean isHierarchicalRowExpandedColumn; - private boolean isRollupAggregatedNodeColumn; - private boolean isRollupConstituentNodeColumn; - private boolean isRollupGroupByColumn; - private String rollupAggregationInputColumn; - - // Indicates that this is a style column for the row - private boolean forRow; - private boolean isInputTableKeyColumn; - private String description; + private final Field field; + private final int columnIndex; + private final String type; + + private final boolean isSortable; + + private final String styleColumn; + private final String formatColumn; + + private final boolean isStyleColumn; + private final boolean isFormatColumn; + private final boolean isPartitionColumn; + private final boolean isHierarchicalExpandByColumn; + private final boolean isHierarchicalRowDepthColumn; + private final boolean isHierarchicalRowExpandedColumn; + private final boolean isRollupAggregatedNodeColumn; + private final boolean isRollupConstituentNodeColumn; + private final boolean isRollupGroupByColumn; + private final String rollupAggregationInputColumn; + + // Indicates that this is a style column for the whole row + private final boolean forRow; + private final boolean isInputTableKeyColumn; + private final String description; + + public ColumnDefinition(int index, Field field) { + Map fieldMetadata = + keyValuePairs("deephaven:", field.customMetadataLength(), field::customMetadata); + this.field = field; + columnIndex = index; + type = fieldMetadata.get("type"); + isSortable = "true".equals(fieldMetadata.get("isSortable")); + isStyleColumn = "true".equals(fieldMetadata.get("isStyle")); + isFormatColumn = "true".equals(fieldMetadata.get("isDateFormat")) + || "true".equals(fieldMetadata.get("isNumberFormat")); + forRow = "true".equals(fieldMetadata.get("isRowStyle")); + + String formatColumnName = fieldMetadata.get("dateFormatColumn"); + if (formatColumnName == null) { + formatColumnName = fieldMetadata.get("numberFormatColumn"); + } + formatColumn = formatColumnName; - public String getName() { - return name; - } + styleColumn = fieldMetadata.get("styleColumn"); - public int getColumnIndex() { - return columnIndex; + isInputTableKeyColumn = "true".equals(fieldMetadata.get("inputtable.isKey")); + + this.description = fieldMetadata.get("description"); + + isPartitionColumn = "true".equals(fieldMetadata.get("isPartitioning")); + + isHierarchicalExpandByColumn = "true".equals(fieldMetadata.get("hierarchicalTable.isExpandByColumn")); + isHierarchicalRowDepthColumn = "true".equals(fieldMetadata.get("hierarchicalTable.isRowDepthColumn")); + isHierarchicalRowExpandedColumn = "true".equals(fieldMetadata.get("hierarchicalTable.isRowExpandedColumn")); + isRollupAggregatedNodeColumn = "true".equals(fieldMetadata.get("rollupTable.isAggregatedNodeColumn")); + isRollupConstituentNodeColumn = "true".equals(fieldMetadata.get("rollupTable.isConstituentNodeColumn")); + isRollupGroupByColumn = "true".equals(fieldMetadata.get("rollupTable.isGroupByColumn")); + rollupAggregationInputColumn = fieldMetadata.get("rollupTable.aggregationInputColumnName"); } - public void setColumnIndex(int columnIndex) { - this.columnIndex = columnIndex; + public String getName() { + return field.name(); } - public void setName(String name) { - this.name = name; + public int getColumnIndex() { + return columnIndex; } public String getType() { return type; } - public void setType(String type) { - this.type = type; - } - public boolean isSortable() { return isSortable; } - public void setIsSortable(boolean sortable) { - isSortable = sortable; - } - public boolean isStyleColumn() { return isStyleColumn; } - public void setStyleColumn(boolean styleColumn) { - isStyleColumn = styleColumn; - } - public boolean isFormatColumn() { return isFormatColumn; } - public void setFormatColumn(boolean formatColumn) { - isFormatColumn = formatColumn; - } - - /** - * @deprecated Use {@link #isFormatColumn()} - */ - @Deprecated - public boolean isNumberFormatColumn() { - return isNumberFormatColumn; - } - - /** - * @deprecated Use {@link #setFormatColumn(boolean)} - */ - @Deprecated - public void setNumberFormatColumn(boolean numberFormatColumn) { - isNumberFormatColumn = numberFormatColumn; - } - public boolean isPartitionColumn() { return isPartitionColumn; } - public void setPartitionColumn(boolean partitionColumn) { - isPartitionColumn = partitionColumn; - } - public boolean isVisible() { return !isStyleColumn() && !isFormatColumn() && !isRollupConstituentNodeColumn() && !isHierarchicalRowDepthColumn() && !isHierarchicalRowExpandedColumn(); @@ -115,38 +108,18 @@ public boolean isForRow() { return forRow; } - public void setForRow(boolean forRow) { - this.forRow = forRow; - } - public String getFormatColumnName() { return formatColumn; } - public void setFormatColumnName(String formatColumn) { - this.formatColumn = formatColumn; - } - public String getStyleColumnName() { return styleColumn; } - public void setStyleColumnName(String styleColumn) { - this.styleColumn = styleColumn; - } - - public void setInputTableKeyColumn(boolean inputTableKeyColumn) { - this.isInputTableKeyColumn = inputTableKeyColumn; - } - public boolean isInputTableKeyColumn() { return isInputTableKeyColumn; } - public void setDescription(String description) { - this.description = description; - } - public String getDescription() { return description; } @@ -161,10 +134,10 @@ public Column makeJsColumn(int index, Map return makeColumn(index, this, - format == null || !format.isNumberFormatColumn() ? null : format.getColumnIndex(), + style == null ? null : style.getColumnIndex(), style == null ? null : style.getColumnIndex(), isPartitionColumn(), - format == null || format.isNumberFormatColumn() ? null : format.getColumnIndex(), + format == null || format.isFormatColumn() ? null : format.getColumnIndex(), getDescription(), isInputTableKeyColumn()); } @@ -181,55 +154,28 @@ public boolean isHierarchicalExpandByColumn() { return isHierarchicalExpandByColumn; } - public void setHierarchicalExpandByColumn(boolean hierarchicalExpandByColumn) { - isHierarchicalExpandByColumn = hierarchicalExpandByColumn; - } - public boolean isHierarchicalRowDepthColumn() { return isHierarchicalRowDepthColumn; } - public void setHierarchicalRowDepthColumn(boolean hierarchicalRowDepthColumn) { - isHierarchicalRowDepthColumn = hierarchicalRowDepthColumn; - } - public boolean isHierarchicalRowExpandedColumn() { return isHierarchicalRowExpandedColumn; } - public void setHierarchicalRowExpandedColumn(boolean hierarchicalRowExpandedColumn) { - isHierarchicalRowExpandedColumn = hierarchicalRowExpandedColumn; - } - public boolean isRollupAggregatedNodeColumn() { return isRollupAggregatedNodeColumn; } - public void setRollupAggregatedNodeColumn(boolean rollupAggregatedNodeColumn) { - isRollupAggregatedNodeColumn = rollupAggregatedNodeColumn; - } - public boolean isRollupConstituentNodeColumn() { return isRollupConstituentNodeColumn; } - public void setRollupConstituentNodeColumn(boolean rollupConstituentNodeColumn) { - isRollupConstituentNodeColumn = rollupConstituentNodeColumn; - } - public boolean isRollupGroupByColumn() { return isRollupGroupByColumn; } - public void setRollupGroupByColumn(boolean rollupGroupByColumn) { - isRollupGroupByColumn = rollupGroupByColumn; - } - public String getRollupAggregationInputColumn() { return rollupAggregationInputColumn; } - public void setRollupAggregationInputColumn(String rollupAggregationInputColumn) { - this.rollupAggregationInputColumn = rollupAggregationInputColumn; - } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/stream/HandshakeStreamFactory.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/stream/HandshakeStreamFactory.java deleted file mode 100644 index bae2d3c41fd..00000000000 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/stream/HandshakeStreamFactory.java +++ /dev/null @@ -1,150 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.client.api.barrage.stream; - -import elemental2.core.Function; -import elemental2.core.JsArray; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.browserflight_pb_service.BrowserFlightService; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.browserflight_pb_service.ResponseStream; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.HandshakeRequest; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.HandshakeResponse; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb_service.BidirectionalStream; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb_service.FlightService; -import io.deephaven.javascript.proto.dhinternal.grpcweb.Grpc; -import io.deephaven.javascript.proto.dhinternal.grpcweb.client.ClientRpcOptions; -import io.deephaven.javascript.proto.dhinternal.grpcweb.grpc.Client; -import io.deephaven.javascript.proto.dhinternal.grpcweb.invoke.InvokeRpcOptions; -import io.deephaven.javascript.proto.dhinternal.grpcweb.invoke.Request; -import io.deephaven.web.client.api.WorkerConnection; -import jsinterop.base.Js; - -import java.util.HashMap; -import java.util.Map; - -/** - * Improbable-eng's grpc-web implementation doesn't pass headers to api callers, only trailers (which sometimes includes - * headers) are included when calls fail, but never when successful. The current Flight auth v2 setup requires reading - * headers from responses, but strictly speaking doesn't require reading them from all calls - we can make extra - * FlightService/Handshake calls as long as we can read the response headers with them. - *

- *

- * This class includes a custom implementation of the Handshake method that is able to notify callers about headers that - * are received. - */ -public class HandshakeStreamFactory { - - private static final String DATA_EVENT_LISTENER_NAME = "data"; - private static final String END_EVENT_LISTENER_NAME = "end"; - private static final String STATUS_EVENT_LISTENER_NAME = "status"; - private static final String HEADERS_EVENT_LISTENER_NAME = "headers"; - - public static BiDiStream create(WorkerConnection connection) { - return connection.streamFactory().create( - metadata -> { - Map> listeners = listenerMap(); - ClientRpcOptions options = ClientRpcOptions.create(); - options.setHost(connection.flightServiceClient().serviceHost); - options.setTransport(null);// ts doesn't expose these two, stick with defaults for now - options.setDebug(false); - Client client = Grpc.client(FlightService.Handshake, - (io.deephaven.javascript.proto.dhinternal.grpcweb.grpc.ClientRpcOptions) options); - client.onEnd((status, statusMessage, trailers) -> { - listeners.get(STATUS_EVENT_LISTENER_NAME).forEach((item, index) -> item.call(null, - ResponseStreamWrapper.Status.of(status, statusMessage, metadata))); - listeners.get(END_EVENT_LISTENER_NAME).forEach((item, index) -> item.call(null, - ResponseStreamWrapper.Status.of(status, statusMessage, metadata))); - listeners.clear(); - }); - client.onMessage(message -> { - listeners.get(DATA_EVENT_LISTENER_NAME).forEach((item, index) -> item.call(null, message)); - }); - client.onHeaders(headers -> { - listeners.get(HEADERS_EVENT_LISTENER_NAME) - .forEach((item, index) -> item.call(null, headers)); - }); - client.start(metadata); - - return new BidirectionalStream() { - @Override - public void cancel() { - listeners.clear(); - client.close(); - } - - @Override - public void end() { - client.finishSend(); - } - - @Override - public BidirectionalStream on(String type, - Function handler) { - listeners.get(type).push(handler); - return this; - } - - @Override - public BidirectionalStream write( - HandshakeRequest message) { - client.send(message); - return this; - } - }; - }, - (first, metadata) -> { - Map> listeners = listenerMap(); - io.deephaven.javascript.proto.dhinternal.grpcweb.grpc.InvokeRpcOptions props = - Js.cast(InvokeRpcOptions.create()); - props.setRequest(first); - props.setHost(connection.browserFlightServiceClient().serviceHost); - props.setMetadata(metadata); - props.setTransport(null);// ts doesnt expose these two, stick with defaults for now - props.setDebug(false); - props.setOnMessage(responseMessage -> { - listeners.get(DATA_EVENT_LISTENER_NAME) - .forEach((item, index) -> item.call(null, responseMessage)); - }); - props.setOnEnd((status, statusMessage, trailers) -> { - listeners.get(STATUS_EVENT_LISTENER_NAME).forEach( - (item, index) -> item.call(null, - ResponseStreamWrapper.Status.of(status, statusMessage, metadata))); - listeners.get(END_EVENT_LISTENER_NAME).forEach( - (item, index) -> item.call(null, - ResponseStreamWrapper.Status.of(status, statusMessage, metadata))); - listeners.clear(); - }); - props.setOnHeaders(headers -> { - listeners.get(HEADERS_EVENT_LISTENER_NAME) - .forEach((item, index) -> item.call(null, headers)); - }); - Request client = Grpc.invoke.onInvoke(BrowserFlightService.OpenHandshake, props); - - return new ResponseStream() { - @Override - public void cancel() { - listeners.clear(); - client.getClose().onInvoke(); - } - - @Override - public ResponseStream on(String type, Function handler) { - listeners.get(type).push(handler); - return this; - } - }; - }, - (next, headers, callback) -> connection.browserFlightServiceClient().nextHandshake(next, headers, - callback::apply), - new HandshakeRequest()); - } - - private static Map> listenerMap() { - Map> listeners = new HashMap<>(); - listeners.put(DATA_EVENT_LISTENER_NAME, new JsArray<>()); - listeners.put(END_EVENT_LISTENER_NAME, new JsArray<>()); - listeners.put(STATUS_EVENT_LISTENER_NAME, new JsArray<>()); - listeners.put(HEADERS_EVENT_LISTENER_NAME, new JsArray<>()); - return listeners; - } -} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/grpc/UnaryWithHeaders.java b/web/client-api/src/main/java/io/deephaven/web/client/api/grpc/UnaryWithHeaders.java new file mode 100644 index 00000000000..2b82a40d2b8 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/grpc/UnaryWithHeaders.java @@ -0,0 +1,44 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.web.client.api.grpc; + +import elemental2.promise.IThenable; +import elemental2.promise.Promise; +import io.deephaven.javascript.proto.dhinternal.grpcweb.Grpc; +import io.deephaven.javascript.proto.dhinternal.grpcweb.grpc.Code; +import io.deephaven.javascript.proto.dhinternal.grpcweb.unary.UnaryOutput; +import io.deephaven.javascript.proto.dhinternal.grpcweb.unary.UnaryRpcOptions; +import io.deephaven.web.client.api.WorkerConnection; + +public class UnaryWithHeaders { + + /** + * Improbable-eng's grpc-web implementation doesn't pass headers to api callers - this changes the contract a bit so + * that we can get a typed UnaryOutput with the headers/trailers intact. + * + * @param connection provides access to the metadata and the server url + * @param methodDescriptor the service method to invoke + * @return a promise that will resolve to the response plus headers/trailers, or reject with the headers/trailers + * @param type of the message object + */ + public static Promise> call(WorkerConnection connection, Object methodDescriptor, + Req request) { + return new Promise<>((resolve, reject) -> { + UnaryRpcOptions props = UnaryRpcOptions.create(); + props.setHost(connection.configServiceClient().serviceHost); + props.setMetadata(connection.metadata()); + props.setTransport(null);// ts doesn't expose these two, stick with defaults for now + props.setDebug(false); + props.setOnEnd(response -> { + if (response.getStatus() != Code.OK) { + reject.onInvoke(response); + } else { + resolve.onInvoke((IThenable>) response); + } + }); + props.setRequest(request); + Grpc.unary.onInvoke(methodDescriptor, props); + }); + } +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/parse/JsDataHandler.java b/web/client-api/src/main/java/io/deephaven/web/client/api/parse/JsDataHandler.java index d3edf0e03f4..4bb583cd86f 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/parse/JsDataHandler.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/parse/JsDataHandler.java @@ -3,6 +3,7 @@ // package io.deephaven.web.client.api.parse; +import com.google.flatbuffers.FlatBufferBuilder; import com.google.gwt.i18n.client.TimeZone; import elemental2.core.ArrayBuffer; import elemental2.core.Float32Array; @@ -14,14 +15,6 @@ import elemental2.core.TypedArray; import elemental2.core.Uint16Array; import elemental2.core.Uint8Array; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Binary; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.FixedSizeBinary; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.FloatingPoint; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Int; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Precision; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Type; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Utf8; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; import io.deephaven.web.client.api.LongWrapper; import io.deephaven.web.client.api.i18n.JsDateTimeFormat; import io.deephaven.web.client.api.i18n.JsTimeZone; @@ -29,6 +22,13 @@ import io.deephaven.web.shared.fu.JsFunction; import jsinterop.base.Js; import jsinterop.base.JsArrayLike; +import org.apache.arrow.flatbuf.Binary; +import org.apache.arrow.flatbuf.FixedSizeBinary; +import org.apache.arrow.flatbuf.FloatingPoint; +import org.apache.arrow.flatbuf.Int; +import org.apache.arrow.flatbuf.Precision; +import org.apache.arrow.flatbuf.Type; +import org.apache.arrow.flatbuf.Utf8; import org.gwtproject.nio.TypedArrayHelper; import java.nio.ByteBuffer; @@ -40,16 +40,16 @@ import java.util.List; import java.util.Map; -import static io.deephaven.web.client.api.subscription.QueryConstants.FALSE_BOOLEAN_AS_BYTE; -import static io.deephaven.web.client.api.subscription.QueryConstants.NULL_BOOLEAN_AS_BYTE; -import static io.deephaven.web.client.api.subscription.QueryConstants.NULL_BYTE; -import static io.deephaven.web.client.api.subscription.QueryConstants.NULL_CHAR; -import static io.deephaven.web.client.api.subscription.QueryConstants.NULL_DOUBLE; -import static io.deephaven.web.client.api.subscription.QueryConstants.NULL_FLOAT; -import static io.deephaven.web.client.api.subscription.QueryConstants.NULL_INT; -import static io.deephaven.web.client.api.subscription.QueryConstants.NULL_LONG; -import static io.deephaven.web.client.api.subscription.QueryConstants.NULL_SHORT; -import static io.deephaven.web.client.api.subscription.QueryConstants.TRUE_BOOLEAN_AS_BYTE; +import static io.deephaven.util.BooleanUtils.FALSE_BOOLEAN_AS_BYTE; +import static io.deephaven.util.BooleanUtils.NULL_BOOLEAN_AS_BYTE; +import static io.deephaven.util.BooleanUtils.TRUE_BOOLEAN_AS_BYTE; +import static io.deephaven.util.QueryConstants.NULL_BYTE; +import static io.deephaven.util.QueryConstants.NULL_CHAR; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; +import static io.deephaven.util.QueryConstants.NULL_FLOAT; +import static io.deephaven.util.QueryConstants.NULL_INT; +import static io.deephaven.util.QueryConstants.NULL_LONG; +import static io.deephaven.util.QueryConstants.NULL_SHORT; /** * Given the expected type of a column, pick one of the enum entries and use that to read the data into arrow buffers. @@ -101,8 +101,9 @@ public Uint8Array build() { } @Override - public double writeType(Builder builder) { - return Utf8.createUtf8(builder); + public int writeType(FlatBufferBuilder builder) { + Utf8.startUtf8(builder); + return Utf8.endUtf8(builder); } @Override @@ -201,7 +202,7 @@ private long parseDateString(String str, ParseContext context) { } @Override - public double writeType(Builder builder) { + public int writeType(FlatBufferBuilder builder) { return Int.createInt(builder, 64, true); } @@ -249,7 +250,7 @@ public void write(Object[] data, ParseContext context, JsConsumer addNode, }, INTEGER(Type.Int, "int") { @Override - public double writeType(Builder builder) { + public int writeType(FlatBufferBuilder builder) { return Int.createInt(builder, 32, true); } @@ -261,7 +262,7 @@ public void write(Object[] data, ParseContext context, JsConsumer addNode, }, SHORT(Type.Int, "short") { @Override - public double writeType(Builder builder) { + public int writeType(FlatBufferBuilder builder) { return Int.createInt(builder, 16, true); } @@ -273,7 +274,7 @@ public void write(Object[] data, ParseContext context, JsConsumer addNode, }, LONG(Type.Int, "long") { @Override - public double writeType(Builder builder) { + public int writeType(FlatBufferBuilder builder) { return Int.createInt(builder, 64, true); } @@ -319,7 +320,7 @@ public void write(Object[] data, ParseContext context, JsConsumer addNode, }, BYTE(Type.Int, "byte") { @Override - public double writeType(Builder builder) { + public int writeType(FlatBufferBuilder builder) { return Int.createInt(builder, 8, true); } @@ -331,7 +332,7 @@ public void write(Object[] data, ParseContext context, JsConsumer addNode, }, CHAR(Type.Int, "char") { @Override - public double writeType(Builder builder) { + public int writeType(FlatBufferBuilder builder) { return Int.createInt(builder, 16, false); } @@ -343,7 +344,7 @@ public void write(Object[] data, ParseContext context, JsConsumer addNode, }, FLOAT(Type.FloatingPoint, "float") { @Override - public double writeType(Builder builder) { + public int writeType(FlatBufferBuilder builder) { return FloatingPoint.createFloatingPoint(builder, Precision.SINGLE); } @@ -355,7 +356,7 @@ public void write(Object[] data, ParseContext context, JsConsumer addNode, }, DOUBLE(Type.FloatingPoint, "double") { @Override - public double writeType(Builder builder) { + public int writeType(FlatBufferBuilder builder) { return FloatingPoint.createFloatingPoint(builder, Precision.DOUBLE); } @@ -368,7 +369,7 @@ public void write(Object[] data, ParseContext context, JsConsumer addNode, }, BOOLEAN(Type.Bool, "boolean", "bool", "java.lang.Boolean") { @Override - public double writeType(Builder builder) { + public int writeType(FlatBufferBuilder builder) { return Int.createInt(builder, 8, true); } @@ -440,25 +441,27 @@ public void write(Object[] data, ParseContext context, JsConsumer addNode, }, BIG_DECIMAL(Type.Binary, "java.util.BigDecimal") { @Override - public double writeType(Builder builder) { - return Binary.createBinary(builder); + public int writeType(FlatBufferBuilder builder) { + Binary.startBinary(builder); + return Binary.endBinary(builder); } }, BIG_INTEGER(Type.Binary, "java.util.BigInteger") { @Override - public double writeType(Builder builder) { - return Binary.createBinary(builder); + public int writeType(FlatBufferBuilder builder) { + Binary.startBinary(builder); + return Binary.endBinary(builder); } }, LOCAL_DATE(Type.FixedSizeBinary, "java.time.LocalDate", "localdate") { @Override - public double writeType(Builder builder) { + public int writeType(FlatBufferBuilder builder) { return FixedSizeBinary.createFixedSizeBinary(builder, 6); } }, LOCAL_TIME(Type.FixedSizeBinary, "java.time.LocalTime", "localtime") { @Override - public double writeType(Builder builder) { + public int writeType(FlatBufferBuilder builder) { return FixedSizeBinary.createFixedSizeBinary(builder, 7); } }, @@ -540,10 +543,10 @@ private static class HandlersHolder { private static final int SEPARATOR_INDEX = DEFAULT_DATE_TIME_PATTERN.indexOf('T'); - private final int arrowTypeType; + private final byte arrowTypeType; private final String deephavenType; - JsDataHandler(int arrowTypeType, String... typeNames) { + JsDataHandler(byte arrowTypeType, String... typeNames) { this.arrowTypeType = arrowTypeType; assert typeNames.length > 0 : "Must have at least one name"; this.deephavenType = typeNames[0]; @@ -553,7 +556,7 @@ private static class HandlersHolder { } } - public int typeType() { + public byte typeType() { return arrowTypeType; } @@ -561,7 +564,7 @@ public String deephavenType() { return deephavenType; } - public abstract double writeType(Builder builder); + public abstract int writeType(FlatBufferBuilder builder); public void write(Object[] data, ParseContext context, JsConsumer addNode, JsConsumer addBuffer) { throw new UnsupportedOperationException("Can't parse " + name()); diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java new file mode 100644 index 00000000000..155dbb8271a --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java @@ -0,0 +1,524 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.web.client.api.subscription; + +import com.google.flatbuffers.FlatBufferBuilder; +import com.vertispan.tsdefs.annotations.TsIgnore; +import elemental2.core.JsArray; +import elemental2.dom.CustomEventInit; +import io.deephaven.barrage.flatbuf.BarrageMessageType; +import io.deephaven.barrage.flatbuf.BarrageSubscriptionRequest; +import io.deephaven.extensions.barrage.BarrageSubscriptionOptions; +import io.deephaven.extensions.barrage.ColumnConversionMode; +import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightData; +import io.deephaven.web.client.api.Column; +import io.deephaven.web.client.api.Format; +import io.deephaven.web.client.api.HasEventHandling; +import io.deephaven.web.client.api.JsRangeSet; +import io.deephaven.web.client.api.LongWrapper; +import io.deephaven.web.client.api.TableData; +import io.deephaven.web.client.api.WorkerConnection; +import io.deephaven.web.client.api.barrage.CompressedRangeSetReader; +import io.deephaven.web.client.api.barrage.WebBarrageMessage; +import io.deephaven.web.client.api.barrage.WebBarrageStreamReader; +import io.deephaven.web.client.api.barrage.WebBarrageUtils; +import io.deephaven.web.client.api.barrage.data.WebBarrageSubscription; +import io.deephaven.web.client.api.barrage.stream.BiDiStream; +import io.deephaven.web.client.api.barrage.stream.ResponseStreamWrapper; +import io.deephaven.web.client.fu.JsSettings; +import io.deephaven.web.client.state.ClientTableState; +import io.deephaven.web.shared.data.RangeSet; +import io.deephaven.web.shared.data.ShiftedRange; +import io.deephaven.web.shared.fu.JsRunnable; +import jsinterop.annotations.JsProperty; +import jsinterop.base.Any; +import jsinterop.base.Js; +import org.jetbrains.annotations.Nullable; + +import java.io.IOException; +import java.util.BitSet; + +/** + * Superclass of various subscription types, allowing specific implementations to customize behavior for their needs. + *

+ * Instances are not ready to use right away, owing to the fact that we need to wait both for the provided state to + * resolve (so that we have the table schema, know what kind of subscription we will make, and know what column types + * will be resolves), and because until the subscription has finished being set up, we will not have received the size + * of the table. When closed, it cannot be reused again. + *

+ * This is also a base class for types exposed to JS. + *

+ * This is a rough analog of the JVM's {@code BarrageSubscriptionImpl} class. In contrast to the JVM code, this is + * exposed to api consumers, rather than wrapping in a Table type, as it handles the barrage stream and provides events + * that client code can listen to. + */ +public abstract class AbstractTableSubscription extends HasEventHandling { + /** + * Indicates that some new data is available on the client, either an initial snapshot or a delta update. The + * detail field of the event will contain a TableSubscriptionEventData detailing what has changed, or + * allowing access to the entire range of items currently in the subscribed columns. + */ + public static final String EVENT_UPDATED = "updated"; + + public enum Status { + /** Waiting for some prerequisite before we can use it for the first time. */ + STARTING, + /** Successfully created, not waiting for any messages to be accurate. */ + ACTIVE, + /** Waiting for an update to return from being active to being active again. */ + PENDING_UPDATE, + /** Closed or otherwise stopped, cannot be used again. */ + DONE; + } + + private final ClientTableState state; + private final WorkerConnection connection; + protected final int rowStyleColumn; + private JsArray columns; + private BitSet columnBitSet; + protected RangeSet viewportRowSet; + private boolean isReverseViewport; + private BarrageSubscriptionOptions options; + + private BiDiStream doExchange; + protected WebBarrageSubscription barrageSubscription; + + protected Status status = Status.STARTING; + + private String failMsg; + + public AbstractTableSubscription(ClientTableState state, WorkerConnection connection) { + state.retain(this); + this.state = state; + this.connection = connection; + rowStyleColumn = state.getRowFormatColumn() == null ? TableData.NO_ROW_FORMAT_COLUMN + : state.getRowFormatColumn().getIndex(); + + revive(); + } + + /** + * Creates the connection to the server. Used on initial connection, and for viewport reconnects. + */ + protected void revive() { + // Once the state is running, set up the actual subscription + // Don't let subscription be used again, table failed and user will have already gotten an error elsewhere + state.onRunning(s -> { + if (status != Status.STARTING) { + // already closed + return; + } + WebBarrageSubscription.ViewportChangedHandler viewportChangedHandler = this::onViewportChange; + WebBarrageSubscription.DataChangedHandler dataChangedHandler = this::onDataChanged; + + status = Status.ACTIVE; + this.barrageSubscription = + WebBarrageSubscription.subscribe(state, viewportChangedHandler, dataChangedHandler); + + doExchange = + connection.streamFactory().create( + headers -> connection.flightServiceClient().doExchange(headers), + (first, headers) -> connection.browserFlightServiceClient().openDoExchange(first, headers), + (next, headers, c) -> connection.browserFlightServiceClient().nextDoExchange(next, headers, + c::apply), + new FlightData()); + + doExchange.onData(this::onFlightData); + doExchange.onEnd(this::onStreamEnd); + + sendFirstSubscriptionRequest(); + }, + // If the upstream table fails, kill the subscription + this::fail, + // If the upstream table is closed, its because this subscription released it, do nothing + JsRunnable.doNothing()); + } + + public Status getStatus() { + return status; + } + + protected static FlatBufferBuilder subscriptionRequest(byte[] tableTicket, BitSet columns, + @Nullable RangeSet viewport, + BarrageSubscriptionOptions options, boolean isReverseViewport) { + FlatBufferBuilder sub = new FlatBufferBuilder(1024); + int colOffset = BarrageSubscriptionRequest.createColumnsVector(sub, columns.toByteArray()); + int viewportOffset = 0; + if (viewport != null) { + viewportOffset = + BarrageSubscriptionRequest.createViewportVector(sub, CompressedRangeSetReader.writeRange(viewport)); + } + int optionsOffset = options.appendTo(sub); + int tableTicketOffset = BarrageSubscriptionRequest.createTicketVector(sub, tableTicket); + BarrageSubscriptionRequest.startBarrageSubscriptionRequest(sub); + BarrageSubscriptionRequest.addColumns(sub, colOffset); + BarrageSubscriptionRequest.addViewport(sub, viewportOffset); + BarrageSubscriptionRequest.addSubscriptionOptions(sub, optionsOffset); + BarrageSubscriptionRequest.addTicket(sub, tableTicketOffset); + BarrageSubscriptionRequest.addReverseViewport(sub, isReverseViewport); + sub.finish(BarrageSubscriptionRequest.endBarrageSubscriptionRequest(sub)); + + return sub; + } + + protected abstract void sendFirstSubscriptionRequest(); + + protected void sendBarrageSubscriptionRequest(RangeSet viewport, JsArray columns, Double updateIntervalMs, + boolean isReverseViewport) { + if (status == Status.DONE) { + if (failMsg == null) { + throw new IllegalStateException("Can't change subscription, already closed"); + } else { + throw new IllegalStateException("Can't change subscription, already failed: " + failMsg); + } + } + status = Status.PENDING_UPDATE; + this.columns = columns; + this.viewportRowSet = viewport; + this.columnBitSet = makeColumnBitset(columns); + this.isReverseViewport = isReverseViewport; + this.options = BarrageSubscriptionOptions.builder() + .batchSize(WebBarrageSubscription.BATCH_SIZE) + .maxMessageSize(WebBarrageSubscription.MAX_MESSAGE_SIZE) + .columnConversionMode(ColumnConversionMode.Stringify) + .minUpdateIntervalMs(updateIntervalMs == null ? 0 : (int) (double) updateIntervalMs) + .columnsAsList(false)// TODO(deephaven-core#5927) flip this to true + .useDeephavenNulls(true) + .build(); + FlatBufferBuilder request = subscriptionRequest( + Js.uncheckedCast(state.getHandle().getTicket()), + columnBitSet, + viewport, + options, + isReverseViewport); + FlightData subscriptionRequest = new FlightData(); + subscriptionRequest + .setAppMetadata(WebBarrageUtils.wrapMessage(request, BarrageMessageType.BarrageSubscriptionRequest)); + doExchange.send(subscriptionRequest); + } + + protected BitSet makeColumnBitset(JsArray columns) { + return state.makeBitset(Js.uncheckedCast(columns)); + } + + public ClientTableState state() { + return state; + } + + protected WorkerConnection connection() { + return connection; + } + + protected boolean isSubscriptionReady() { + return status == Status.ACTIVE; + } + + public double size() { + if (status == Status.ACTIVE) { + return barrageSubscription.getCurrentRowSet().size(); + } + if (status == Status.DONE) { + throw new IllegalStateException("Can't read size when already closed"); + } + return state.getSize(); + } + + private void onDataChanged(RangeSet rowsAdded, RangeSet rowsRemoved, RangeSet totalMods, ShiftedRange[] shifted, + BitSet modifiedColumnSet) { + if (!isSubscriptionReady()) { + return; + } + + notifyUpdate(rowsAdded, rowsRemoved, totalMods, shifted); + } + + protected void notifyUpdate(RangeSet rowsAdded, RangeSet rowsRemoved, RangeSet totalMods, ShiftedRange[] shifted) { + // TODO (deephaven-core#2435) Rewrite shifts as adds/removed/modifies + UpdateEventData detail = new SubscriptionEventData( + barrageSubscription, + rowStyleColumn, + columns, + rowsAdded, + rowsRemoved, + totalMods, + shifted); + CustomEventInit event = CustomEventInit.create(); + event.setDetail(detail); + fireEvent(TableSubscription.EVENT_UPDATED, event); + } + + public static class SubscriptionRow implements TableData.Row { + private final WebBarrageSubscription subscription; + private final int rowStyleColumn; + protected final long index; + public LongWrapper indexCached; + + public SubscriptionRow(WebBarrageSubscription subscription, int rowStyleColumn, long index) { + this.subscription = subscription; + this.rowStyleColumn = rowStyleColumn; + this.index = index; + } + + @Override + public LongWrapper getIndex() { + if (indexCached == null) { + indexCached = LongWrapper.of(index); + } + return indexCached; + } + + @Override + public Any get(Column column) { + return subscription.getData(index, column.getIndex()); + } + + @Override + public Format getFormat(Column column) { + long cellColors = 0; + long rowColors = 0; + String numberFormat = null; + String formatString = null; + if (column.getStyleColumnIndex() != null) { + LongWrapper wrapper = subscription.getData(index, column.getStyleColumnIndex()).uncheckedCast(); + cellColors = wrapper == null ? 0 : wrapper.getWrapped(); + } + if (rowStyleColumn != TableData.NO_ROW_FORMAT_COLUMN) { + LongWrapper wrapper = subscription.getData(index, rowStyleColumn).uncheckedCast(); + rowColors = wrapper == null ? 0 : wrapper.getWrapped(); + } + if (column.getFormatStringColumnIndex() != null) { + numberFormat = subscription.getData(index, column.getFormatStringColumnIndex()).uncheckedCast(); + } + if (column.getFormatStringColumnIndex() != null) { + formatString = subscription.getData(index, column.getFormatStringColumnIndex()).uncheckedCast(); + } + return new Format(cellColors, rowColors, numberFormat, formatString); + } + } + + /** + * TableData type for both viewports and full table subscriptions. + */ + @TsIgnore + public static class SubscriptionEventData extends UpdateEventData implements ViewportData, SubscriptionTableData { + public SubscriptionEventData(WebBarrageSubscription subscription, int rowStyleColumn, JsArray columns, + RangeSet added, RangeSet removed, RangeSet modified, ShiftedRange[] shifted) { + super(subscription, rowStyleColumn, columns, added, removed, modified, shifted); + } + + @Override + public JsRangeSet getAdded() { + return added; + } + + @Override + public JsRangeSet getRemoved() { + return removed; + } + + @Override + public JsRangeSet getModified() { + return modified; + } + + @Override + public JsRangeSet getFullIndex() { + return fullRowSet; + } + } + + /** + * Base type to allow trees to extend from here separately from tables. + */ + @TsIgnore + public abstract static class UpdateEventData implements TableData { + protected final WebBarrageSubscription subscription; + private final int rowStyleColumn; + private final JsArray columns; + protected final JsRangeSet added; + protected final JsRangeSet removed; + protected final JsRangeSet modified; + protected final JsRangeSet fullRowSet; + + // cached copy in case it was requested, could be requested again + private JsArray allRows; + + private double offset; + + public UpdateEventData(WebBarrageSubscription subscription, int rowStyleColumn, JsArray columns, + RangeSet added, RangeSet removed, RangeSet modified, ShiftedRange[] shifted) { + this.subscription = subscription; + this.rowStyleColumn = rowStyleColumn; + this.columns = columns; + this.added = new JsRangeSet(added); + this.removed = new JsRangeSet(removed); + this.modified = new JsRangeSet(modified); + this.fullRowSet = new JsRangeSet(transformRowsetForConsumer(subscription.getCurrentRowSet(), + subscription.getServerViewport(), subscription.isReversed())); + } + + // for ViewportData + @JsProperty + public Double getOffset() { + return offset; + } + + public void setOffset(double offset) { + this.offset = offset; + } + + @Override + public JsArray getRows() { + if (allRows == null) { + allRows = new JsArray<>(); + fullRowSet.getRange().indexIterator().forEachRemaining((long index) -> { + allRows.push(makeRow(index)); + }); + if (JsSettings.isDevMode()) { + assert allRows.length == fullRowSet.getSize(); + } + } + return (JsArray) (JsArray) allRows; + } + + protected SubscriptionRow makeRow(long index) { + return new SubscriptionRow(subscription, rowStyleColumn, index); + } + + @Override + public Row get(int index) { + return this.get((long) index); + } + + @Override + public Row get(long index) { + return makeRow(index); + } + + @Override + public Any getData(int index, Column column) { + return getData((long) index, column); + } + + @Override + public Any getData(long key, Column column) { + return subscription.getData(fullRowSet.getRange().get(key), column.getIndex()); + } + + @Override + public Format getFormat(int index, Column column) { + return getFormat((long) index, column); + } + + @Override + public Format getFormat(long index, Column column) { + long key = fullRowSet.getRange().get(index); + long cellColors = 0; + long rowColors = 0; + String numberFormat = null; + String formatString = null; + if (column.getStyleColumnIndex() != null) { + LongWrapper wrapper = subscription.getData(key, column.getStyleColumnIndex()).uncheckedCast(); + cellColors = wrapper == null ? 0 : wrapper.getWrapped(); + } + if (rowStyleColumn != NO_ROW_FORMAT_COLUMN) { + LongWrapper wrapper = subscription.getData(key, column.getStyleColumnIndex()).uncheckedCast(); + rowColors = wrapper == null ? 0 : wrapper.getWrapped(); + } + if (column.getFormatStringColumnIndex() != null) { + numberFormat = subscription.getData(key, column.getFormatStringColumnIndex()).uncheckedCast(); + } + if (column.getFormatStringColumnIndex() != null) { + formatString = subscription.getData(key, column.getFormatStringColumnIndex()).uncheckedCast(); + } + return new Format(cellColors, rowColors, numberFormat, formatString); + } + + @Override + public JsArray getColumns() { + return columns; + } + } + + /** + * If a viewport is in use, transforms the given rowset to position space based on that viewport. + * + * @param rowSet the rowset to possibly transform + * @return a transformed rowset + */ + private static RangeSet transformRowsetForConsumer(RangeSet rowSet, @Nullable RangeSet viewport, boolean reversed) { + if (viewport != null) { + return rowSet.subsetForPositions(viewport, reversed); + } + return rowSet; + } + + private void onViewportChange(RangeSet serverViewport, BitSet serverColumns, boolean serverReverseViewport) { + boolean subscriptionReady = ((serverColumns == null && columnBitSet == null) + || (serverColumns == null && columnBitSet.cardinality() == state.getTableDef().getColumns().length) + || (serverColumns != null && serverColumns.equals(this.columnBitSet))) + && (serverViewport == null && this.viewportRowSet == null + || (serverViewport != null && serverViewport.equals(this.viewportRowSet))) + && serverReverseViewport == isReverseViewport; + if (subscriptionReady) { + status = Status.ACTIVE; + } + } + + private final WebBarrageStreamReader reader = new WebBarrageStreamReader(); + + private void onFlightData(FlightData data) { + WebBarrageMessage message; + try { + message = reader.parseFrom(options, state.chunkTypes(), state.columnTypes(), state.componentTypes(), data); + } catch (IOException e) { + throw new RuntimeException(e); + } + if (message != null) { + // This payload resulted in an update to the table's contents, inform the subscription + barrageSubscription.applyUpdates(message); + } + } + + protected void onStreamEnd(ResponseStreamWrapper.Status status) { + if (this.status == Status.DONE) { + return; + } + if (status.isTransportError()) { + // If the subscription isn't closed and we hit a transport error, allow it to restart + this.status = Status.STARTING; + } else { + // Subscription failed somehow, fire an event + fail(status.getDetails()); + } + } + + private void fail(String message) { + failureHandled(message); + this.status = Status.DONE; + doExchange = null; + failMsg = message; + } + + /** + * The columns that were subscribed to when this subscription was created + * + * @return {@link Column} + */ + public JsArray getColumns() { + return columns; + } + + /** + * Stops the subscription on the server. + */ + public void close() { + state.unretain(this); + if (doExchange != null) { + doExchange.end(); + doExchange.cancel(); + } + status = Status.DONE; + } +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/QueryConstants.java b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/QueryConstants.java deleted file mode 100644 index e49eb7c2846..00000000000 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/QueryConstants.java +++ /dev/null @@ -1,21 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.client.api.subscription; - - -/** - * Constants for null values within the Deephaven engine From io.deephaven.util.QueryConstants - */ -public interface QueryConstants { - char NULL_CHAR = Character.MAX_VALUE; - byte NULL_BYTE = Byte.MIN_VALUE; - short NULL_SHORT = Short.MIN_VALUE; - int NULL_INT = Integer.MIN_VALUE; - long NULL_LONG = Long.MIN_VALUE; - float NULL_FLOAT = -Float.MAX_VALUE; - double NULL_DOUBLE = -Double.MAX_VALUE; - byte NULL_BOOLEAN_AS_BYTE = NULL_BYTE; - byte TRUE_BOOLEAN_AS_BYTE = (byte) 1; - byte FALSE_BOOLEAN_AS_BYTE = (byte) 0; -} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/SubscriptionTableData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/SubscriptionTableData.java index 7c08d330e91..2067e9068db 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/SubscriptionTableData.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/SubscriptionTableData.java @@ -5,672 +5,53 @@ import com.vertispan.tsdefs.annotations.TsInterface; import com.vertispan.tsdefs.annotations.TsName; -import elemental2.core.JsArray; -import elemental2.dom.CustomEventInit; -import io.deephaven.web.client.api.*; -import io.deephaven.web.client.fu.JsSettings; -import io.deephaven.web.shared.data.*; -import io.deephaven.web.shared.data.columns.ColumnData; -import jsinterop.annotations.JsFunction; -import jsinterop.annotations.JsIgnore; -import jsinterop.annotations.JsMethod; +import io.deephaven.web.client.api.JsRangeSet; +import io.deephaven.web.client.api.TableData; import jsinterop.annotations.JsProperty; -import jsinterop.annotations.JsType; -import jsinterop.base.Any; -import jsinterop.base.Js; -import jsinterop.base.JsArrayLike; -import javax.annotation.Nullable; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.NavigableSet; -import java.util.PrimitiveIterator; -import java.util.TreeMap; -import static io.deephaven.web.client.api.subscription.ViewportData.NO_ROW_FORMAT_COLUMN; - -public class SubscriptionTableData { - @JsFunction - private interface ArrayCopy { - @SuppressWarnings("unusable-by-js") - void copyTo(Object destArray, long destPos, Object srcArray, int srcPos); - } - - private final JsArray columns; - private final int rowStyleColumn; - private final HasEventHandling evented; - - // the actual rows present on the client, in their correct order - private RangeSet index; - - // mappings from the index to the position of a row in the data array - private TreeMap redirectedIndexes; - - // rows in the data columns that no longer contain data and can be reused - private RangeSet reusableDestinations; - - // array of data columns, cast each to a jsarray to read rows - private Object[] data; - - public SubscriptionTableData(JsArray columns, int rowStyleColumn, HasEventHandling evented) { - this.columns = columns; - this.rowStyleColumn = rowStyleColumn; - this.evented = evented; - } - - // TODO support this being called multiple times so we can keep viewports going without clearing the data - public TableData handleSnapshot(TableSnapshot snapshot) { - // when changing snapshots we should actually rewrite the columns, possibly emulate ViewportData more? - ColumnData[] dataColumns = snapshot.getDataColumns(); - data = new Object[dataColumns.length]; - reusableDestinations = RangeSet.empty(); - redirectedIndexes = new TreeMap<>(); - index = snapshot.getIncludedRows(); - - long includedRowCount = snapshot.getIncludedRows().size(); - RangeSet destination = freeRows(includedRowCount); - boolean indexUpdated = false; - - for (int index = 0; index < dataColumns.length; index++) { - ColumnData dataColumn = dataColumns[index]; - if (dataColumn == null) { - // no data in this column, wasn't requested - continue; - } - - final int i = index; - Column column = columns.find((c, i1, i2) -> c.getIndex() == i); - - ArrayCopy arrayCopy = arrayCopyFuncForColumn(column); - - Object[] localCopy = new Object[(int) includedRowCount]; - data[index] = localCopy; - PrimitiveIterator.OfLong destIter = destination.indexIterator(); - PrimitiveIterator.OfLong indexIter = snapshot.getIncludedRows().indexIterator(); - int j = 0; - while (indexIter.hasNext()) { - assert destIter.hasNext(); - long dest = destIter.nextLong(); - long nextIndex = indexIter.nextLong(); - if (!indexUpdated) { - redirectedIndexes.put(nextIndex, dest); - } - arrayCopy.copyTo(localCopy, dest, dataColumn.getData(), j++); - } - assert !destIter.hasNext(); - indexUpdated = true; - } - - return notifyUpdates(index, RangeSet.empty(), RangeSet.empty()); - } +/** + * Event data, describing the indexes that were added/removed/updated, and providing access to Rows (and thus data in + * columns) either by index, or scanning the complete present index. + *

+ * This class supports two ways of reading the table - checking the changes made since the last update, and reading all + * data currently in the table. While it is more expensive to always iterate over every single row in the table, it may + * in some cases actually be cheaper than maintaining state separately and updating only the changes, though both + * options should be considered. + *

+ * The RangeSet objects allow iterating over the LongWrapper indexes in the table. Note that these "indexes" are not + * necessarily contiguous and may be negative, and represent some internal state on the server, allowing it to keep + * track of data efficiently. Those LongWrapper objects can be passed to the various methods on this instance to read + * specific rows or cells out of the table. + */ +@TsInterface +@TsName(name = "SubscriptionTableData", namespace = "dh") +public interface SubscriptionTableData extends TableData { + + + @JsProperty + JsRangeSet getFullIndex(); /** - * Helper to avoid appending many times when modifying indexes. The append() method should be called for each key - * _in order_ to ensure that RangeSet.addRange isn't called excessively. When no more items will be added, flush() - * must be called. + * The ordered set of row indexes added since the last update. + * + * @return the rangeset of rows added */ - private static class RangeSetAppendHelper { - private final RangeSet rangeSet; - - private long currentFirst = -1; - private long currentLast; - - public RangeSetAppendHelper(final RangeSet rangeSet) { - this.rangeSet = rangeSet; - } - - public void append(long key) { - assert key >= 0; - - if (currentFirst == -1) { - // first key to be added, move both first and last - currentFirst = key; - currentLast = key; - - return; - } - - if (key == currentLast + 1) { - // key appends to our current range - currentLast = key; - } else if (key == currentFirst - 1) { - // key appends to our current range - currentFirst = key; - } else { - // existing range doesn't match the new item, finish the old range and start a new one - rangeSet.addRange(new Range(currentFirst, currentLast)); - currentFirst = key; - currentLast = key; - } - } - - public void flush() { - if (currentFirst != -1) { - rangeSet.addRange(new Range(currentFirst, currentLast)); - currentFirst = -1; - } - } - } - - public TableData handleDelta(DeltaUpdates delta) { - // delete old data, track slots freed up. we do this by row since they might be non-contiguous or out of order - RangeSetAppendHelper reusableHelper = new RangeSetAppendHelper(reusableDestinations); - delta.getRemoved().indexIterator().forEachRemaining((long index) -> { - long dest = redirectedIndexes.remove(index); - reusableHelper.append(dest); - // TODO consider trimming the columns down too, and truncating the reusable slots at the end - }); - reusableHelper.flush(); - // clean up index by ranges, not by row - delta.getRemoved().rangeIterator().forEachRemaining(index::removeRange); - - // Shift moved rows in the redir index - boolean hasReverseShift = false; - final ShiftedRange[] shiftedRanges = delta.getShiftedRanges(); - RangeSetAppendHelper shifter = new RangeSetAppendHelper(index); - for (int i = shiftedRanges.length - 1; i >= 0; --i) { - final ShiftedRange shiftedRange = shiftedRanges[i]; - final long offset = shiftedRange.getDelta(); - if (offset < 0) { - hasReverseShift = true; - continue; - } - index.removeRange(shiftedRange.getRange()); - final NavigableSet toMove = redirectedIndexes.navigableKeySet() - .subSet(shiftedRange.getRange().getFirst(), true, shiftedRange.getRange().getLast(), true); - // iterate backward and move them forward - for (Long key : toMove.descendingSet()) { - long shiftedKey = key + offset; - Long oldValue = redirectedIndexes.put(shiftedKey, redirectedIndexes.remove(key)); - assert oldValue == null : shiftedKey + " already has a value, " + oldValue; - shifter.append(shiftedKey); - } - } - if (hasReverseShift) { - for (int i = 0; i < shiftedRanges.length; ++i) { - final ShiftedRange shiftedRange = shiftedRanges[i]; - final long offset = shiftedRange.getDelta(); - if (offset > 0) { - continue; - } - index.removeRange(shiftedRange.getRange()); - final NavigableSet toMove = redirectedIndexes.navigableKeySet() - .subSet(shiftedRange.getRange().getFirst(), true, shiftedRange.getRange().getLast(), true); - // iterate forward and move them backward - for (Long key : toMove) { - long shiftedKey = key + offset; - Long oldValue = redirectedIndexes.put(shiftedKey, redirectedIndexes.remove(key)); - assert oldValue == null : shiftedKey + " already has a value, " + oldValue; - shifter.append(shiftedKey); - } - } - } - shifter.flush(); - - // Find space for the rows we're about to add. We must not adjust the index until this is done, it is used - // to see where the end of the data is - RangeSet addedDestination = freeRows(delta.getAdded().size()); - // Within each column, append additions - DeltaUpdates.ColumnAdditions[] additions = delta.getSerializedAdditions(); - for (int i = 0; i < additions.length; i++) { - DeltaUpdates.ColumnAdditions addedColumn = delta.getSerializedAdditions()[i]; - Column column = columns.find((c, i1, i2) -> c.getIndex() == addedColumn.getColumnIndex()); - - ArrayCopy arrayCopy = arrayCopyFuncForColumn(column); - - PrimitiveIterator.OfLong addedIndexes = delta.getAdded().indexIterator(); - PrimitiveIterator.OfLong destIter = addedDestination.indexIterator(); - int j = 0; - while (addedIndexes.hasNext()) { - long origIndex = addedIndexes.nextLong(); - assert delta.getIncludedAdditions().contains(origIndex); - assert destIter.hasNext(); - long dest = destIter.nextLong(); - Long old = redirectedIndexes.put(origIndex, dest); - assert old == null || old == dest; - arrayCopy.copyTo(data[addedColumn.getColumnIndex()], dest, addedColumn.getValues().getData(), j++); - } - } - - // Update the index to reflect the added items - delta.getAdded().rangeIterator().forEachRemaining(index::addRange); - - // Within each column, apply modifications - DeltaUpdates.ColumnModifications[] modifications = delta.getSerializedModifications(); - RangeSet allModified = new RangeSet(); - for (int i = 0; i < modifications.length; ++i) { - final DeltaUpdates.ColumnModifications modifiedColumn = modifications[i]; - if (modifiedColumn == null) { - continue; - } - - modifiedColumn.getRowsIncluded().rangeIterator().forEachRemaining(allModified::addRange); - Column column = columns.find((c, i1, i2) -> c.getIndex() == modifiedColumn.getColumnIndex()); - - ArrayCopy arrayCopy = arrayCopyFuncForColumn(column); - - PrimitiveIterator.OfLong modifiedIndexes = modifiedColumn.getRowsIncluded().indexIterator(); - int j = 0; - while (modifiedIndexes.hasNext()) { - long origIndex = modifiedIndexes.nextLong(); - arrayCopy.copyTo(data[modifiedColumn.getColumnIndex()], redirectedIndexes.get(origIndex), - modifiedColumn.getValues().getData(), j++); - } - } - - // Check that the index sizes make sense - assert redirectedIndexes.size() == index.size(); - // Note that we can't do this assert, since we don't truncate arrays, we just leave nulls at the end - // assert Js.asArrayLike(data[0]).getLength() == redirectedIndexes.size(); - - return notifyUpdates(delta.getAdded(), delta.getRemoved(), allModified); - } - - private TableData notifyUpdates(RangeSet added, RangeSet removed, RangeSet modified) { - UpdateEventData detail = new UpdateEventData(added, removed, modified); - if (evented != null) { - CustomEventInit event = CustomEventInit.create(); - event.setDetail(detail); - evented.fireEvent(TableSubscription.EVENT_UPDATED, event); - } - return detail; - } - - private ArrayCopy arrayCopyFuncForColumn(@Nullable Column column) { - final String type = column != null ? column.getType() : ""; - switch (type) { - case "long": - return (destArray, destPos, srcArray, srcPos) -> { - final long value = Js.asArrayLike(srcArray).getAtAsAny(srcPos).asLong(); - if (value == QueryConstants.NULL_LONG) { - Js.asArrayLike(destArray).setAt((int) destPos, null); - } else { - Js.asArrayLike(destArray).setAt((int) destPos, LongWrapper.of(value)); - } - }; - case "java.time.Instant": - case "java.time.ZonedDateTime": - return (destArray, destPos, srcArray, srcPos) -> { - long value = Js.asArrayLike(srcArray).getAtAsAny(srcPos).asLong(); - if (value == QueryConstants.NULL_LONG) { - Js.asArrayLike(destArray).setAt((int) destPos, null); - } else { - Js.asArrayLike(destArray).setAt((int) destPos, new DateWrapper(value)); - } - }; - case "java.math.BigDecimal": - return (destArray, destPos, srcArray, srcPos) -> { - final BigDecimal value = Js.cast(Js.asArrayLike(srcArray).getAt(srcPos)); - if (value == null) { - Js.asArrayLike(destArray).setAt((int) destPos, null); - } else { - Js.asArrayLike(destArray).setAt((int) destPos, new BigDecimalWrapper(value)); - } - }; - case "java.math.BigInteger": - return (destArray, destPos, srcArray, srcPos) -> { - final BigInteger value = Js.cast(Js.asArrayLike(srcArray).getAt(srcPos)); - if (value == null) { - Js.asArrayLike(destArray).setAt((int) destPos, null); - } else { - Js.asArrayLike(destArray).setAt((int) destPos, new BigIntegerWrapper(value)); - } - }; - case "java.time.LocalDate": - return (destArray, destPos, srcArray, srcPos) -> { - final LocalDate value = Js.cast(Js.asArrayLike(srcArray).getAt(srcPos)); - if (value == null) { - Js.asArrayLike(destArray).setAt((int) destPos, null); - } else { - Js.asArrayLike(destArray).setAt((int) destPos, new LocalDateWrapper(value)); - } - }; - case "java.time.LocalTime": - return (destArray, destPos, srcArray, srcPos) -> { - final LocalTime value = Js.cast(Js.asArrayLike(srcArray).getAt(srcPos)); - if (value == null) { - Js.asArrayLike(destArray).setAt((int) destPos, null); - } else { - Js.asArrayLike(destArray).setAt((int) destPos, new LocalTimeWrapper(value)); - } - }; - case "java.lang.Boolean": - return (destArray, destPos, srcArray, srcPos) -> { - final Any value = Js.asArrayLike(srcArray).getAtAsAny(srcPos); - - if (value == null) { - Js.asArrayLike(destArray).setAt((int) destPos, null); - } else if (value.asBoolean()) { - Js.asArrayLike(destArray).setAt((int) destPos, true); - } else { - Js.asArrayLike(destArray).setAt((int) destPos, false); - } - }; - case "int": - return (destArray, destPos, srcArray, srcPos) -> { - final int value = Js.asArrayLike(srcArray).getAtAsAny(srcPos).asInt(); - if (value == QueryConstants.NULL_INT) { - Js.asArrayLike(destArray).setAt((int) destPos, null); - } else { - Js.asArrayLike(destArray).setAt((int) destPos, value); - } - }; - case "byte": - return (destArray, destPos, srcArray, srcPos) -> { - final byte value = Js.asArrayLike(srcArray).getAtAsAny(srcPos).asByte(); - if (value == QueryConstants.NULL_BYTE) { - Js.asArrayLike(destArray).setAt((int) destPos, null); - } else { - Js.asArrayLike(destArray).setAt((int) destPos, value); - } - }; - case "short": - return (destArray, destPos, srcArray, srcPos) -> { - final short value = Js.asArrayLike(srcArray).getAtAsAny(srcPos).asShort(); - if (value == QueryConstants.NULL_SHORT) { - Js.asArrayLike(destArray).setAt((int) destPos, null); - } else { - Js.asArrayLike(destArray).setAt((int) destPos, value); - } - }; - case "double": - return (destArray, destPos, srcArray, srcPos) -> { - final double value = Js.asArrayLike(srcArray).getAtAsAny(srcPos).asDouble(); - if (value == QueryConstants.NULL_DOUBLE) { - Js.asArrayLike(destArray).setAt((int) destPos, null); - } else { - Js.asArrayLike(destArray).setAt((int) destPos, value); - } - }; - case "float": - return (destArray, destPos, srcArray, srcPos) -> { - final float value = Js.asArrayLike(srcArray).getAtAsAny(srcPos).asFloat(); - if (value == QueryConstants.NULL_FLOAT) { - Js.asArrayLike(destArray).setAt((int) destPos, null); - } else { - Js.asArrayLike(destArray).setAt((int) destPos, value); - } - }; - case "char": - return (destArray, destPos, srcArray, srcPos) -> { - final char value = Js.asArrayLike(srcArray).getAtAsAny(srcPos).asChar(); - if (value == QueryConstants.NULL_CHAR) { - Js.asArrayLike(destArray).setAt((int) destPos, null); - } else { - Js.asArrayLike(destArray).setAt((int) destPos, value); - } - }; - default: - // exit so we can handle null also in the method's final return - } - return (destArray, destPos, srcArray, srcPos) -> { - // boring column or format data, just copy it - Js.asArrayLike(destArray).setAt((int) destPos, Js.asArrayLike(srcArray).getAt(srcPos)); - }; - } - - private RangeSet freeRows(long required) { - if (required == 0) { - return RangeSet.empty(); - } - long existingSlotsToReuse = reusableDestinations.size(); - if (existingSlotsToReuse > required) { - // only take some of the ranges from the reusable list - RangeSet reused = RangeSet.empty(); - long taken = 0; - RangeSet stillUnused = RangeSet.empty(); - // TODO this could be more efficient, iterating entire ranges until we only need a partial range - PrimitiveIterator.OfLong iterator = reusableDestinations.indexIterator(); - while (taken < required) { - assert iterator.hasNext(); - long value = iterator.nextLong(); - reused.addRange(new Range(value, value)); - taken++; - } - assert taken == required; - while (iterator.hasNext()) { - long value = iterator.nextLong(); - stillUnused.addRange(new Range(value, value)); - } - reusableDestinations = stillUnused; - assert required == reused.size(); - return reused; - } - // take all ranges from the reusable list (plus make more if needed) - RangeSet created = reusableDestinations; - if (existingSlotsToReuse < required) { - long nextIndex; - if (created.isEmpty()) { - if (index.isEmpty()) { - nextIndex = 0; - } else { - nextIndex = redirectedIndexes.size(); - } - } else if (index.isEmpty()) { - nextIndex = created.getLastRow() + 1; - } else { - nextIndex = Math.max(created.getLastRow(), index.getLastRow()) + 1; - } - created.addRange(new Range(nextIndex, nextIndex + required - existingSlotsToReuse - 1)); - } - - reusableDestinations = RangeSet.empty(); - assert required == created.size(); - return created; - } - - @TsInterface - @TsName(namespace = "dh") - public class SubscriptionRow implements TableData.Row { - private final long index; - public LongWrapper indexCached; - - public SubscriptionRow(long index) { - this.index = index; - } - - @Override - public LongWrapper getIndex() { - if (indexCached == null) { - indexCached = LongWrapper.of(index); - } - return indexCached; - } - - @Override - public Any get(Column column) { - int redirectedIndex = (int) (long) redirectedIndexes.get(this.index); - JsArrayLike columnData = Js.asArrayLike(data[column.getIndex()]); - return columnData.getAtAsAny(redirectedIndex); - } - - @Override - public Format getFormat(Column column) { - long cellColors = 0; - long rowColors = 0; - String numberFormat = null; - String formatString = null; - int redirectedIndex = (int) (long) redirectedIndexes.get(this.index); - if (column.getStyleColumnIndex() != null) { - JsArray colors = Js.uncheckedCast(data[column.getStyleColumnIndex()]); - cellColors = colors.getAtAsAny(redirectedIndex).asLong(); - } - if (rowStyleColumn != NO_ROW_FORMAT_COLUMN) { - JsArray rowStyle = Js.uncheckedCast(data[rowStyleColumn]); - rowColors = rowStyle.getAtAsAny(redirectedIndex).asLong(); - } - if (column.getFormatColumnIndex() != null) { - JsArray formatStrings = Js.uncheckedCast(data[column.getFormatColumnIndex()]); - numberFormat = formatStrings.getAtAsAny(redirectedIndex).asString(); - } - if (column.getFormatStringColumnIndex() != null) { - JsArray formatStrings = Js.uncheckedCast(data[column.getFormatStringColumnIndex()]); - formatString = formatStrings.getAtAsAny(redirectedIndex).asString(); - } - return new Format(cellColors, rowColors, numberFormat, formatString); - } - } - + @JsProperty + JsRangeSet getAdded(); /** - * Event data, describing the indexes that were added/removed/updated, and providing access to Rows (and thus data - * in columns) either by index, or scanning the complete present index. + * The ordered set of row indexes removed since the last update * - * This class supports two ways of reading the table - checking the changes made since the last update, and reading - * all data currently in the table. While it is more expensive to always iterate over every single row in the table, - * it may in some cases actually be cheaper than maintaining state separately and updating only the changes, though - * both options should be considered. - * - * The RangeSet objects allow iterating over the LongWrapper indexes in the table. Note that these "indexes" are not - * necessarily contiguous and may be negative, and represent some internal state on the server, allowing it to keep - * track of data efficiently. Those LongWrapper objects can be passed to the various methods on this instance to - * read specific rows or cells out of the table. + * @return the rangeset of removed rows */ - @TsInterface - @TsName(name = "SubscriptionTableData", namespace = "dh") - public class UpdateEventData implements TableData { - private JsRangeSet added; - private JsRangeSet removed; - private JsRangeSet modified; - - // cached copy in case it was requested, could be requested again - private JsArray allRows; + @JsProperty + JsRangeSet getRemoved(); - public UpdateEventData(RangeSet added, RangeSet removed, RangeSet modified) { - this.added = new JsRangeSet(added); - this.removed = new JsRangeSet(removed); - this.modified = new JsRangeSet(modified); - } - - /** - * A lazily computed array of all rows in the entire table - * - * @return {@link SubscriptionRow} array. - */ - @Override - public JsArray getRows() { - if (allRows == null) { - allRows = new JsArray<>(); - index.indexIterator().forEachRemaining((long index) -> { - allRows.push(new SubscriptionRow(index)); - }); - if (JsSettings.isDevMode()) { - assert allRows.length == index.size(); - } - } - return allRows; - } - - @Override - public Row get(int index) { - return this.get((long) index); - } - - /** - * Reads a row object from the table, from which any subscribed column can be read - * - * @param index - * @return {@link SubscriptionRow} - */ - @Override - public SubscriptionRow get(long index) { - return new SubscriptionRow(index); - } - - @Override - public Any getData(int index, Column column) { - return getData((long) index, column); - } - - /** - * a specific cell from the table, from the specified row and column - * - * @param index - * @param column - * @return Any - */ - @Override - public Any getData(long index, Column column) { - int redirectedIndex = (int) (long) redirectedIndexes.get(index); - JsArrayLike columnData = Js.asArrayLike(data[column.getIndex()]); - return columnData.getAtAsAny(redirectedIndex); - } - - /** - * the Format to use for a cell from the specified row and column - * - * @param index - * @param column - * @return {@link Format} - */ - @Override - public Format getFormat(int index, Column column) { - return getFormat((long) index, column); - } - - @Override - public Format getFormat(long index, Column column) { - long cellColors = 0; - long rowColors = 0; - String numberFormat = null; - String formatString = null; - int redirectedIndex = (int) (long) redirectedIndexes.get(index); - if (column.getStyleColumnIndex() != null) { - JsArray colors = Js.uncheckedCast(data[column.getStyleColumnIndex()]); - cellColors = colors.getAtAsAny(redirectedIndex).asLong(); - } - if (rowStyleColumn != NO_ROW_FORMAT_COLUMN) { - JsArray rowStyle = Js.uncheckedCast(data[rowStyleColumn]); - rowColors = rowStyle.getAtAsAny(redirectedIndex).asLong(); - } - if (column.getFormatColumnIndex() != null) { - JsArray formatStrings = Js.uncheckedCast(data[column.getFormatColumnIndex()]); - numberFormat = formatStrings.getAtAsAny(redirectedIndex).asString(); - } - if (column.getFormatStringColumnIndex() != null) { - JsArray formatStrings = Js.uncheckedCast(data[column.getFormatStringColumnIndex()]); - formatString = formatStrings.getAtAsAny(redirectedIndex).asString(); - } - return new Format(cellColors, rowColors, numberFormat, formatString); - } - - @Override - public JsArray getColumns() { - return columns; - } - - /** - * The ordered set of row indexes added since the last update - * - * @return dh.RangeSet - */ - @JsProperty - public JsRangeSet getAdded() { - return added; - } - - /** - * The ordered set of row indexes removed since the last update - * - * @return dh.RangeSet - */ - @JsProperty - public JsRangeSet getRemoved() { - return removed; - } - - /** - * The ordered set of row indexes updated since the last update - * - * @return dh.RangeSet - */ - @JsProperty - public JsRangeSet getModified() { - return modified; - } - - @JsProperty - public JsRangeSet getFullIndex() { - return new JsRangeSet(index); - } - } + /** + * The ordered set of row indexes updated since the last update + * + * @return the rnageset of modified rows + */ + @JsProperty + JsRangeSet getModified(); } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableSubscription.java index f94aa9a5775..e8399666d9c 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableSubscription.java @@ -3,21 +3,14 @@ // package io.deephaven.web.client.api.subscription; -import com.vertispan.tsdefs.annotations.TsName; import elemental2.core.JsArray; -import elemental2.promise.Promise; import io.deephaven.web.client.api.Column; -import io.deephaven.web.client.api.HasEventHandling; import io.deephaven.web.client.api.JsTable; -import io.deephaven.web.shared.data.DeltaUpdates; -import io.deephaven.web.shared.data.TableSnapshot; import jsinterop.annotations.JsIgnore; -import jsinterop.annotations.JsMethod; +import jsinterop.annotations.JsNullable; import jsinterop.annotations.JsProperty; import jsinterop.annotations.JsType; -import static io.deephaven.web.client.api.subscription.ViewportData.NO_ROW_FORMAT_COLUMN; - /** * Represents a non-viewport subscription to a table, and all data currently known to be present in the subscribed * columns. This class handles incoming snapshots and deltas, and fires events to consumers to notify of data changes. @@ -31,80 +24,40 @@ * viewports to make it less expensive to compute for large tables. */ @JsType(namespace = "dh") -public class TableSubscription extends HasEventHandling { - - /** - * Indicates that some new data is available on the client, either an initial snapshot or a delta update. The - * detail field of the event will contain a TableSubscriptionEventData detailing what has changed, or - * allowing access to the entire range of items currently in the subscribed columns. - */ - public static final String EVENT_UPDATED = "updated"; - - - // column defs in this subscription - private JsArray columns; - // holder for data - private SubscriptionTableData data; +public final class TableSubscription extends AbstractTableSubscription { - // table created for this subscription - private Promise copy; + private final JsArray columns; + private final Double updateIntervalMs; - // copy from the initially given table so we don't need to way @JsIgnore public TableSubscription(JsArray columns, JsTable existingTable, Double updateIntervalMs) { - - copy = existingTable.copy(false).then(table -> new Promise<>((resolve, reject) -> { - table.state().onRunning(newState -> { - // TODO handle updateInterval core#188 - table.internalSubscribe(columns, this); - - resolve.onInvoke(table); - }, table::close); - })); - + super(existingTable.state(), existingTable.getConnection()); this.columns = columns; - Integer rowStyleColumn = existingTable.state().getRowFormatColumn() == null ? NO_ROW_FORMAT_COLUMN - : existingTable.state().getRowFormatColumn().getIndex(); - this.data = new SubscriptionTableData(columns, rowStyleColumn, this); - - } - - // public void changeSubscription(JsArray columns) { - // copy.then(t ->{ - // t.internalSubscribe(columns, this); - // return Promise.resolve(t); - // }); - // this.columns = columns; - // } - - - @JsIgnore - public void handleSnapshot(TableSnapshot snapshot) { - data.handleSnapshot(snapshot); + this.updateIntervalMs = updateIntervalMs; } - @JsIgnore - public void handleDelta(DeltaUpdates delta) { - data.handleDelta(delta); + @Override + protected void sendFirstSubscriptionRequest() { + changeSubscription(columns, updateIntervalMs); } /** - * The columns that were subscribed to when this subscription was created + * Updates the subscription to use the given columns and update interval. * - * @return {@link Column} + * @param columns the new columns to subscribe to + * @param updateIntervalMs the new update interval, or null/omit to use the default of one second */ - @JsProperty - public JsArray getColumns() { - return columns; + public void changeSubscription(JsArray columns, @JsNullable Double updateIntervalMs) { + if (updateIntervalMs != null && !updateIntervalMs.equals(this.updateIntervalMs)) { + throw new IllegalArgumentException( + "Can't change refreshIntervalMs on a later call to setViewport, it must be consistent or omitted"); + } + sendBarrageSubscriptionRequest(null, columns, updateIntervalMs, false); } - /** - * Stops the subscription on the server. - */ - public void close() { - copy.then(table -> { - table.close(); - return Promise.resolve(table); - }); + @JsProperty + @Override + public JsArray getColumns() { + return super.getColumns(); } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java index d567fbd3094..203dc0c7890 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java @@ -3,188 +3,240 @@ // package io.deephaven.web.client.api.subscription; -import com.vertispan.tsdefs.annotations.TsInterface; +import com.google.flatbuffers.FlatBufferBuilder; import com.vertispan.tsdefs.annotations.TsName; -import elemental2.core.Uint8Array; +import com.vertispan.tsdefs.annotations.TsTypeRef; +import elemental2.core.JsArray; import elemental2.dom.CustomEvent; import elemental2.dom.CustomEventInit; -import elemental2.dom.DomGlobal; -import elemental2.promise.IThenable; import elemental2.promise.Promise; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf.Message; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf.MessageHeader; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf.RecordBatch; +import io.deephaven.barrage.flatbuf.BarrageMessageType; +import io.deephaven.barrage.flatbuf.BarrageSnapshotRequest; +import io.deephaven.extensions.barrage.BarrageSnapshotOptions; +import io.deephaven.extensions.barrage.ColumnConversionMode; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightData; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageMessageType; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageMessageWrapper; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageSnapshotOptions; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageSnapshotRequest; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageUpdateMetadata; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.ColumnConversionMode; -import io.deephaven.web.client.api.Callbacks; +import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.config_pb.ConfigValue; +import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.table_pb.FlattenRequest; +import io.deephaven.util.mutable.MutableLong; import io.deephaven.web.client.api.Column; -import io.deephaven.web.client.api.HasEventHandling; import io.deephaven.web.client.api.JsRangeSet; import io.deephaven.web.client.api.JsTable; import io.deephaven.web.client.api.TableData; import io.deephaven.web.client.api.WorkerConnection; +import io.deephaven.web.client.api.barrage.WebBarrageMessage; +import io.deephaven.web.client.api.barrage.WebBarrageStreamReader; import io.deephaven.web.client.api.barrage.WebBarrageUtils; -import io.deephaven.web.client.api.barrage.def.ColumnDefinition; +import io.deephaven.web.client.api.barrage.data.WebBarrageSubscription; import io.deephaven.web.client.api.barrage.stream.BiDiStream; import io.deephaven.web.client.fu.JsLog; +import io.deephaven.web.client.fu.LazyPromise; import io.deephaven.web.client.state.ClientTableState; import io.deephaven.web.shared.data.Range; -import io.deephaven.web.shared.data.TableSnapshot; +import io.deephaven.web.shared.data.RangeSet; +import io.deephaven.web.shared.data.ShiftedRange; +import io.deephaven.web.shared.fu.RemoverFn; import jsinterop.annotations.JsMethod; import jsinterop.annotations.JsNullable; import jsinterop.annotations.JsOptional; import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; +import java.io.IOException; import java.util.Arrays; -import java.util.BitSet; import java.util.Collections; -import java.util.Iterator; +import java.util.Comparator; +import java.util.PrimitiveIterator; -import static io.deephaven.web.client.api.barrage.WebBarrageUtils.makeUint8ArrayFromBitset; +import static io.deephaven.web.client.api.JsTable.EVENT_ROWADDED; +import static io.deephaven.web.client.api.JsTable.EVENT_ROWREMOVED; +import static io.deephaven.web.client.api.JsTable.EVENT_ROWUPDATED; import static io.deephaven.web.client.api.barrage.WebBarrageUtils.serializeRanges; -import static io.deephaven.web.client.api.subscription.ViewportData.NO_ROW_FORMAT_COLUMN; /** - * Encapsulates event handling around table subscriptions by "cheating" and wrapping up a JsTable instance to do the - * real dirty work. This allows a viewport to stay open on the old table if desired, while this one remains open. - *

- * As this just wraps a JsTable (and thus a CTS), it holds its own flattened, pUT'd handle to get deltas from the - * server. The setViewport method can be used to adjust this table instead of creating a new one. - *

- * Existing methods on JsTable like setViewport and getViewportData are intended to proxy to this, which then will talk - * to the underlying handle and accumulated data. - *

- * As long as we keep the existing methods/events on JsTable, close() is not required if no other method is called, with - * the idea then that the caller did not actually use this type. This means that for every exported method (which then - * will mark the instance of "actually being used, please don't automatically close me"), there must be an internal - * version called by those existing JsTable method, which will allow this instance to be cleaned up once the JsTable - * deems it no longer in use. - *

- * Note that if the caller does close an instance, this shuts down the JsTable's use of this (while the converse is not - * true), providing a way to stop the server from streaming updates to the client. - * * This object serves as a "handle" to a subscription, allowing it to be acted on directly or canceled outright. If you * retain an instance of this, you have two choices - either only use it to call `close()` on it to stop the table's * viewport without creating a new one, or listen directly to this object instead of the table for data events, and * always call `close()` when finished. Calling any method on this object other than close() will result in it * continuing to live on after `setViewport` is called on the original table, or after the table is modified. */ -@TsInterface @TsName(namespace = "dh") -public class TableViewportSubscription extends HasEventHandling { - /** - * Describes the possible lifecycle of the viewport as far as anything external cares about it - */ - public enum Status { - /** - * Waiting for some prerequisite before we can begin, usually waiting to make sure the original table is ready - * to be subscribed to. Once the original table is ready, we will enter the ACTIVE state, even if the first - * update hasn't yet arrived. - */ - STARTING, - /** - * Successfully created, viewport is at least begun on the server, updates are subscribed and if changes happen - * on the server, we will be notified. - */ - ACTIVE, - /** - * Closed or otherwise dead, can not be used again. - */ - DONE - } +public class TableViewportSubscription extends AbstractTableSubscription { - private final double refresh; + private double firstRow; + private double lastRow; + private Column[] columns; + private double refresh; private final JsTable original; - private final ClientTableState originalState; - private final Promise copy; - private JsTable realized; + private final RemoverFn reconnectSubscription; - private boolean retained;// if the sub is set up to not close the underlying table once the original table is done - // with it + /** The initial state of the provided table, before flattening. */ + private final ClientTableState initialState; + + /** + * true if the sub is set up to not close the underlying table once the original table is done with it, otherwise + * false. + */ private boolean originalActive = true; + /** + * true if the developer has called methods directly on the subscription, otherwise false. + */ + private boolean retained; + + private UpdateEventData viewportData; + + public static TableViewportSubscription make(double firstRow, double lastRow, Column[] columns, + Double updateIntervalMs, JsTable existingTable) { + ClientTableState tableState = existingTable.state(); + WorkerConnection connection = existingTable.getConnection(); + + final ClientTableState stateToSubscribe; + ConfigValue flattenViewport = connection.getServerConfigValue("web.flattenViewports"); + if (flattenViewport != null && flattenViewport.hasStringValue() + && "true".equalsIgnoreCase(flattenViewport.getStringValue())) { + stateToSubscribe = connection.newState((callback, newState, metadata) -> { + FlattenRequest flatten = new FlattenRequest(); + flatten.setSourceId(tableState.getHandle().makeTableReference()); + flatten.setResultId(newState.getHandle().makeTicket()); + connection.tableServiceClient().flatten(flatten, metadata, callback::apply); + }, "flatten"); + stateToSubscribe.refetch(null, connection.metadata()).then(result -> null, err -> null); + } else { + stateToSubscribe = tableState; + } - private Status status = Status.STARTING; + TableViewportSubscription sub = new TableViewportSubscription(stateToSubscribe, connection, existingTable); + sub.setInternalViewport(firstRow, lastRow, columns, updateIntervalMs, false); + return sub; + } - public TableViewportSubscription(double firstRow, double lastRow, Column[] columns, Double updateIntervalMs, - JsTable existingTable) { - refresh = updateIntervalMs == null ? 1000.0 : updateIntervalMs; - // first off, copy the table, and flatten/pUT it, then apply the new viewport to that + public TableViewportSubscription(ClientTableState state, WorkerConnection connection, JsTable existingTable) { + super(state, connection); this.original = existingTable; - this.originalState = original.state(); - copy = existingTable.copy(false).then(table -> new Promise<>((resolve, reject) -> { - // Wait until the state is running to copy it - originalState.onRunning(newState -> { - if (this.status == Status.DONE) { - JsLog.debug("TableViewportSubscription closed before originalState.onRunning completed, ignoring"); - table.close(); - return; - } - table.batch(batcher -> { - batcher.customColumns(newState.getCustomColumns()); - batcher.filter(newState.getFilters()); - batcher.sort(newState.getSorts()); - batcher.setFlat(true); - }); - // TODO handle updateInterval core#188 - Column[] columnsToSub = table.isBlinkTable() ? Js.uncheckedCast(table.getColumns()) : columns; - table.setInternalViewport(firstRow, lastRow, columnsToSub); - - // Listen for events and refire them on ourselves, optionally on the original table - table.addEventListener(JsTable.EVENT_UPDATED, this::refire); - table.addEventListener(JsTable.EVENT_ROWADDED, this::refire); - table.addEventListener(JsTable.EVENT_ROWREMOVED, this::refire); - table.addEventListener(JsTable.EVENT_ROWUPDATED, this::refire); - table.addEventListener(JsTable.EVENT_SIZECHANGED, this::refire); - // TODO (core#1181): fix this hack that enables barrage errors to propagate to the UI widget - table.addEventListener(JsTable.EVENT_REQUEST_FAILED, this::refire); - - // Take over for the "parent" table - // Cache original table size so we can tell if we need to notify about a change - double originalSize = newState.getSize(); - realized = table; - status = Status.ACTIVE; - // At this point we're now responsible for notifying of size changes, since we will shortly have a - // viewport, - // a more precise way to track the table size (at least w.r.t. the range of the viewport), so if there - // is any difference in size between "realized" and "original", notify now to finish the transition. - if (realized.getSize() != originalSize) { - JsLog.debug( - "firing size changed to transition between table managing its own size changes and viewport sub taking over", - realized.getSize()); - CustomEventInit init = CustomEventInit.create(); - init.setDetail(realized.getSize()); - refire(new CustomEvent(JsTable.EVENT_SIZECHANGED, init)); + initialState = existingTable.state(); + this.reconnectSubscription = existingTable.addEventListener(JsTable.EVENT_RECONNECT, e -> { + if (existingTable.state() == initialState) { + revive(); + } + }); + } + + // Expose this as public + @Override + public void revive() { + super.revive(); + } + + @Override + protected void sendFirstSubscriptionRequest() { + setInternalViewport(firstRow, lastRow, columns, refresh, null); + } + + @Override + protected void notifyUpdate(RangeSet rowsAdded, RangeSet rowsRemoved, RangeSet totalMods, ShiftedRange[] shifted) { + // viewport subscriptions are sometimes required to notify of size change events + if (rowsAdded.size() != rowsRemoved.size() && originalActive) { + fireEventWithDetail(JsTable.EVENT_SIZECHANGED, size()); + } + UpdateEventData detail = new SubscriptionEventData(barrageSubscription, rowStyleColumn, getColumns(), rowsAdded, + rowsRemoved, totalMods, shifted); + + detail.setOffset(this.viewportRowSet.getFirstRow()); + this.viewportData = detail; + CustomEventInit event = CustomEventInit.create(); + event.setDetail(detail); + refire(new CustomEvent<>(EVENT_UPDATED, event)); + + if (hasListeners(EVENT_ROWADDED) || hasListeners(EVENT_ROWREMOVED) || hasListeners(EVENT_ROWUPDATED)) { + RangeSet modifiedCopy = totalMods.copy(); + // exclude added items from being marked as modified, since we're hiding shifts from api consumers + modifiedCopy.removeRangeSet(rowsAdded); + RangeSet removedCopy = rowsRemoved.copy(); + RangeSet addedCopy = rowsAdded.copy(); + + // Any position which was both added and removed should instead be marked as modified, this cleans + // up anything excluded above that didn't otherwise make sense + for (PrimitiveIterator.OfLong it = removedCopy.indexIterator(); it.hasNext();) { + long index = it.nextLong(); + if (addedCopy.contains(index)) { + addedCopy.removeRange(new Range(index, index)); + it.remove(); + modifiedCopy.addRange(new Range(index, index)); } + } - resolve.onInvoke(table); - }, table::close); - })); + fireLegacyEventOnRowsetEntries(EVENT_ROWADDED, detail, rowsAdded); + fireLegacyEventOnRowsetEntries(EVENT_ROWUPDATED, detail, totalMods); + fireLegacyEventOnRowsetEntries(EVENT_ROWREMOVED, detail, rowsRemoved); + } } - /** - * Reflects the state of the original table, before being flattened. - */ - public ClientTableState state() { - return originalState; + private void fireLegacyEventOnRowsetEntries(String eventName, UpdateEventData updateEventData, RangeSet rowset) { + if (hasListeners(eventName)) { + rowset.indexIterator().forEachRemaining((long row) -> { + CustomEventInit> addedEvent = CustomEventInit.create(); + addedEvent.setDetail(wrap((SubscriptionRow) updateEventData.getRows().getAt((int) row), (int) row)); + fireEvent(eventName, addedEvent); + }); + } } + private static JsPropertyMap wrap(SubscriptionRow rowObj, int row) { + return JsPropertyMap.of("row", rowObj, "index", (double) row); + } + + @Override + public void fireEvent(String type) { + refire(new CustomEvent<>(type)); + } + + @Override + public void fireEventWithDetail(String type, T detail) { + CustomEventInit init = CustomEventInit.create(); + init.setDetail(detail); + refire(new CustomEvent(type, init)); + } + + @Override + public void fireEvent(String type, CustomEventInit init) { + refire(new CustomEvent(type, init)); + } + + @Override + public void fireEvent(String type, CustomEvent e) { + if (!type.equals(e.type)) { + throw new IllegalArgumentException(type + " != " + e.type); + } + refire(e); + } + + @Override + public boolean hasListeners(String name) { + if (originalActive && initialState == original.state()) { + if (original.hasListeners(name)) { + return true; + } + } + return super.hasListeners(name); + } + + /** + * Utility to fire an event on this object and also optionally on the parent if still active. All {@code fireEvent} + * overloads dispatch to this. + * + * @param e the event to fire + * @param the type of the custom event data + */ private void refire(CustomEvent e) { - this.fireEvent(e.type, e); - if (originalActive && state() == original.state()) { + // explicitly calling super.fireEvent to avoid calling ourselves recursively + super.fireEvent(e.type, e); + if (originalActive && initialState == original.state()) { // When these fail to match, it probably means that the original's state was paused, but we're still // holding on to it. Since we haven't been internalClose()d yet, that means we're still waiting for // the new state to resolve or fail, so we can be restored, or stopped. In theory, we should put this // assert back, and make the pause code also tell us to pause. - // assert state() == original.state() : "Table owning this viewport subscription forgot to release it"; + // assert initialState == original.state() : "Table owning this viewport subscription forgot to release it"; original.fireEvent(e.type, e); } } @@ -203,23 +255,40 @@ private void retainForExternalUse() { */ @JsMethod public void setViewport(double firstRow, double lastRow, @JsOptional @JsNullable Column[] columns, - @JsOptional @JsNullable Double updateIntervalMs) { + @JsOptional @JsNullable Double updateIntervalMs, + @JsOptional @JsNullable Boolean isReverseViewport) { retainForExternalUse(); - setInternalViewport(firstRow, lastRow, columns, updateIntervalMs); + setInternalViewport(firstRow, lastRow, columns, updateIntervalMs, isReverseViewport); } - public void setInternalViewport(double firstRow, double lastRow, Column[] columns, Double updateIntervalMs) { + public void setInternalViewport(double firstRow, double lastRow, Column[] columns, Double updateIntervalMs, + Boolean isReverseViewport) { + if (status == Status.STARTING) { + this.firstRow = firstRow; + this.lastRow = lastRow; + this.columns = columns; + this.refresh = updateIntervalMs == null ? 1000.0 : updateIntervalMs; + return; + } + if (columns == null) { + // Null columns means the user wants all columns, only supported on viewports. This can't be done until the + // CTS has resolved + columns = state().getColumns(); + } else { + // If columns were provided, sort a copy so that we have them in the expected order + columns = Js.>uncheckedCast(columns).slice().asArray(new Column[0]); + Arrays.sort(columns, Comparator.comparing(Column::getIndex)); + } if (updateIntervalMs != null && refresh != updateIntervalMs) { throw new IllegalArgumentException( "Can't change refreshIntervalMs on a later call to setViewport, it must be consistent or omitted"); } - copy.then(table -> { - if (!table.isBlinkTable()) { - // we only set blink table viewports once; and that's in the constructor - table.setInternalViewport(firstRow, lastRow, columns); - } - return Promise.resolve(table); - }); + if (isReverseViewport == null) { + isReverseViewport = false; + } + RangeSet viewport = RangeSet.ofRange((long) firstRow, (long) lastRow); + this.sendBarrageSubscriptionRequest(viewport, Js.uncheckedCast(columns), updateIntervalMs, + isReverseViewport); } /** @@ -231,6 +300,8 @@ public void close() { JsLog.warn("TableViewportSubscription.close called on subscription that's already done."); } retained = false; + + // Instead of calling super.close(), we delegate to internalClose() internalClose(); } @@ -242,6 +313,8 @@ public void internalClose() { // indicate that the base table shouldn't get events anymore, even if it is still retained elsewhere originalActive = false; + reconnectSubscription.remove(); + if (retained || status == Status.DONE) { // the JsTable has indicated it is no longer interested in this viewport, but other calling // code has retained it, keep it open for now. @@ -250,13 +323,7 @@ public void internalClose() { status = Status.DONE; - // not retained externally, and the original is inactive, mark as "not realized" - realized = null; - - copy.then(table -> { - table.close(); - return Promise.resolve(table); - }); + super.close(); } /** @@ -265,149 +332,107 @@ public void internalClose() { * @return Promise of {@link TableData}. */ @JsMethod - public Promise getViewportData() { + public Promise<@TsTypeRef(ViewportData.class) UpdateEventData> getViewportData() { retainForExternalUse(); return getInternalViewportData(); } - public Promise getInternalViewportData() { - return copy.then(JsTable::getInternalViewportData); - } - - public Status getStatus() { - if (realized == null) { - assert status != Status.ACTIVE - : "when the realized table is null, status should only be DONE or STARTING, instead is " + status; - } else { - if (realized.isAlive()) { - assert status == Status.ACTIVE - : "realized table is alive, expected status ACTIVE, instead is " + status; - } else { - assert status == Status.DONE : "realized table is closed, expected status DONE, instead is " + status; - } + public Promise<@TsTypeRef(ViewportData.class) UpdateEventData> getInternalViewportData() { + if (isSubscriptionReady()) { + return Promise.resolve(viewportData); } - - return status; - } - - public double size() { - assert getStatus() == Status.ACTIVE; - return realized.getSize(); - } - - public double totalSize() { - assert getStatus() == Status.ACTIVE; - return realized.getTotalSize(); + final LazyPromise promise = new LazyPromise<>(); + addEventListenerOneShot(EVENT_UPDATED, ignored -> promise.succeed(viewportData)); + return promise.asPromise(); } @JsMethod public Promise snapshot(JsRangeSet rows, Column[] columns) { retainForExternalUse(); // TODO #1039 slice rows and drop columns - return copy.then(table -> { - final ClientTableState state = table.lastVisibleState(); - String[] columnTypes = Arrays.stream(state.getTableDef().getColumns()) - .map(ColumnDefinition::getType) - .toArray(String[]::new); - - final BitSet columnBitset = table.lastVisibleState().makeBitset(columns); - return Callbacks.promise(this, callback -> { - WorkerConnection connection = table.getConnection(); - BiDiStream stream = connection.streamFactory().create( - headers -> connection.flightServiceClient().doExchange(headers), - (first, headers) -> connection.browserFlightServiceClient().openDoExchange(first, headers), - (next, headers, c) -> connection.browserFlightServiceClient().nextDoExchange(next, headers, - c::apply), - new FlightData()); - - Builder doGetRequest = new Builder(1024); - double columnsOffset = BarrageSnapshotRequest.createColumnsVector(doGetRequest, - makeUint8ArrayFromBitset(columnBitset)); - double viewportOffset = BarrageSnapshotRequest.createViewportVector(doGetRequest, serializeRanges( - Collections.singleton(rows.getRange()))); - double serializationOptionsOffset = BarrageSnapshotOptions - .createBarrageSnapshotOptions(doGetRequest, ColumnConversionMode.Stringify, true, 0, 0); - double tableTicketOffset = - BarrageSnapshotRequest.createTicketVector(doGetRequest, state.getHandle().getTicket()); - BarrageSnapshotRequest.startBarrageSnapshotRequest(doGetRequest); - BarrageSnapshotRequest.addTicket(doGetRequest, tableTicketOffset); - BarrageSnapshotRequest.addColumns(doGetRequest, columnsOffset); - BarrageSnapshotRequest.addSnapshotOptions(doGetRequest, serializationOptionsOffset); - BarrageSnapshotRequest.addViewport(doGetRequest, viewportOffset); - doGetRequest.finish(BarrageSnapshotRequest.endBarrageSnapshotRequest(doGetRequest)); - - FlightData request = new FlightData(); - request.setAppMetadata( - WebBarrageUtils.wrapMessage(doGetRequest, BarrageMessageType.BarrageSnapshotRequest)); - stream.send(request); - stream.end(); - stream.onData(flightData -> { - - Message message = Message.getRootAsMessage(new ByteBuffer(flightData.getDataHeader_asU8())); - if (message.headerType() == MessageHeader.Schema) { - // ignore for now, we'll handle this later - return; - } - assert message.headerType() == MessageHeader.RecordBatch; - RecordBatch header = message.header(new RecordBatch()); - Uint8Array appMetadataBytes = flightData.getAppMetadata_asU8(); - BarrageUpdateMetadata update = null; - if (appMetadataBytes.length != 0) { - BarrageMessageWrapper barrageMessageWrapper = - BarrageMessageWrapper.getRootAsBarrageMessageWrapper( - new io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer( - appMetadataBytes)); - - update = BarrageUpdateMetadata.getRootAsBarrageUpdateMetadata( - new ByteBuffer( - new Uint8Array(barrageMessageWrapper.msgPayloadArray()))); - } - TableSnapshot snapshot = WebBarrageUtils.createSnapshot(header, - WebBarrageUtils.typedArrayToLittleEndianByteBuffer(flightData.getDataBody_asU8()), update, - true, - columnTypes); - - // TODO deephaven-core(#188) this check no longer makes sense - Iterator rangeIterator = rows.getRange().rangeIterator(); - long expectedCount = 0; - while (rangeIterator.hasNext()) { - Range range = rangeIterator.next(); - if (range.getFirst() >= snapshot.getTableSize()) { - break; - } - long end = Math.min(range.getLast(), snapshot.getTableSize()); - expectedCount += end - range.getFirst() + 1; - } - if (expectedCount != snapshot.getIncludedRows().size()) { - callback.onFailure("Server did not send expected number of rows, expected " + expectedCount - + ", actual " + snapshot.getIncludedRows().size()); - } else { - callback.onSuccess(snapshot); - } - }); - stream.onStatus(status -> { - if (!status.isOk()) { - callback.onFailure(status.getDetails()); - } + BarrageSnapshotOptions options = BarrageSnapshotOptions.builder() + .batchSize(WebBarrageSubscription.BATCH_SIZE) + .maxMessageSize(WebBarrageSubscription.MAX_MESSAGE_SIZE) + .columnConversionMode(ColumnConversionMode.Stringify) + .useDeephavenNulls(true) + .build(); + + WebBarrageSubscription snapshot = + WebBarrageSubscription.subscribe(state(), (serverViewport1, serverColumns, serverReverseViewport) -> { + }, (rowsAdded, rowsRemoved, totalMods, shifted, modifiedColumnSet) -> { }); - }).then(defer()).then(snapshot -> { - SubscriptionTableData pretendSubscription = new SubscriptionTableData(Js.uncheckedCast(columns), - state.getRowFormatColumn() == null ? NO_ROW_FORMAT_COLUMN - : state.getRowFormatColumn().getIndex(), - null); - TableData data = pretendSubscription.handleSnapshot(snapshot); - return Promise.resolve(data); - }).then(defer()); - }); - } - /** - * Instead of a micro-task between chained promises, insert a regular task so that control is returned to the - * browser long enough to prevent the UI hanging. - */ - private IThenable.ThenOnFulfilledCallbackFn defer() { - return val -> new Promise<>((resolve, reject) -> { - DomGlobal.setTimeout(ignoreArgs -> resolve.onInvoke(val), 0); + WebBarrageStreamReader reader = new WebBarrageStreamReader(); + return new Promise<>((resolve, reject) -> { + + BiDiStream doExchange = connection().streamFactory().create( + headers -> connection().flightServiceClient().doExchange(headers), + (first, headers) -> connection().browserFlightServiceClient().openDoExchange(first, headers), + (next, headers, c) -> connection().browserFlightServiceClient().nextDoExchange(next, headers, + c::apply), + new FlightData()); + MutableLong rowsReceived = new MutableLong(0); + doExchange.onData(data -> { + WebBarrageMessage message; + try { + message = reader.parseFrom(options, state().chunkTypes(), state().columnTypes(), + state().componentTypes(), data); + } catch (IOException e) { + throw new RuntimeException(e); + } + if (message != null) { + // Replace rowsets with flat versions + long resultSize = message.rowsIncluded.size(); + message.rowsAdded = RangeSet.ofRange(rowsReceived.get(), rowsReceived.get() + resultSize - 1); + message.rowsIncluded = message.rowsAdded; + rowsReceived.add(resultSize); + + // Update our table data with the complete message + snapshot.applyUpdates(message); + } + }); + FlightData payload = new FlightData(); + final FlatBufferBuilder metadata = new FlatBufferBuilder(); + + int colOffset = 0; + if (columns != null) { + colOffset = + BarrageSnapshotRequest.createColumnsVector(metadata, state().makeBitset(columns).toByteArray()); + } + int vpOffset = BarrageSnapshotRequest.createViewportVector(metadata, + serializeRanges(Collections.singleton(rows.getRange()))); + int optOffset = 0; + if (options != null) { + optOffset = options.appendTo(metadata); + } + + final int ticOffset = BarrageSnapshotRequest.createTicketVector(metadata, + Js.uncheckedCast(state().getHandle().getTicket())); + BarrageSnapshotRequest.startBarrageSnapshotRequest(metadata); + BarrageSnapshotRequest.addColumns(metadata, colOffset); + BarrageSnapshotRequest.addViewport(metadata, vpOffset); + BarrageSnapshotRequest.addSnapshotOptions(metadata, optOffset); + BarrageSnapshotRequest.addTicket(metadata, ticOffset); + BarrageSnapshotRequest.addReverseViewport(metadata, false); + metadata.finish(BarrageSnapshotRequest.endBarrageSnapshotRequest(metadata)); + + payload.setAppMetadata(WebBarrageUtils.wrapMessage(metadata, BarrageMessageType.BarrageSnapshotRequest)); + doExchange.onEnd(status -> { + if (status.isOk()) { + // notify the caller that the snapshot is finished + resolve.onInvoke(new SubscriptionEventData(snapshot, rowStyleColumn, Js.uncheckedCast(columns), + RangeSet.ofRange(0, rowsReceived.get() - 1), + RangeSet.empty(), + RangeSet.empty(), + null)); + } else { + reject.onInvoke(status); + } + }); + + doExchange.send(payload); + doExchange.end(); + }); } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/ViewportData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/ViewportData.java index 3ca96d1ed2e..04831819539 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/ViewportData.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/ViewportData.java @@ -3,558 +3,48 @@ // package io.deephaven.web.client.api.subscription; -import com.vertispan.tsdefs.annotations.TsInterface; -import com.vertispan.tsdefs.annotations.TsName; +import com.vertispan.tsdefs.annotations.TsTypeRef; import elemental2.core.JsArray; -import elemental2.core.JsObject; -import io.deephaven.web.client.api.*; -import io.deephaven.web.shared.data.*; -import jsinterop.annotations.JsFunction; +import io.deephaven.web.client.api.TableData; import jsinterop.annotations.JsProperty; -import jsinterop.base.Any; -import jsinterop.base.Js; -import jsinterop.base.JsArrayLike; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.HashSet; -import java.util.Iterator; -import java.util.PrimitiveIterator.OfLong; -import java.util.Set; +import jsinterop.annotations.JsType; /** - * Contains data in the current viewport. Also contains the offset to this data, so that the actual row number may be - * determined. Do not assume that the first row in `rows` is the first visible row, because extra rows may be provided - * for easier scrolling without going to the server. + * Extends {@link TableData}, but only contains data in the current viewport. The only API change from TableData is that + * ViewportData also contains the offset to this data, so that the actual row number may be determined. + *

+ * For viewport subscriptions, it is not necessary to read with the key, only with the position. + *

+ * Do not assume that the first row in `rows` is the first visible row, because extra rows may be provided for easier + * scrolling without going to the server. */ -@TsInterface -@TsName(namespace = "dh") -public class ViewportData implements TableData { - private static final Any NULL_SENTINEL = Js.asAny(new JsObject()); +@JsType(namespace = "dh") +public interface ViewportData extends TableData { /** - * Clean the data at the provided index - */ - @JsFunction - private interface DataCleaner { - void clean(JsArray data, int index); - } - - public static final int NO_ROW_FORMAT_COLUMN = -1; - - public class MergeResults { - public Set added = new HashSet<>(); - public Set modified = new HashSet<>(); - public Set removed = new HashSet<>(); - } - - private long offset; - private int length; - private final int maxLength; - private JsArray rows; - private final JsArray columns; - - private final Object[] data; - - private final int rowFormatColumn; - - public ViewportData(RangeSet includedRows, Object[] dataColumns, JsArray columns, int rowFormatColumn, - long maxLength) { - assert maxLength <= Integer.MAX_VALUE; - this.maxLength = (int) maxLength; - - Iterator rangeIterator = includedRows.rangeIterator(); - data = new Object[dataColumns.length]; - if (rangeIterator.hasNext()) { - Range range = rangeIterator.next(); - assert !rangeIterator.hasNext() : "Snapshot only supports one range"; - - offset = range.getFirst(); - length = (int) (range.getLast() - range.getFirst() + 1); - assert length == range.size(); - } else { - offset = -1; - } - - // Clean data for requested columns, and provide format column data as well, if any - for (int i = 0; i < columns.length; i++) { - Column c = columns.getAt(i); - int index = c.getIndex(); - if (dataColumns[index] == null) { - // no data for this column, not requested in viewport - continue; - } - data[index] = cleanData(dataColumns[index], c); - if (c.getStyleColumnIndex() != null) { - data[c.getStyleColumnIndex()] = dataColumns[c.getStyleColumnIndex()]; - } - if (c.getFormatStringColumnIndex() != null) { - data[c.getFormatStringColumnIndex()] = dataColumns[c.getFormatStringColumnIndex()]; - } - } - - // Handle row format column, if any - this.rowFormatColumn = rowFormatColumn; - if (rowFormatColumn != NO_ROW_FORMAT_COLUMN) { - data[rowFormatColumn] = dataColumns[rowFormatColumn]; - } - - // Grow all columns to match the size of the viewport, if necessary - if (length < maxLength) { - for (int i = 0; i < data.length; i++) { - if (data[i] != null) { - JsArray existingColumnData = Js.uncheckedCast(data[i]); - existingColumnData.length = this.maxLength; - existingColumnData.fill(NULL_SENTINEL, length, this.maxLength); - } - } - } - - rows = new JsArray<>(); - for (int i = 0; i < length; i++) { - rows.push(new ViewportRow(i, data, data[rowFormatColumn])); - } - this.columns = JsObject.freeze(Js.uncheckedCast(columns.slice())); - } - - private static DataCleaner getDataCleanerForColumnType(String columnType) { - switch (columnType) { - case "int": - return (data, i) -> { - int value = data.getAtAsAny(i).asInt(); - if (value == QueryConstants.NULL_INT) { - data.setAt(i, null); - } - }; - case "byte": - return (data, i) -> { - byte value = data.getAtAsAny(i).asByte(); - if (value == QueryConstants.NULL_BYTE) { - data.setAt(i, null); - } - }; - case "short": - return (data, i) -> { - short value = data.getAtAsAny(i).asShort(); - if (value == QueryConstants.NULL_SHORT) { - data.setAt(i, null); - } - }; - case "double": - return (data, i) -> { - double value = data.getAtAsAny(i).asDouble(); - if (value == QueryConstants.NULL_DOUBLE) { - data.setAt(i, null); - } - }; - case "float": - return (data, i) -> { - float value = data.getAtAsAny(i).asFloat(); - if (value == QueryConstants.NULL_FLOAT) { - data.setAt(i, null); - } - }; - case "char": - return (data, i) -> { - char value = data.getAtAsAny(i).asChar(); - if (value == QueryConstants.NULL_CHAR) { - data.setAt(i, null); - } - }; - default: - return null; - } - } - - public static Object cleanData(Object dataColumn, Column column) { - if (dataColumn == null) { - return null; - } - if (column == null) { - return dataColumn; - } - - switch (column.getType()) { - case "long": { - JsArray values = Js.uncheckedCast(dataColumn); - LongWrapper[] cleanData = new LongWrapper[values.length]; - for (int i = 0; i < values.length; i++) { - long value = values.getAtAsAny(i).asLong(); - if (value == QueryConstants.NULL_LONG) { - cleanData[i] = null; - } else { - cleanData[i] = LongWrapper.of(value); - } - } - return cleanData; - } - case "java.time.Instant": - case "java.time.ZonedDateTime": { - JsArray values = Js.uncheckedCast(dataColumn); - DateWrapper[] cleanData = new DateWrapper[values.length]; - for (int i = 0; i < values.length; i++) { - long value = values.getAtAsAny(i).asLong(); - if (value == QueryConstants.NULL_LONG) { - cleanData[i] = null; - } else { - cleanData[i] = new DateWrapper(value); - } - } - return cleanData; - } - case "java.math.BigDecimal": { - final JsArray values = Js.uncheckedCast(dataColumn); - final BigDecimalWrapper[] cleanData = new BigDecimalWrapper[values.length]; - for (int i = 0; i < values.length; i++) { - final BigDecimal value = Js.cast(values.getAt(i)); - if (value == null) { - cleanData[i] = null; - } else { - cleanData[i] = new BigDecimalWrapper(value); - } - } - return cleanData; - } - case "java.math.BigInteger": { - final JsArray values = Js.uncheckedCast(dataColumn); - final BigIntegerWrapper[] cleanData = new BigIntegerWrapper[values.length]; - for (int i = 0; i < values.length; i++) { - final BigInteger value = Js.cast(values.getAt(i)); - if (value == null) { - cleanData[i] = null; - } else { - cleanData[i] = new BigIntegerWrapper(value); - } - } - return cleanData; - } - case "java.time.LocalDate": { - final JsArray values = Js.uncheckedCast(dataColumn); - final LocalDateWrapper[] cleanData = new LocalDateWrapper[values.length]; - for (int i = 0; i < values.length; i++) { - final LocalDate value = Js.cast(values.getAt(i)); - if (value == null) { - cleanData[i] = null; - } else { - cleanData[i] = new LocalDateWrapper(value); - } - } - return cleanData; - } - case "java.time.LocalTime": { - final JsArray values = Js.uncheckedCast(dataColumn); - final LocalTimeWrapper[] cleanData = new LocalTimeWrapper[values.length]; - for (int i = 0; i < values.length; i++) { - final LocalTime value = Js.cast(values.getAt(i)); - if (value == null) { - cleanData[i] = null; - } else { - cleanData[i] = new LocalTimeWrapper(value); - } - } - return cleanData; - } - default: - DataCleaner dataCleaner = getDataCleanerForColumnType(column.getType()); - if (dataCleaner != null) { - JsArray values = Js.uncheckedCast(dataColumn); - JsArray cleanData = Js.uncheckedCast(JsArray.from((JsArrayLike) values)); - - for (int i = 0; i < values.length; i++) { - dataCleaner.clean(cleanData, i); - } - - return cleanData; - } else { - return dataColumn; - } - } - } - - /** - * The index of the first returned row - * - * @return double + * The position of the first returned row, null if this data is not for a viewport. */ @JsProperty - public double getOffset() { - return offset; - - } - - @Override - public Row get(long index) { - return getRows().getAt((int) index); - } - - @Override - public Row get(int index) { - return getRows().getAt(index); - } - - @Override - public Any getData(int index, Column column) { - return getRows().getAt(index).get(column); - } - - @Override - public Any getData(long index, Column column) { - return getRows().getAt((int) index).get(column); - } - - @Override - public Format getFormat(int index, Column column) { - return getRows().getAt(index).getFormat(column); - } + Double getOffset(); + @JsProperty @Override - public Format getFormat(long index, Column column) { - return getRows().getAt((int) index).getFormat(column); - } + JsArray getRows(); /** - * An array of rows of data - * - * @return {@link ViewportRow} array. + * Reads a row object from the viewport, based on its position in the table. */ @Override - @JsProperty - public JsArray getRows() { - if (rows.length != length) { - rows = new JsArray<>(); - for (int i = 0; i < length; i++) { - rows.push(new ViewportRow(i, data, data[rowFormatColumn])); - } - JsObject.freeze(rows); - } - return rows; + @TsTypeRef(ViewportRow.class) + default TableData.Row get(RowPositionUnion index) { + return TableData.super.get(index); } /** - * A list of columns describing the data types in each row - * - * @return {@link Column} array. + * This object may be pooled internally or discarded and not updated. Do not retain references to it. Instead, + * request the viewport again. */ - @Override - @JsProperty - public JsArray getColumns() { - return columns; - } - - public MergeResults merge(DeltaUpdates updates) { - if (offset == -1 && updates.getIncludedAdditions().size() > 0) { - offset = updates.getIncludedAdditions().getFirstRow(); - } - final MergeResults updated = new MergeResults(); - - // First we remove rows by nulling them out. - updates.getRemoved().indexIterator().forEachRemaining((long removedIndex) -> { - int internalOffset = (int) (removedIndex - offset); - if (internalOffset < 0 || internalOffset >= length) { - return; - } - for (int i = 0; i < data.length; i++) { - JsArray existingColumnData = Js.uncheckedCast(data[i]); - if (existingColumnData == null) { - continue; - } - existingColumnData.setAt(internalOffset, NULL_SENTINEL); - } - updated.removed.add(internalOffset); - }); - - // Now we shift data around. - boolean hasReverseShift = false; - final ShiftedRange[] shiftedRanges = updates.getShiftedRanges(); - - // must apply shifts in mem-move semantics; so we shift forward from right to left first - for (int si = shiftedRanges.length - 1; si >= 0; --si) { - final ShiftedRange shiftedRange = shiftedRanges[si]; - final long shiftDelta = shiftedRange.getDelta(); - if (shiftDelta < 0) { - hasReverseShift = true; - continue; - } - - final long beginAsLong = Math.max(shiftedRange.getRange().getFirst() - offset, 0); - final int end = (int) Math.min(shiftedRange.getRange().getLast() - offset, length - 1); - if (end < beginAsLong) { - // this range is out of our viewport - continue; - } - - // long math is expensive; so convert to int early/once - final int begin = (int) beginAsLong; - - // iterate backward and move them forward - for (int j = end; j >= begin; --j) { - for (int i = 0; i < data.length; ++i) { - final JsArray existingColumnData = Js.uncheckedCast(data[i]); - if (existingColumnData == null) { - continue; - } - - final long internalOffsetAsLong = (j + shiftDelta); - if (internalOffsetAsLong >= 0 && internalOffsetAsLong < maxLength) { - // because internalOffsetAsLong is less than maxLen; we know it must be fit in an int - final int internalOffset = (int) internalOffsetAsLong; - updated.added.add(internalOffset); - Any toMove = existingColumnData.getAt(j); - existingColumnData.setAt(internalOffset, toMove); - } - - updated.removed.add(j); - existingColumnData.setAt(j, NULL_SENTINEL); - } - } - } - if (hasReverseShift) { - // then we shift in reverse from left to right - for (int si = 0; si < shiftedRanges.length; ++si) { - final ShiftedRange shiftedRange = shiftedRanges[si]; - final long shiftDelta = shiftedRange.getDelta(); - if (shiftDelta > 0) { - continue; - } - - final long begin = Math.max(shiftedRange.getRange().getFirst() - offset, 0); - final int end = (int) Math.min(shiftedRange.getRange().getLast() - offset, length - 1); - if (end < begin) { - // this range is out of our viewport - continue; - } - - // iterate forward and move them backward (note: since begin is <= end, we now know it fits in an int) - for (int j = (int) begin; j <= end; ++j) { - for (int i = 0; i < data.length; ++i) { - final JsArray existingColumnData = Js.uncheckedCast(data[i]); - if (existingColumnData == null) { - continue; - } - - final long internalOffsetAsLong = j + shiftDelta; - if (internalOffsetAsLong >= 0 && internalOffsetAsLong < maxLength) { - // because internalOffsetAsLong is less than maxLen; we know it must be fit in an int - final int internalOffset = (int) internalOffsetAsLong; - updated.added.add(internalOffset); - existingColumnData.setAt(internalOffset, existingColumnData.getAt(j)); - } - - updated.removed.add(j); - existingColumnData.setAt(j, NULL_SENTINEL); - } - } - } - } - - DeltaUpdates.ColumnModifications[] serializedModifications = updates.getSerializedModifications(); - for (int modifiedColIndex = 0; modifiedColIndex < serializedModifications.length; modifiedColIndex++) { - final DeltaUpdates.ColumnModifications modifiedColumn = serializedModifications[modifiedColIndex]; - final OfLong it = modifiedColumn == null ? null : modifiedColumn.getRowsIncluded().indexIterator(); - - if (it == null || !it.hasNext()) { - continue; - } - - // look for a local Column which matches this index so we know how to clean it - final Column column = columns.find((c, i1, i2) -> c.getIndex() == modifiedColumn.getColumnIndex()); - final JsArray updatedColumnData = - Js.uncheckedCast(cleanData(modifiedColumn.getValues().getData(), column)); - final JsArray existingColumnData = Js.uncheckedCast(data[modifiedColumn.getColumnIndex()]); - if (updatedColumnData.length == 0) { - continue; - } - - // for each change provided for this column, replace the values in our store - int i = 0; - while (it.hasNext()) { - long modifiedOffset = it.nextLong(); - int internalOffset = (int) (modifiedOffset - offset); - if (internalOffset < 0 || internalOffset >= maxLength) { - i++; - continue;// data we don't need to see, either meant for another table, or we just sent a viewport - // update - } - existingColumnData.setAt(internalOffset, updatedColumnData.getAtAsAny(i)); - updated.modified.add(internalOffset); - i++; - } - } - - if (!updates.getIncludedAdditions().isEmpty()) { - DeltaUpdates.ColumnAdditions[] serializedAdditions = updates.getSerializedAdditions(); - for (int addedColIndex = 0; addedColIndex < serializedAdditions.length; addedColIndex++) { - DeltaUpdates.ColumnAdditions addedColumn = serializedAdditions[addedColIndex]; - - Column column = columns.find((c, i1, i2) -> c.getIndex() == addedColumn.getColumnIndex()); - final JsArray addedColumnData = - Js.uncheckedCast(cleanData(addedColumn.getValues().getData(), column)); - final JsArray existingColumnData = Js.uncheckedCast(data[addedColumn.getColumnIndex()]); - if (addedColumnData.length == 0) { - continue; - } - - int i = 0; - OfLong it = updates.getIncludedAdditions().indexIterator(); - while (it.hasNext()) { - long addedOffset = it.nextLong(); - int internalOffset = (int) (addedOffset - offset); - if (internalOffset < 0 || internalOffset >= maxLength) { - i++; - continue;// data we don't need to see, either meant for another table, or we just sent a - // viewport update - } - assert internalOffset < existingColumnData.length; - - Any existing = existingColumnData.getAt(internalOffset); - if (existing == NULL_SENTINEL || internalOffset >= length) { - // space was set aside or was left at the end of the array for this value, it is a new addition - updated.added.add(internalOffset); - } else { - // we're overwriting some existing value - updated.modified.add(internalOffset); - } - existingColumnData.setAt(internalOffset, addedColumnData.getAtAsAny(i)); - i++; - } - } - } - - // exclude added items from being marked as modified, since we're hiding shifts from api consumers - updated.modified.removeAll(updated.added); - - // Any position which was both added and removed should instead be marked as modified, this cleans - // up anything excluded above that didn't otherwise make sense - for (Iterator it = updated.removed.iterator(); it.hasNext();) { - int ii = it.next(); - if (updated.added.remove(ii)) { - it.remove(); - updated.modified.add(ii); - } - } - - length = length + updated.added.size() - updated.removed.size(); - assert 0 <= length && length <= maxLength; - - // Viewport footprint should be small enough that we can afford to see if this update corrupted our view of the - // world: - assert !dataContainsNullSentinels(); - - return updated; - } - - private boolean dataContainsNullSentinels() { - for (int i = 0; i < data.length; i++) { - JsArray existingColumnData = Js.uncheckedCast(data[i]); - if (existingColumnData == null) { - continue; - } - for (int j = 0; j < length; ++j) { - if (existingColumnData.getAt(j) == NULL_SENTINEL) { - return true; - } - } - } - return false; + @JsType(namespace = "dh") + interface ViewportRow extends TableData.Row { } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/ViewportRow.java b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/ViewportRow.java deleted file mode 100644 index ade086f4503..00000000000 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/ViewportRow.java +++ /dev/null @@ -1,86 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.client.api.subscription; - -import com.vertispan.tsdefs.annotations.TsInterface; -import com.vertispan.tsdefs.annotations.TsName; -import elemental2.core.JsArray; -import io.deephaven.web.client.api.Column; -import io.deephaven.web.client.api.Format; -import io.deephaven.web.client.api.LongWrapper; -import io.deephaven.web.client.api.TableData; -import jsinterop.annotations.JsMethod; -import jsinterop.base.Any; -import jsinterop.base.Js; - -/** - * This object may be pooled internally or discarded and not updated. Do not retain references to it. Instead, request - * the viewport again. - */ -@TsInterface -@TsName(namespace = "dh") -public class ViewportRow implements TableData.Row { - protected final int offsetInSnapshot; - private final Object[] dataColumns; - private final JsArray rowStyleColumn; - - public ViewportRow(int offsetInSnapshot, Object[] dataColumns, Object rowStyleColumn) { - this.offsetInSnapshot = offsetInSnapshot; - this.dataColumns = dataColumns; - this.rowStyleColumn = Js.uncheckedCast(rowStyleColumn); - } - - @Override - public LongWrapper getIndex() { - throw new UnsupportedOperationException("Viewports don't currently represent their position with an index"); - } - - /** - * the data for the given column's cell - * - * @param column - * @return Any - */ - @Override - @JsMethod - public Any get(Column column) { - JsArray uncheckedData = Js.uncheckedCast(dataColumns[column.getIndex()]); - if (uncheckedData == null) { - throw new java.util.NoSuchElementException( - "Column " + column.getName() + " not found in row, was it specified in the viewport?"); - } - return uncheckedData.getAtAsAny(offsetInSnapshot); - } - - /** - * the format object for the given columns' cell - * - * @param column - * @return {@link Format}. - */ - @Override - @JsMethod - public Format getFormat(Column column) { - long cellColors = 0; - long rowColors = 0; - String numberFormat = null; - String formatString = null; - if (column.getStyleColumnIndex() != null) { - JsArray colors = Js.uncheckedCast(dataColumns[column.getStyleColumnIndex()]); - cellColors = colors.getAtAsAny(offsetInSnapshot).asLong(); - } - if (rowStyleColumn != null) { - rowColors = rowStyleColumn.getAtAsAny(offsetInSnapshot).asLong(); - } - if (column.getFormatColumnIndex() != null) { - JsArray formatStrings = Js.uncheckedCast(dataColumns[column.getFormatColumnIndex()]); - numberFormat = formatStrings.getAtAsAny(offsetInSnapshot).asString(); - } - if (column.getFormatStringColumnIndex() != null) { - JsArray formatStrings = Js.uncheckedCast(dataColumns[column.getFormatStringColumnIndex()]); - formatString = formatStrings.getAtAsAny(offsetInSnapshot).asString(); - } - return new Format(cellColors, rowColors, numberFormat, formatString); - } -} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/tree/JsTreeTable.java b/web/client-api/src/main/java/io/deephaven/web/client/api/tree/JsTreeTable.java index 50808f9776d..72e7ea11116 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/tree/JsTreeTable.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/tree/JsTreeTable.java @@ -3,55 +3,44 @@ // package io.deephaven.web.client.api.tree; -import com.vertispan.tsdefs.annotations.TsInterface; -import com.vertispan.tsdefs.annotations.TsName; +import com.vertispan.tsdefs.annotations.TsIgnore; +import com.vertispan.tsdefs.annotations.TsTypeRef; import com.vertispan.tsdefs.annotations.TsUnion; import com.vertispan.tsdefs.annotations.TsUnionMember; import elemental2.core.JsArray; import elemental2.core.JsObject; import elemental2.core.Uint8Array; +import elemental2.dom.CustomEvent; import elemental2.dom.CustomEventInit; import elemental2.dom.DomGlobal; import elemental2.promise.IThenable; import elemental2.promise.Promise; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf.Message; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf.MessageHeader; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf.RecordBatch; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Schema; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightData; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageMessageType; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageMessageWrapper; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageSubscriptionOptions; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageSubscriptionRequest; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.BarrageUpdateMetadata; -import io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf.ColumnConversionMode; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.hierarchicaltable_pb.HierarchicalTableApplyRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.hierarchicaltable_pb.HierarchicalTableDescriptor; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.hierarchicaltable_pb.HierarchicalTableSourceExportRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.hierarchicaltable_pb.HierarchicalTableViewKeyTableDescriptor; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.hierarchicaltable_pb.HierarchicalTableViewRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.table_pb.Condition; +import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.table_pb.ExportedTableCreationResponse; +import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.table_pb.TableReference; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.ticket_pb.Ticket; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.ticket_pb.TypedTicket; import io.deephaven.web.client.api.*; import io.deephaven.web.client.api.barrage.WebBarrageUtils; +import io.deephaven.web.client.api.barrage.data.WebBarrageSubscription; import io.deephaven.web.client.api.barrage.def.ColumnDefinition; import io.deephaven.web.client.api.barrage.def.InitialTableDefinition; -import io.deephaven.web.client.api.barrage.stream.BiDiStream; +import io.deephaven.web.client.api.barrage.stream.ResponseStreamWrapper; import io.deephaven.web.client.api.filter.FilterCondition; import io.deephaven.web.client.api.impl.TicketAndPromise; import io.deephaven.web.client.api.lifecycle.HasLifecycle; -import io.deephaven.web.client.api.subscription.ViewportData; -import io.deephaven.web.client.api.subscription.ViewportRow; -import io.deephaven.web.client.api.tree.JsTreeTable.TreeViewportData.TreeRow; +import io.deephaven.web.client.api.subscription.AbstractTableSubscription; import io.deephaven.web.client.api.widget.JsWidget; import io.deephaven.web.client.fu.JsItr; import io.deephaven.web.client.fu.JsLog; import io.deephaven.web.client.fu.LazyPromise; +import io.deephaven.web.client.state.ClientTableState; import io.deephaven.web.shared.data.*; -import io.deephaven.web.shared.data.columns.ColumnData; import javaemul.internal.annotations.DoNotAutobox; import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsNullable; @@ -67,10 +56,6 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static io.deephaven.web.client.api.barrage.WebBarrageUtils.makeUint8ArrayFromBitset; -import static io.deephaven.web.client.api.barrage.WebBarrageUtils.serializeRanges; -import static io.deephaven.web.client.api.subscription.ViewportData.NO_ROW_FORMAT_COLUMN; - /** * Behaves like a {@link JsTable} externally, but data, state, and viewports are managed by an entirely different * mechanism, and so reimplemented here. @@ -108,7 +93,7 @@ * roll-up table, the totals only include leaf nodes (as non-leaf nodes are generated through grouping the contents of * the original table). Roll-ups also have the {@link JsRollupConfig#includeConstituents} property, indicating that a * {@link Column} in the tree may have a {@link Column#getConstituentType()} property reflecting that the type of cells - * where {@link TreeRow#hasChildren()} is false will be different from usual. + * where {@link TreeSubscription.TreeRowImpl#hasChildren()} is false will be different from usual. * */ @JsType(namespace = "dh", name = "TreeTable") @@ -126,216 +111,6 @@ public class JsTreeTable extends HasLifecycle implements ServerObject { private static final double ACTION_EXPAND_WITH_DESCENDENTS = 0b011; private static final double ACTION_COLLAPSE = 0b100; - @TsInterface - @TsName(namespace = "dh") - public class TreeViewportData implements TableData { - private final Boolean[] expandedColumn; - private final int[] depthColumn; - private final double offset; - private final double treeSize; - - private final JsArray columns; - private final JsArray rows; - - private TreeViewportData(double offset, long viewportSize, double treeSize, ColumnData[] dataColumns, - Column[] columns) { - this.offset = offset; - this.treeSize = treeSize; - this.columns = JsObject.freeze(Js.cast(Js.>uncheckedCast(columns).slice())); - - // Unlike ViewportData, assume that we own this copy of the data and can mutate at will. As such, - // we'll just clean the data that the requested columns know about for now. - // TODO to improve this, we can have synthetic columns to handle data that wasn't requested/expected, - // and then can share code with ViewportData - Object[] data = new Object[dataColumns.length]; - - expandedColumn = Js.uncheckedCast( - ViewportData.cleanData(dataColumns[rowExpandedCol.getIndex()].getData(), rowExpandedCol)); - depthColumn = Js.uncheckedCast( - ViewportData.cleanData(dataColumns[rowDepthCol.getIndex()].getData(), rowDepthCol)); - - int constituentDepth = keyColumns.length + 2; - - // Without modifying this.columns (copied and frozen), make sure our key columns are present - // in the list of columns that we will copy data for the viewport - keyColumns.forEach((col, p1) -> { - if (this.columns.indexOf(col) == -1) { - columns[columns.length] = col; - } - return null; - }); - - for (int i = 0; i < columns.length; i++) { - Column c = columns[i]; - int index = c.getIndex(); - - // clean the data, since it will be exposed to the client - data[index] = ViewportData.cleanData(dataColumns[index].getData(), c); - if (c.getStyleColumnIndex() != null) { - data[c.getStyleColumnIndex()] = dataColumns[c.getStyleColumnIndex()].getData(); - } - if (c.getFormatStringColumnIndex() != null) { - data[c.getFormatStringColumnIndex()] = dataColumns[c.getFormatStringColumnIndex()].getData(); - } - - // if there is a matching constituent column array, clean it and copy from it - Column sourceColumn = sourceColumns.get(c.getName()); - if (sourceColumn != null) { - ColumnData constituentColumn = dataColumns[sourceColumn.getIndex()]; - if (constituentColumn != null) { - JsArray cleanConstituentColumn = - Js.uncheckedCast(ViewportData.cleanData(constituentColumn.getData(), sourceColumn)); - // Overwrite the data with constituent values, if any - // We use cleanConstituentColumn to find max item rather than data[index], since we - // are okay stopping at the last constituent value, in case the server sends shorter - // arrays. - for (int rowIndex = 0; rowIndex < cleanConstituentColumn.length; rowIndex++) { - if (depthColumn[rowIndex] == constituentDepth) - Js.asArrayLike(data[index]).setAt(rowIndex, cleanConstituentColumn.getAt(rowIndex)); - } - - if (sourceColumn.getStyleColumnIndex() != null) { - assert c.getStyleColumnIndex() != null; - ColumnData styleData = dataColumns[sourceColumn.getStyleColumnIndex()]; - if (styleData != null) { - JsArray styleArray = Js.cast(styleData.getData()); - for (int rowIndex = 0; rowIndex < styleArray.length; rowIndex++) { - if (depthColumn[rowIndex] == constituentDepth) - Js.asArrayLike(data[c.getStyleColumnIndex()]).setAt(rowIndex, - styleArray.getAt(rowIndex)); - } - } - } - if (sourceColumn.getFormatStringColumnIndex() != null) { - assert c.getFormatStringColumnIndex() != null; - ColumnData formatData = dataColumns[sourceColumn.getFormatStringColumnIndex()]; - if (formatData != null) { - JsArray formatArray = Js.cast(formatData.getData()); - for (int rowIndex = 0; rowIndex < formatArray.length; rowIndex++) { - if (depthColumn[rowIndex] == constituentDepth) { - Js.asArrayLike(data[c.getFormatStringColumnIndex()]).setAt(rowIndex, - formatArray.getAt(rowIndex)); - } - } - } - } - } - } - } - if (rowFormatColumn != NO_ROW_FORMAT_COLUMN) { - data[rowFormatColumn] = dataColumns[rowFormatColumn].getData(); - } - - rows = new JsArray<>(); - for (int i = 0; i < viewportSize; i++) { - rows.push(new TreeRow(i, data, data[rowFormatColumn])); - } - } - - @Override - public Row get(long index) { - return getRows().getAt((int) index); - } - - @Override - public Row get(int index) { - return getRows().getAt((int) index); - } - - @Override - public Any getData(int index, Column column) { - return getRows().getAt(index).get(column); - } - - @Override - public Any getData(long index, Column column) { - return getRows().getAt((int) index).get(column); - } - - @Override - public Format getFormat(int index, Column column) { - return getRows().getAt(index).getFormat(column); - } - - @Override - public Format getFormat(long index, Column column) { - return getRows().getAt((int) index).getFormat(column); - } - - @JsProperty - public double getOffset() { - return offset; - } - - @JsProperty - public JsArray getColumns() { - return columns; - } - - @JsProperty - public JsArray getRows() { - return rows; - } - - public double getTreeSize() { - return treeSize; - } - - /** - * Row implementation that also provides additional read-only properties. represents visible rows in the table, - * but with additional properties to reflect the tree structure. - */ - @TsInterface - @TsName(namespace = "dh") - public class TreeRow extends ViewportRow { - public TreeRow(int offsetInSnapshot, Object[] dataColumns, Object rowStyleColumn) { - super(offsetInSnapshot, dataColumns, rowStyleColumn); - } - - /** - * True if this node is currently expanded to show its children; false otherwise. Those children will be the - * rows below this one with a greater depth than this one - * - * @return boolean - */ - @JsProperty(name = "isExpanded") - public boolean isExpanded() { - return expandedColumn[offsetInSnapshot] == Boolean.TRUE; - } - - /** - * True if this node has children and can be expanded; false otherwise. Note that this value may change when - * the table updates, depending on the table's configuration - * - * @return boolean - */ - @JsProperty(name = "hasChildren") - public boolean hasChildren() { - return expandedColumn[offsetInSnapshot] != null; - } - - /** - * The number of levels above this node; zero for top level nodes. Generally used by the UI to indent the - * row and its expand/collapse icon - * - * @return int - */ - @JsProperty(name = "depth") - public int depth() { - return depthColumn[offsetInSnapshot]; - } - - public void appendKeyData(Object[][] keyTableData, double action) { - int i; - for (i = 0; i < keyColumns.length; i++) { - Js.>cast(keyTableData[i]).push(keyColumns.getAt(i).get(this)); - } - Js.>cast(keyTableData[i++]).push((double) depth()); - Js.>cast(keyTableData[i++]).push(action); - } - } - } - /** * Ordered series of steps that must be performed when changes are made to the table. When any change is applied, * all subsequent steps must be performed as well. @@ -352,7 +127,6 @@ private enum RebuildStep { private final InitialTableDefinition tableDefinition; private final Column[] visibleColumns; private final Map columnsByName = new HashMap<>(); - private final int rowFormatColumn; private final Map sourceColumns; private final JsArray keyColumns = new JsArray<>(); private Column rowDepthCol; @@ -376,7 +150,7 @@ private enum RebuildStep { private Promise keyTable; private TicketAndPromise viewTicket; - private Promise> stream; + private Promise stream; // the "next" set of filters/sorts that we'll use. these either are "==" to the above fields, or are scheduled // to replace them soon. @@ -389,7 +163,7 @@ private enum RebuildStep { private Column[] columns; private int updateInterval = 1000; - private TreeViewportData currentViewportData; + private TreeSubscription.TreeViewportDataImpl currentViewportData; private boolean alwaysFireNextEvent = false; @@ -408,10 +182,9 @@ public JsTreeTable(WorkerConnection workerConnection, JsWidget widget) { HierarchicalTableDescriptor.deserializeBinary(widget.getDataAsU8()); Uint8Array flightSchemaMessage = treeDescriptor.getSnapshotSchema_asU8(); - Schema schema = WebBarrageUtils.readSchemaMessage(flightSchemaMessage); this.isRefreshing = !treeDescriptor.getIsStatic(); - this.tableDefinition = WebBarrageUtils.readTableDefinition(schema); + this.tableDefinition = WebBarrageUtils.readTableDefinition(flightSchemaMessage); Column[] columns = new Column[0]; Map> columnDefsByName = tableDefinition.getColumnsByName(); int rowFormatColumn = -1; @@ -457,7 +230,6 @@ public JsTreeTable(WorkerConnection workerConnection, JsWidget widget) { columnDefsByName.get(true).get(definition.getRollupAggregationInputColumn()).getType()); } } - this.rowFormatColumn = rowFormatColumn; this.groupedColumns = JsObject.freeze(groupedColumns); sourceColumns = columnDefsByName.get(false).values().stream() @@ -602,6 +374,212 @@ private TicketAndPromise makeView(TicketAndPromise prevTicket) { return viewTicket; } + private int constituentDepth() { + return keyColumns.length + 2; + } + + public class TreeSubscription extends AbstractTableSubscription { + @TsIgnore + public class TreeViewportDataImpl extends AbstractTableSubscription.UpdateEventData + implements TreeViewportData { + private final double treeSize; + + private final JsArray columns; + + private TreeViewportDataImpl(WebBarrageSubscription subscription, int rowStyleColumn, + JsArray columns, + RangeSet added, RangeSet removed, RangeSet modified, ShiftedRange[] shifted) { + super(subscription, rowStyleColumn, columns, added, removed, modified, shifted); + + this.treeSize = barrageSubscription.getCurrentRowSet().size(); + this.columns = JsObject.freeze(Js.cast(Js.>uncheckedCast(columns).slice())); + } + + @Override + public Any getData(int index, Column column) { + Column sourceColumn = sourceColumns.get(column.getName()); + if (sourceColumn == null) { + // no constituent column, call super + return super.getData(index, column); + } + if (barrageSubscription.getData(index, rowDepthCol.getIndex()).asInt() != constituentDepth()) { + // not at constituent depth, call super + return super.getData(index, column); + } + // read source col instead + return super.getData(index, sourceColumn); + } + + @Override + public Any getData(long index, Column column) { + Column sourceColumn = sourceColumns.get(column.getName()); + if (sourceColumn == null) { + // no constituent column, call super + return super.getData(index, column); + } + if (barrageSubscription.getData(index, rowDepthCol.getIndex()).asInt() != constituentDepth()) { + // not at constituent depth, call super + return super.getData(index, column); + } + // read source col instead + return super.getData(index, sourceColumn); + } + + @Override + public Format getFormat(int index, Column column) { + Column sourceColumn = sourceColumns.get(column.getName()); + if (sourceColumn == null) { + // no constituent column, call super + return super.getFormat(index, column); + } + if (barrageSubscription.getData(index, rowDepthCol.getIndex()).asInt() != constituentDepth()) { + // not at constituent depth, call super + return super.getFormat(index, column); + } + // read source col instead + return super.getFormat(index, sourceColumn); + } + + @Override + public Format getFormat(long index, Column column) { + Column sourceColumn = sourceColumns.get(column.getName()); + if (sourceColumn == null) { + // no constituent column, call super + return super.getFormat(index, column); + } + if (barrageSubscription.getData(index, rowDepthCol.getIndex()).asInt() != constituentDepth()) { + // not at constituent depth, call super + return super.getFormat(index, column); + } + // read source col instead + return super.getFormat(index, sourceColumn); + } + + @Override + public JsArray getColumns() { + // This looks like its superclass, but we're actually returning a different field + return columns; + } + + @Override + protected SubscriptionRow makeRow(long index) { + return new TreeRowImpl(subscription, index); + } + + @JsProperty + public double getTreeSize() { + return treeSize; + } + } + + public class TreeRowImpl extends SubscriptionRow implements TreeViewportData.TreeRow { + + public TreeRowImpl(WebBarrageSubscription subscription, long index) { + super(subscription, rowStyleColumn, index); + } + + @Override + public boolean isExpanded() { + return barrageSubscription.getData(index, rowExpandedCol.getIndex()).uncheckedCast() == Boolean.TRUE; + } + + @Override + public boolean hasChildren() { + return barrageSubscription.getData(index, rowExpandedCol.getIndex()).uncheckedCast() != null; + } + + @Override + public int depth() { + return Js.coerceToInt(barrageSubscription.getData(index, rowDepthCol.getIndex())); + } + + public void appendKeyData(Object[][] keyTableData, double action) { + int i; + for (i = 0; i < keyColumns.length; i++) { + Js.>cast(keyTableData[i]).push(keyColumns.getAt(i).get(this)); + } + Js.>cast(keyTableData[i++]).push((double) depth()); + Js.>cast(keyTableData[i++]).push(action); + } + + @Override + public Any get(Column column) { + Column sourceColumn = sourceColumns.get(column.getName()); + if (sourceColumn == null) { + // no constituent column, call super + return super.get(column); + } + if (barrageSubscription.getData(index, rowDepthCol.getIndex()).asInt() != constituentDepth()) { + // not at constituent depth, call super + return super.get(column); + } + // read source col instead + return super.get(sourceColumn); + } + + @Override + public Format getFormat(Column column) { + Column sourceColumn = sourceColumns.get(column.getName()); + if (sourceColumn == null) { + // no constituent column, call super + return super.getFormat(column); + } + if (barrageSubscription.getData(index, rowDepthCol.getIndex()).asInt() != constituentDepth()) { + // not at constituent depth, call super + return super.getFormat(column); + } + // read source col instead + return super.getFormat(sourceColumn); + } + } + + private RangeSet serverViewport; + + public TreeSubscription(ClientTableState state, WorkerConnection connection) { + super(state, connection); + } + + @Override + protected void sendFirstSubscriptionRequest() { + setViewport(firstRow, lastRow, Js.uncheckedCast(columns), (double) updateInterval); + } + + @Override + protected BitSet makeColumnBitset(JsArray columns) { + BitSet requested = super.makeColumnBitset(columns); + requested.or(makeColumnSubscriptionBitset()); + return requested; + } + + @Override + protected void onStreamEnd(ResponseStreamWrapper.Status status) { + super.onStreamEnd(status); + JsTreeTable.this.stream = null; + if (!status.isOk()) { + failureHandled(status.getDetails()); + } + } + + public void setViewport(double firstRow, double lastRow, JsArray columns, Double updateInterval) { + serverViewport = RangeSet.ofRange((long) firstRow, (long) lastRow); + + sendBarrageSubscriptionRequest(RangeSet.ofRange((long) firstRow, (long) lastRow), Js.uncheckedCast(columns), + updateInterval, false); + } + + @Override + protected void notifyUpdate(RangeSet rowsAdded, RangeSet rowsRemoved, RangeSet totalMods, + ShiftedRange[] shifted) { + TreeViewportDataImpl detail = + new TreeViewportDataImpl(barrageSubscription, rowStyleColumn, getColumns(), rowsAdded, + rowsRemoved, totalMods, shifted); + detail.setOffset(this.serverViewport.getFirstRow()); + CustomEventInit event = CustomEventInit.create(); + event.setDetail(detail); + fireEvent(EVENT_UPDATED, event); + } + } + private void replaceSubscription(RebuildStep step) { // Perform steps required to remove the existing intermediate tickets. // Fall-through between steps is deliberate. @@ -621,18 +599,28 @@ private void replaceSubscription(RebuildStep step) { viewTicket.release(); viewTicket = null; } - case SUBSCRIPTION: + + // In all of the above cases, we replace the subscription if (stream != null) { stream.then(stream -> { - stream.end(); - stream.cancel(); + stream.close(); return null; }); stream = null; } + break; + case SUBSCRIPTION: + // If it exists, adjust the existing subscription, otherwise create a new one + if (stream != null) { + stream.then(subscription -> { + subscription.setViewport(firstRow, lastRow, Js.uncheckedCast(columns), (double) updateInterval); + return null; + }); + return; + } } - Promise> stream = Promise.resolve(defer()) + Promise stream = Promise.resolve(defer()) .then(ignore -> { makeKeyTable(); TicketAndPromise filter = prepareFilter(); @@ -648,8 +636,6 @@ private void replaceSubscription(RebuildStep step) { BitSet columnsBitset = makeColumnSubscriptionBitset(); RangeSet range = RangeSet.ofRange((long) (double) firstRow, (long) (double) lastRow); - Column[] queryColumns = this.columns; - boolean alwaysFireEvent = this.alwaysFireNextEvent; this.alwaysFireNextEvent = false; @@ -658,91 +644,29 @@ private void replaceSubscription(RebuildStep step) { columnsBitset, range, alwaysFireEvent); - BiDiStream doExchange = - connection.streamFactory().create( - headers -> connection.flightServiceClient().doExchange(headers), - (first, headers) -> connection.browserFlightServiceClient().openDoExchange(first, - headers), - (next, headers, c) -> connection.browserFlightServiceClient().nextDoExchange(next, - headers, - c::apply), - new FlightData()); - - FlightData subscriptionRequestWrapper = new FlightData(); - Builder doGetRequest = new Builder(1024); - double columnsOffset = BarrageSubscriptionRequest.createColumnsVector(doGetRequest, - makeUint8ArrayFromBitset(columnsBitset)); - double viewportOffset = BarrageSubscriptionRequest.createViewportVector(doGetRequest, - serializeRanges( - Collections.singleton( - range))); - double serializationOptionsOffset = BarrageSubscriptionOptions - .createBarrageSubscriptionOptions(doGetRequest, ColumnConversionMode.Stringify, true, - updateInterval, 0, 0); - double tableTicketOffset = - BarrageSubscriptionRequest.createTicketVector(doGetRequest, - viewTicket.ticket().getTicket_asU8()); - BarrageSubscriptionRequest.startBarrageSubscriptionRequest(doGetRequest); - BarrageSubscriptionRequest.addTicket(doGetRequest, tableTicketOffset); - BarrageSubscriptionRequest.addColumns(doGetRequest, columnsOffset); - BarrageSubscriptionRequest.addSubscriptionOptions(doGetRequest, serializationOptionsOffset); - BarrageSubscriptionRequest.addViewport(doGetRequest, viewportOffset); - doGetRequest.finish(BarrageSubscriptionRequest.endBarrageSubscriptionRequest(doGetRequest)); - - subscriptionRequestWrapper.setAppMetadata( - WebBarrageUtils.wrapMessage(doGetRequest, BarrageMessageType.BarrageSubscriptionRequest)); - doExchange.send(subscriptionRequestWrapper); - - String[] columnTypes = Arrays.stream(tableDefinition.getColumns()) - .map(ColumnDefinition::getType) - .toArray(String[]::new); - doExchange.onStatus(status -> { - if (!status.isOk()) { - failureHandled(status.getDetails()); - } - }); - doExchange.onEnd(status -> { - this.stream = null; - }); - doExchange.onData(flightData -> { - Message message = Message.getRootAsMessage(new ByteBuffer(flightData.getDataHeader_asU8())); - if (message.headerType() == MessageHeader.Schema) { - // ignore for now, we'll handle this later - return; - } - assert message.headerType() == MessageHeader.RecordBatch; - RecordBatch header = message.header(new RecordBatch()); - Uint8Array appMetadataBytes = flightData.getAppMetadata_asU8(); - BarrageUpdateMetadata update = null; - if (appMetadataBytes.length != 0) { - BarrageMessageWrapper barrageMessageWrapper = - BarrageMessageWrapper.getRootAsBarrageMessageWrapper( - new ByteBuffer( - appMetadataBytes)); - - update = BarrageUpdateMetadata.getRootAsBarrageUpdateMetadata( - new ByteBuffer( - new Uint8Array(barrageMessageWrapper.msgPayloadArray()))); - } - TableSnapshot snapshot = WebBarrageUtils.createSnapshot(header, - WebBarrageUtils.typedArrayToLittleEndianByteBuffer(flightData.getDataBody_asU8()), - update, - true, - columnTypes); - - final RangeSet includedRows = snapshot.getIncludedRows(); - double offset = firstRow; - assert includedRows.isEmpty() || Js.asInt(offset) == includedRows.getFirstRow(); - TreeViewportData vd = new TreeViewportData( - offset, - includedRows.isEmpty() ? 0 : includedRows.size(), - snapshot.getTableSize(), - snapshot.getDataColumns(), - queryColumns); - - handleUpdate(nextSort, nextFilters, vd, alwaysFireEvent); - }); - return Promise.resolve(doExchange); + + ClientTableState state = new ClientTableState(connection, + new TableTicket(viewTicket.ticket().getTicket_asU8()), (callback, newState, metadata) -> { + callback.apply("fail, trees dont reconnect like this", null); + }, ""); + ExportedTableCreationResponse def = new ExportedTableCreationResponse(); + HierarchicalTableDescriptor treeDescriptor = + HierarchicalTableDescriptor.deserializeBinary(widget.getDataAsU8()); + def.setSchemaHeader(treeDescriptor.getSnapshotSchema_asU8()); + def.setResultId(new TableReference()); + def.getResultId().setTicket(viewTicket.ticket()); + state.applyTableCreationResponse(def); + + TreeSubscription subscription = new TreeSubscription(state, connection); + + subscription.addEventListener(TreeSubscription.EVENT_UPDATED, + (CustomEvent data) -> { + TreeSubscription.TreeViewportDataImpl detail = + (TreeSubscription.TreeViewportDataImpl) data.detail; + + handleUpdate(nextSort, nextFilters, detail, alwaysFireEvent); + }); + return Promise.resolve(subscription); }); stream.catch_(err -> { // if this is the active attempt at a subscription, report the error @@ -765,7 +689,7 @@ private IThenable.ThenOnFulfilledCallbackFn defer() { } private void handleUpdate(List nextSort, List nextFilters, - TreeViewportData viewportData, boolean alwaysFireEvent) { + TreeSubscription.TreeViewportDataImpl viewportData, boolean alwaysFireEvent) { JsLog.debug("tree table response arrived", viewportData); if (closed) { // ignore @@ -781,7 +705,7 @@ private void handleUpdate(List nextSort, List nextFilters this.filters = nextFilters; if (fireEvent) { - CustomEventInit updatedEvent = CustomEventInit.create(); + CustomEventInit updatedEvent = CustomEventInit.create(); updatedEvent.setDetail(viewportData); fireEvent(EVENT_UPDATED, updatedEvent); } @@ -863,7 +787,7 @@ static RowReferenceUnion of(@DoNotAutobox Object o) { @JsOverlay default boolean isTreeRow() { - return this instanceof TreeRow; + return this instanceof TreeSubscription.TreeRowImpl; } @JsOverlay @@ -873,7 +797,7 @@ default boolean isNumber() { @JsOverlay @TsUnionMember - default TreeRow asTreeRow() { + default TreeViewportData.TreeRow asTreeRow() { return Js.cast(this); } @@ -904,11 +828,11 @@ public void setExpanded(RowReferenceUnion row, boolean isExpanded, @JsOptional B action = ACTION_EXPAND; } - final TreeRow r; + final TreeSubscription.TreeRowImpl r; if (row.isNumber()) { - r = currentViewportData.rows.getAt((int) (row.asNumber() - currentViewportData.offset)); + r = (TreeSubscription.TreeRowImpl) currentViewportData.getRows().getAt((int) (row.asNumber())); } else if (row.isTreeRow()) { - r = row.asTreeRow(); + r = (TreeSubscription.TreeRowImpl) row.asTreeRow(); } else { throw new IllegalArgumentException("row parameter must be an index or a row"); } @@ -933,11 +857,11 @@ public void collapseAll() { * @return boolean */ public boolean isExpanded(RowReferenceUnion row) { - final TreeRow r; + final TreeSubscription.TreeRowImpl r; if (row.isNumber()) { - r = currentViewportData.rows.getAt((int) (row.asNumber() - currentViewportData.offset)); + r = (TreeSubscription.TreeRowImpl) currentViewportData.getRows().getAt((int) (row.asNumber())); } else if (row.isTreeRow()) { - r = row.asTreeRow(); + r = (TreeSubscription.TreeRowImpl) row.asTreeRow(); } else { throw new IllegalArgumentException("row parameter must be an index or a row"); } @@ -956,8 +880,8 @@ public void setViewport(double firstRow, double lastRow, @JsOptional @JsNullable replaceSubscription(RebuildStep.SUBSCRIPTION); } - public Promise getViewportData() { - LazyPromise promise = new LazyPromise<>(); + public Promise<@TsTypeRef(TreeViewportData.class) Object> getViewportData() { + LazyPromise promise = new LazyPromise<>(); if (currentViewportData == null) { // only one of these two will fire, and when they do, they'll remove both handlers. @@ -1020,8 +944,7 @@ public void close() { } if (stream != null) { stream.then(stream -> { - stream.end(); - stream.cancel(); + stream.close(); return null; }); stream = null; diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/tree/TreeViewportData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/tree/TreeViewportData.java new file mode 100644 index 00000000000..6e00b17bf39 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/tree/TreeViewportData.java @@ -0,0 +1,60 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.web.client.api.tree; + +import com.vertispan.tsdefs.annotations.TsTypeRef; +import elemental2.core.JsArray; +import io.deephaven.web.client.api.TableData; +import jsinterop.annotations.JsProperty; +import jsinterop.annotations.JsType; + +@JsType(namespace = "dh") +public interface TreeViewportData extends TableData { + @JsProperty + double getTreeSize(); + + @Override + @TsTypeRef(TreeRow.class) + default TableData.Row get(RowPositionUnion index) { + return TableData.super.get(index); + } + + @JsProperty + @Override + JsArray getRows(); + + /** + * Row implementation that also provides additional read-only properties. represents visible rows in the table, but + * with additional properties to reflect the tree structure. + */ + @JsType + interface TreeRow extends TableData.Row { + /** + * True if this node is currently expanded to show its children; false otherwise. Those children will be the + * rows below this one with a greater depth than this one. + * + * @return boolean + */ + @JsProperty(name = "isExpanded") + boolean isExpanded(); + + /** + * True if this node has children and can be expanded; false otherwise. Note that this value may change when the + * table updates, depending on the table's configuration. + * + * @return boolean + */ + @JsProperty(name = "hasChildren") + boolean hasChildren(); + + /** + * The number of levels above this node; zero for top level nodes. Generally used by the UI to indent the row + * and its expand/collapse icon. + * + * @return int + */ + @JsProperty(name = "depth") + int depth(); + } +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/widget/plot/ChartData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/widget/plot/ChartData.java index 8a30411fd33..ac70fd09128 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/widget/plot/ChartData.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/widget/plot/ChartData.java @@ -8,7 +8,8 @@ import io.deephaven.web.client.api.JsRangeSet; import io.deephaven.web.client.api.JsTable; import io.deephaven.web.client.api.TableData; -import io.deephaven.web.client.api.subscription.SubscriptionTableData.UpdateEventData; +import io.deephaven.web.client.api.subscription.AbstractTableSubscription; +import io.deephaven.web.client.api.subscription.SubscriptionTableData; import io.deephaven.web.client.fu.JsSettings; import io.deephaven.web.shared.data.Range; import io.deephaven.web.shared.fu.JsFunction; @@ -35,10 +36,11 @@ public ChartData(JsTable table) { this.table = table; } - public void update(UpdateEventData tableData) { - Iterator addedIterator = tableData.getAdded().getRange().rangeIterator(); - Iterator removedIterator = tableData.getRemoved().getRange().rangeIterator(); - Iterator modifiedIterator = tableData.getModified().getRange().rangeIterator(); + public void update(AbstractTableSubscription.UpdateEventData tableData) { + SubscriptionTableData data = (SubscriptionTableData) tableData; + Iterator addedIterator = data.getAdded().getRange().rangeIterator(); + Iterator removedIterator = data.getRemoved().getRange().rangeIterator(); + Iterator modifiedIterator = data.getModified().getRange().rangeIterator(); Range nextAdded = addedIterator.hasNext() ? addedIterator.next() : null; Range nextRemoved = removedIterator.hasNext() ? removedIterator.next() : null; @@ -130,7 +132,7 @@ public void update(UpdateEventData tableData) { assert cachedData.values().stream().flatMap(m -> m.values().stream()).allMatch(arr -> arr .reduce((Object val, Any p1, int p2) -> ((Integer) val) + 1, 0) == indexes.length); - JsRangeSet fullIndex = tableData.getFullIndex(); + JsRangeSet fullIndex = ((SubscriptionTableData) tableData).getFullIndex(); PrimitiveIterator.OfLong iter = fullIndex.getRange().indexIterator(); for (int j = 0; j < indexes.length; j++) { assert indexes[j] == iter.nextLong(); @@ -138,7 +140,7 @@ public void update(UpdateEventData tableData) { } } - private void replaceDataRange(UpdateEventData tableData, Range range, int offset) { + private void replaceDataRange(AbstractTableSubscription.UpdateEventData tableData, Range range, int offset) { // we don't touch the indexes at all, only need to walk each column and replace values in this range for (Entry, JsArray>> columnMap : cachedData.entrySet()) { Column col = table.findColumn(columnMap.getKey()); @@ -160,7 +162,7 @@ private void replaceDataRange(UpdateEventData tableData, Range range, int offset } } - private void insertDataRange(UpdateEventData tableData, Range range, int offset) { + private void insertDataRange(AbstractTableSubscription.UpdateEventData tableData, Range range, int offset) { // splice in the new indexes batchSplice(offset, asArray(indexes), longs(range)); @@ -195,7 +197,8 @@ private Any[] batchSplice(int offset, JsArray existingData, Any[] dataToIns return Js.uncheckedCast(existingData); } - private Any[] values(UpdateEventData tableData, JsFunction mapFunc, Column col, Range insertedRange) { + private Any[] values(AbstractTableSubscription.UpdateEventData tableData, JsFunction mapFunc, Column col, + Range insertedRange) { JsArray result = new JsArray<>(); if (mapFunc == null) { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/widget/plot/FigureSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/widget/plot/FigureSubscription.java index 2b1ed636861..bb3d6e0817d 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/widget/plot/FigureSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/widget/plot/FigureSubscription.java @@ -3,13 +3,12 @@ // package io.deephaven.web.client.api.widget.plot; -import elemental2.dom.CustomEvent; import elemental2.dom.CustomEventInit; import elemental2.promise.Promise; import io.deephaven.web.client.api.DateWrapper; import io.deephaven.web.client.api.JsTable; import io.deephaven.web.client.api.LongWrapper; -import io.deephaven.web.client.api.subscription.SubscriptionTableData; +import io.deephaven.web.client.api.subscription.AbstractTableSubscription; import io.deephaven.web.client.api.subscription.TableSubscription; import io.deephaven.web.client.fu.JsLog; @@ -259,8 +258,8 @@ private Promise subscribe(final Promise tablePromise this.currentData = new ChartData(table); sub.addEventListener(TableSubscription.EVENT_UPDATED, e -> { // refire with specifics for the columns that we're watching here, after updating data arrays - SubscriptionTableData.UpdateEventData subscriptionUpdateData = - (SubscriptionTableData.UpdateEventData) ((CustomEvent) e).detail; + AbstractTableSubscription.UpdateEventData subscriptionUpdateData = + (AbstractTableSubscription.UpdateEventData) e.detail; currentData.update(subscriptionUpdateData); CustomEventInit event = CustomEventInit.create(); diff --git a/web/client-api/src/main/java/io/deephaven/web/client/fu/JsWeakRef.java b/web/client-api/src/main/java/io/deephaven/web/client/fu/JsWeakRef.java new file mode 100644 index 00000000000..35de0ed3e11 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/fu/JsWeakRef.java @@ -0,0 +1,16 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.web.client.fu; + +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType(namespace = JsPackage.GLOBAL, name = "WeakRef", isNative = true) +public class JsWeakRef { + public JsWeakRef(T target) { + + } + + public native T deref(); +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/state/ActiveTableBinding.java b/web/client-api/src/main/java/io/deephaven/web/client/state/ActiveTableBinding.java index 5aed4d5c121..722daed1b95 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/state/ActiveTableBinding.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/state/ActiveTableBinding.java @@ -3,11 +3,8 @@ // package io.deephaven.web.client.state; -import io.deephaven.web.client.api.Column; import io.deephaven.web.client.api.JsTable; import io.deephaven.web.client.api.state.HasTableState; -import io.deephaven.web.shared.data.RangeSet; -import io.deephaven.web.shared.data.Viewport; /** * An active binding describes the link between a {@link JsTable} and the {@link ClientTableState} it is currently @@ -60,10 +57,6 @@ public class ActiveTableBinding implements HasTableState { * new table must copy all bindings to the new table. */ private PausedTableBinding rollback; - private Viewport viewport; - private RangeSet rows; - private Column[] columns; - private boolean subscriptionPending; private ActiveTableBinding( JsTable table, @@ -234,51 +227,4 @@ private void copyRollbacks(ActiveTableBinding sub) { public PausedTableBinding getPaused() { return paused; } - - public Viewport getSubscription() { - return viewport; - } - - public void setViewport(Viewport viewport) { - this.viewport = viewport; - } - - public RangeSet setDesiredViewport(long firstRow, long lastRow, Column[] columns) { - this.rows = RangeSet.ofRange(firstRow, lastRow); - this.columns = columns; - subscriptionPending = true; - return rows; - } - - public void setDesiredSubscription(Column[] columns) { - assert this.rows == null; - this.columns = columns; - subscriptionPending = true; - } - - public RangeSet getRows() { - return rows; - } - - public Column[] getColumns() { - return columns; - } - - public void setSubscriptionPending(boolean subscriptionPending) { - this.subscriptionPending = subscriptionPending; - } - - public boolean isSubscriptionPending() { - return subscriptionPending; - } - - public void maybeReviveSubscription() { - if (subscriptionPending || viewport != null) { - if (rows != null) { - state.setDesiredViewport(table, rows.getFirstRow(), rows.getLastRow(), columns); - } else { - state.subscribe(table, columns); - } - } - } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/state/ClientTableState.java b/web/client-api/src/main/java/io/deephaven/web/client/state/ClientTableState.java index 8a8be5f1dfc..7eff8266775 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/state/ClientTableState.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/state/ClientTableState.java @@ -8,7 +8,7 @@ import elemental2.core.JsSet; import elemental2.core.Uint8Array; import elemental2.promise.Promise; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Schema; +import io.deephaven.chunk.ChunkType; import io.deephaven.javascript.proto.dhinternal.browserheaders.BrowserHeaders; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.table_pb.ExportedTableCreationResponse; import io.deephaven.web.client.api.*; @@ -57,6 +57,7 @@ * Consider making this a js type with restricted, read-only property access. */ public final class ClientTableState extends TableConfig { + public enum ResolutionState { /** * Table has been created on the client, but client does not yet have a handle ID referring to the table on the @@ -121,7 +122,6 @@ public enum ResolutionState { // A bit of state management private String failMsg; - private boolean subscribed; private Double queuedSize; // Leftovers from Table.StackEntry @@ -233,6 +233,89 @@ public TableTicket getHandle() { return handle; } + /** + * Returns the ChunkType to use for each column in the table. This is roughly + * {@link io.deephaven.engine.table.impl.sources.ReinterpretUtils#maybeConvertToWritablePrimitiveChunkType(Class)} + * but without the trip through Class. Note also that effectively all types are stored as Objects except non-long + * primitives, so that they can be appropriately wrapped before storing (though the storage process will handle DH + * nulls). + */ + public ChunkType[] chunkTypes() { + return Arrays.stream(columnTypes()).map(dataType -> { + if (dataType == Boolean.class || dataType == boolean.class) { + return ChunkType.Object; + } + if (dataType == Long.class || dataType == long.class) { + // JS client holds longs as LongWrappers + return ChunkType.Object; + } + return ChunkType.fromElementType(dataType); + }).toArray(ChunkType[]::new); + } + + /** + * Returns the Java Class to represent each column in the table. This lets the client replace certain JVM-only + * classes with alternative implementations, but still use the simple + * {@link io.deephaven.extensions.barrage.chunk.ChunkReader.TypeInfo} wrapper. + */ + public Class[] columnTypes() { + return Arrays.stream(tableDef.getColumns()) + .map(ColumnDefinition::getType) + .map(t -> { + switch (t) { + case "boolean": + case "java.lang.Boolean": + return boolean.class; + case "char": + case "java.lang.Character": + return char.class; + case "byte": + case "java.lang.Byte": + return byte.class; + case "int": + case "java.lang.Integer": + return int.class; + case "short": + case "java.lang.Short": + return short.class; + case "long": + case "java.lang.Long": + return long.class; + case "java.lang.Float": + case "float": + return float.class; + case "java.lang.Double": + case "double": + return double.class; + case "java.time.Instant": + return DateWrapper.class; + case "java.math.BigInteger": + return BigIntegerWrapper.class; + case "java.math.BigDecimal": + return BigDecimalWrapper.class; + default: + return Object.class; + } + }) + .toArray(Class[]::new); + } + + /** + * Returns the Java Class to represent the component type in any list/array type. At this time, this value is not + * used by the chunk reading implementation. + */ + public Class[] componentTypes() { + return Arrays.stream(tableDef.getColumns()).map(ColumnDefinition::getType).map(t -> { + // All arrays and vectors will be handled as objects for now. + // TODO (deephaven-core#2102) clarify if we need to handle these cases at all + if (t.endsWith("[]") || t.endsWith("Vector")) { + return Object.class; + } + // Non-arrays or vectors should return null + return null; + }).toArray(Class[]::new); + } + public ClientTableState newState(TableTicket newHandle, TableConfig config) { if (config == null) { config = this; @@ -349,27 +432,18 @@ public void setSize(long size) { } this.size = size; - JsConsumer doSetSize = table -> { - long localSize = size; - final ActiveTableBinding binding = getActiveBinding(table); - if (binding != null && table.isBlinkTable() && binding.getRows() != null) { - localSize = Math.min(size, binding.getRows().size()); - } - table.setSize(localSize); - }; - if (isRunning()) { if (log) { JsLog.debug("CTS immediate size update ", this.size, " actives: ", active); } - forActiveTables(doSetSize); + forActiveTables(t -> t.setSize(size)); } else { if (queuedSize == null) { JsLog.debug("Queuing size changed until RUNNING STATE; ", size); onRunning(self -> { JsLog.debug("Firing queued size change event (", queuedSize, ")"); forActiveTables(table -> { - doSetSize.apply(table); + table.setSize(size); queuedSize = null; }); }, JsRunnable.doNothing()); @@ -390,7 +464,6 @@ private void setTableDef(InitialTableDefinition tableDef) { // iterate through the columns, combine format columns into the normal model Map> byNameMap = tableDef.getColumnsByName(); - assert byNameMap.get(true).isEmpty() : "Unexpected constituent columns in table " + byNameMap.get(true); Column[] columns = new Column[0]; allColumns = new Column[0]; for (ColumnDefinition definition : columnDefinitions) { @@ -630,12 +703,10 @@ public Column findColumn(String key) { } /** - * @return true if there are no tables bound to this state. - * - * If a table that had a subscription for this state was orphaned by a pending request, we want to clear the - * subscription immediately so it becomes inert (immediately remove the subscription), but we may need to - * rollback the request, and we don't want to release the handle until the pending request is finished - * (whereupon we will remove the binding). + * @return true if there are no tables bound to this state. If a table that had a subscription for this state was + * orphaned by a pending request, we want to clear the subscription immediately so it becomes inert + * (immediately remove the subscription), but we may need to rollback the request, and we don't want to + * release the handle until the pending request is finished (whereupon we will remove the binding). */ public boolean isEmpty() { return active.size == 0 && paused.size == 0 && retainers.size == 0; @@ -658,14 +729,6 @@ public void unretain(Object retainer) { connection.scheduleCheck(this); } - public boolean isActiveEmpty() { - return active.size == 0; - } - - public boolean hasNoSubscriptions() { - return JsItr.iterate(active.values()).allMatch(binding -> binding.getSubscription() == null); - } - public boolean hasSort(Sort candidate) { return getSorts().contains(candidate); } @@ -715,59 +778,6 @@ public boolean releaseTable(JsTable table) { return had; } - public void setDesiredViewport(JsTable table, long firstRow, long lastRow, Column[] columns) { - touch(); - final ActiveTableBinding sub = active.get(table); - assert sub != null : "You cannot set the desired viewport on a non-active state + table combination"; - final RangeSet rows = sub.setDesiredViewport(firstRow, lastRow, columns); - // let event loop eat multiple viewport sets and only apply the last one (winner of who gets spot in map) - LazyPromise.runLater(() -> { - if (sub.getRows() == rows) { - // winner! now, on to the next hurdle... ensuring we have columns. - // TODO: have an onColumnsReady callback, for cases when we know we're only waiting on - // non-column-modifying operations - onRunning(self -> { - if (sub.getRows() == rows) { - // winner again! - applyViewport(sub); - } - }, JsRunnable.doNothing()); - } - }); - } - - public void subscribe(JsTable table, Column[] columns) { - touch(); - ActiveTableBinding binding = active.get(table); - assert binding != null : "No active binding found for table " + table; - - onRunning(self -> { - binding.setSubscriptionPending(true); - - if (getHandle().equals(table.getHandle())) { - binding.setViewport(new Viewport(null, makeBitset(columns))); - table.getConnection().scheduleCheck(this); - } - }, JsRunnable.doNothing()); - } - - private void applyViewport(ActiveTableBinding sub) { - sub.setSubscriptionPending(false); - final JsTable table = sub.getTable(); - // make sure we're still the tail entry before trying to apply viewport - assert isRunning() : "Do not call this method unless you are in a running state! " + this; - if (getHandle().equals(table.getHandle())) { - final RangeSet rows = sub.getRows(); - Column[] desired = sub.getColumns(); - if (Js.isFalsy(desired)) { - desired = getColumns(); - } - Viewport vp = new Viewport(rows, makeBitset(desired)); - sub.setViewport(vp); - table.refreshViewport(this, vp); - } - } - public BitSet makeBitset(Column[] columns) { BitSet bitSet = new BitSet(getTableDef().getColumns().length); Arrays.stream(columns).flatMapToInt(Column::getRequiredColumns).forEach(bitSet::set); @@ -784,16 +794,6 @@ public MappedIterable getBoundTables() { return iterate(active.keys()).plus(iterate(paused.keys())); } - public void forActiveSubscriptions(JsBiConsumer callback) { - JsItr.forEach(active, (table, binding) -> { - if (binding.getSubscription() != null) { - assert binding.getTable() == table - : "Corrupt binding between " + table + " and " + binding + " in " + active; - callback.apply((JsTable) table, binding.getSubscription()); - } - }); - } - public void forActiveTables(JsConsumer callback) { JsItr.forEach(active, (table, sub) -> { callback.apply(table); @@ -818,24 +818,6 @@ public void forActiveLifecycles(JsConsumer callback) { } } - public void handleDelta(DeltaUpdates updates) { - assert size != SIZE_UNINITIALIZED : "Received delta before receiving initial size"; - setSize(size + updates.getAdded().size() - updates.getRemoved().size()); - forActiveSubscriptions((table, subscription) -> { - assert table.getHandle().equals(handle); - // we are the active state of this table, so forward along the delta. - table.handleDelta(this, updates); - }); - } - - public void setSubscribed(boolean subscribed) { - this.subscribed = subscribed; - } - - public boolean isSubscribed() { - return subscribed; - } - @Override public String toString() { return "ClientTableState{" + @@ -905,15 +887,9 @@ public void unpause(JsTable table) { assert was != null : "Cannot unpause a table that is not paused " + this; paused.delete(table); active.set(table, was.getActiveBinding()); - // Now, we want to put back the viewport, if any. - refreshSubscription(was.getActiveBinding()); - } - - private void refreshSubscription(ActiveTableBinding sub) { - assert active.get(sub.getTable()) == sub; - if (!sub.isSubscriptionPending()) { - sub.maybeReviveSubscription(); - } + // Now, we want to put back the viewport, if any, since those may be still used by the original table + ActiveTableBinding sub = was.getActiveBinding(); + table.maybeReviveSubscription(); } public MappedIterable ancestors() { @@ -925,7 +901,7 @@ public MappedIterable reversed() { } /** - * Look through paused tables to see if any of them have been + * Look through paused tables to see if any of them have been closed. */ public void cleanup() { assert JsItr.iterate(active.keys()).allMatch(t -> !t.isAlive() || t.state() == this) @@ -1011,9 +987,7 @@ public void applyTableCreationResponse(ExportedTableCreationResponse def) { Uint8Array flightSchemaMessage = def.getSchemaHeader_asU8(); isStatic = def.getIsStatic(); - Schema schema = WebBarrageUtils.readSchemaMessage(flightSchemaMessage); - - setTableDef(WebBarrageUtils.readTableDefinition(schema)); + setTableDef(WebBarrageUtils.readTableDefinition(WebBarrageUtils.readSchemaMessage(flightSchemaMessage))); setResolution(ResolutionState.RUNNING); setSize(Long.parseLong(def.getSize())); diff --git a/web/client-api/src/main/resources/io/deephaven/barrage/flatbuf/BarrageFlatbufFormat.gwt.xml b/web/client-api/src/main/resources/io/deephaven/barrage/flatbuf/BarrageFlatbufFormat.gwt.xml new file mode 100644 index 00000000000..9fd0a379083 --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/barrage/flatbuf/BarrageFlatbufFormat.gwt.xml @@ -0,0 +1,4 @@ + + + + diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/com/google/common/io/LittleEndianDataInputStream.java b/web/client-api/src/main/resources/io/deephaven/web/super/com/google/common/io/LittleEndianDataInputStream.java new file mode 100644 index 00000000000..e1f587ac72e --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/com/google/common/io/LittleEndianDataInputStream.java @@ -0,0 +1,198 @@ +/* + * Copyright (C) 2007 The Guava Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.common.io; + +import org.gwtproject.nio.Numbers; + +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; + +/** + * An implementation of {@link DataInput} that uses little-endian byte ordering for reading {@code + * short}, {@code int}, {@code float}, {@code double}, and {@code long} values. + * + *

Note: This class intentionally violates the specification of its supertype {@code + * DataInput}, which explicitly requires big-endian byte order. + * + * @author Chris Nokleberg + * @author Keith Bottner + * @since 8.0 + */ +public final class LittleEndianDataInputStream extends FilterInputStream implements DataInput { + + public LittleEndianDataInputStream(InputStream in) { + super(Objects.requireNonNull(in)); + } + + /** This method will throw an {@link UnsupportedOperationException}. */ + @Override + public String readLine() { + throw new UnsupportedOperationException("readLine is not supported"); + } + + @Override + public void readFully(byte[] b) throws IOException { + int total = 0; + while (total < b.length) { + int result = read(b, total, b.length - total); + if (result == -1) { + break; + } + total += result; + } + } + + @Override + public void readFully(byte[] b, int off, int len) throws IOException { + int total = 0; + while (total < len) { + int result = read(b, off + total, len - total); + if (result == -1) { + break; + } + total += result; + } + } + + @Override + public int skipBytes(int n) throws IOException { + return (int) in.skip(n); + } + + @Override + public int readUnsignedByte() throws IOException { + int b1 = in.read(); + if (0 > b1) { + throw new EOFException(); + } + + return b1; + } + + @Override + public int readUnsignedShort() throws IOException { + byte b1 = readAndCheckByte(); + byte b2 = readAndCheckByte(); + + int result = b2; + result = (result << 8) | (b1 & 0xFF); + + return result; + } + + @Override + public int readInt() throws IOException { + byte b1 = readAndCheckByte(); + byte b2 = readAndCheckByte(); + byte b3 = readAndCheckByte(); + byte b4 = readAndCheckByte(); + + int result = b4; + result = (result << 8) | (b3 & 0xFF); + result = (result << 8) | (b2 & 0xFF); + result = (result << 8) | (b1 & 0xFF); + + return result; + } + + @Override + public long readLong() throws IOException { + byte b1 = readAndCheckByte(); + byte b2 = readAndCheckByte(); + byte b3 = readAndCheckByte(); + byte b4 = readAndCheckByte(); + byte b5 = readAndCheckByte(); + byte b6 = readAndCheckByte(); + byte b7 = readAndCheckByte(); + byte b8 = readAndCheckByte(); + + long result = b8; + result = (result << 8) | (b7 & 0xFF); + result = (result << 8) | (b6 & 0xFF); + result = (result << 8) | (b5 & 0xFF); + result = (result << 8) | (b4 & 0xFF); + result = (result << 8) | (b3 & 0xFF); + result = (result << 8) | (b2 & 0xFF); + result = (result << 8) | (b1 & 0xFF); + + return result; + } + + /** + * Reads a {@code float} as specified by {@link DataInputStream#readFloat()}, except using + * little-endian byte order. + * + * @return the next four bytes of the input stream, interpreted as a {@code float} in + * little-endian byte order + * @throws IOException if an I/O error occurs + */ + @Override + public float readFloat() throws IOException { + return Numbers.intBitsToFloat(readInt()); + } + + /** + * Reads a {@code double} as specified by {@link DataInputStream#readDouble()}, except using + * little-endian byte order. + * + * @return the next eight bytes of the input stream, interpreted as a {@code double} in + * little-endian byte order + * @throws IOException if an I/O error occurs + */ + @Override + public double readDouble() throws IOException { + return Numbers.longBitsToDouble(readLong()); + } + + @Override + public String readUTF() throws IOException { + throw new UnsupportedOperationException("readUTF"); + } + + @Override + public short readShort() throws IOException { + return (short) readUnsignedShort(); + } + + @Override + public char readChar() throws IOException { + return (char) readUnsignedShort(); + } + + @Override + public byte readByte() throws IOException { + return (byte) readUnsignedByte(); + } + + @Override + public boolean readBoolean() throws IOException { + return readUnsignedByte() != 0; + } + + private byte readAndCheckByte() throws IOException, EOFException { + int b1 = in.read(); + + if (-1 == b1) { + throw new EOFException(); + } + + return (byte) b1; + } +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/com/google/common/io/LittleEndianDataOutputStream.java b/web/client-api/src/main/resources/io/deephaven/web/super/com/google/common/io/LittleEndianDataOutputStream.java new file mode 100644 index 00000000000..12651d83ddf --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/com/google/common/io/LittleEndianDataOutputStream.java @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2007 The Guava Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.common.io; + +import org.gwtproject.nio.Numbers; + +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Objects; + +/** + * An implementation of {@link DataOutput} that uses little-endian byte ordering for writing {@code + * char}, {@code short}, {@code int}, {@code float}, {@code double}, and {@code long} values. + * + *

Note: This class intentionally violates the specification of its supertype {@code + * DataOutput}, which explicitly requires big-endian byte order. + * + * @author Chris Nokleberg + * @author Keith Bottner + * @since 8.0 + */ +public final class LittleEndianDataOutputStream extends FilterOutputStream implements DataOutput { + + /** + * Creates a {@code LittleEndianDataOutputStream} that wraps the given stream. + * + * @param out the stream to delegate to + */ + public LittleEndianDataOutputStream(OutputStream out) { + super(new DataOutputStream(Objects.requireNonNull(out))); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + // Override slow FilterOutputStream impl + out.write(b, off, len); + } + + @Override + public void writeBoolean(boolean v) throws IOException { + ((DataOutputStream) out).writeBoolean(v); + } + + @Override + public void writeByte(int v) throws IOException { + ((DataOutputStream) out).writeByte(v); + } + + /** + * @deprecated The semantics of {@code writeBytes(String s)} are considered dangerous. Please use + * {@link #writeUTF(String s)}, {@link #writeChars(String s)} or another write method instead. + */ + @Deprecated + @Override + public void writeBytes(String s) throws IOException { + ((DataOutputStream) out).writeBytes(s); + } + + /** + * Writes a char as specified by {@link DataOutputStream#writeChar(int)}, except using + * little-endian byte order. + * + * @throws IOException if an I/O error occurs + */ + @Override + public void writeChar(int v) throws IOException { + writeShort(v); + } + + /** + * Writes a {@code String} as specified by {@link DataOutputStream#writeChars(String)}, except + * each character is written using little-endian byte order. + * + * @throws IOException if an I/O error occurs + */ + @Override + public void writeChars(String s) throws IOException { + for (int i = 0; i < s.length(); i++) { + writeChar(s.charAt(i)); + } + } + + /** + * Writes a {@code double} as specified by {@link DataOutputStream#writeDouble(double)}, except + * using little-endian byte order. + * + * @throws IOException if an I/O error occurs + */ + @Override + public void writeDouble(double v) throws IOException { + writeLong(Double.doubleToLongBits(v)); + } + + /** + * Writes a {@code float} as specified by {@link DataOutputStream#writeFloat(float)}, except using + * little-endian byte order. + * + * @throws IOException if an I/O error occurs + */ + @Override + public void writeFloat(float v) throws IOException { + writeInt(Numbers.floatToIntBits(v)); + } + + /** + * Writes an {@code int} as specified by {@link DataOutputStream#writeInt(int)}, except using + * little-endian byte order. + * + * @throws IOException if an I/O error occurs + */ + @Override + public void writeInt(int v) throws IOException { + out.write(0xFF & v); + out.write(0xFF & (v >> 8)); + out.write(0xFF & (v >> 16)); + out.write(0xFF & (v >> 24)); + } + + /** + * Writes a {@code long} as specified by {@link DataOutputStream#writeLong(long)}, except using + * little-endian byte order. + * + * @throws IOException if an I/O error occurs + */ + @Override + public void writeLong(long v) throws IOException { + out.write(0xFF & (int) v); + out.write(0xFF & (int) (v >> 8)); + out.write(0xFF & (int) (v >> 16)); + out.write(0xFF & (int) (v >> 24)); + out.write(0xFF & (int) (v >> 32)); + out.write(0xFF & (int) (v >> 40)); + out.write(0xFF & (int) (v >> 48)); + out.write(0xFF & (int) (v >> 56)); + } + + /** + * Writes a {@code short} as specified by {@link DataOutputStream#writeShort(int)}, except using + * little-endian byte order. + * + * @throws IOException if an I/O error occurs + */ + @Override + public void writeShort(int v) throws IOException { + out.write(0xFF & v); + out.write(0xFF & (v >> 8)); + } + + @Override + public void writeUTF(String s) throws IOException { + throw new UnsupportedOperationException("modified utf-8"); + } +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/chunk/util/pools/ChunkPoolReleaseTracking.java b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/chunk/util/pools/ChunkPoolReleaseTracking.java new file mode 100644 index 00000000000..c6e26966d4e --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/chunk/util/pools/ChunkPoolReleaseTracking.java @@ -0,0 +1,41 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.chunk.util.pools; + +/** + * Support for release tracking, in order to detect chunk release errors. + */ +public final class ChunkPoolReleaseTracking { + + public static void enableStrict() { + } + + public static void enable() { + } + + private static void enable(final Object factory, boolean preCheck) { + + } + + public static void disable() { + } + + static CHUNK_TYPE onTake(CHUNK_TYPE chunk) { + return chunk; + } + + static CHUNK_TYPE onGive(CHUNK_TYPE chunk) { + return chunk; + } + + public static void check() { + + } + + public static void checkAndDisable() { + + } + + private ChunkPoolReleaseTracking() {} +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/RowSequence.java b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/RowSequence.java new file mode 100644 index 00000000000..1ba218f9a73 --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/RowSequence.java @@ -0,0 +1,24 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.rowset; + +import io.deephaven.util.SafeCloseable; +import io.deephaven.util.datastructures.LongAbortableConsumer; +import io.deephaven.util.datastructures.LongRangeConsumer; +import io.deephaven.util.datastructures.LongSizedDataStructure; + +public interface RowSequence extends SafeCloseable, LongSizedDataStructure { + long NULL_ROW_KEY = -1L; + boolean isEmpty(); + long lastRowKey(); + boolean forEachRowKey(LongAbortableConsumer lac); + default void forAllRowKeys(java.util.function.LongConsumer lc) { + forEachRowKey((final long v) -> { + lc.accept(v); + return true; + }); + } + + void forAllRowKeyRanges(LongRangeConsumer lrc); +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/RowSequenceFactory.java b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/RowSequenceFactory.java new file mode 100644 index 00000000000..9175bd144c4 --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/RowSequenceFactory.java @@ -0,0 +1,14 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.rowset; + +import io.deephaven.engine.rowset.impl.WritableRowSetImpl; +import io.deephaven.web.shared.data.RangeSet; + +public class RowSequenceFactory { + public static final RowSequence EMPTY = new WebRowSetImpl(RangeSet.empty()); + public static RowSequence forRange(final long firstRowKey, final long lastRowKey) { + return new WebRowSetImpl(RangeSet.ofRange(firstRowKey, lastRowKey)); + } +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/RowSet.java b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/RowSet.java new file mode 100644 index 00000000000..a86d285d230 --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/RowSet.java @@ -0,0 +1,14 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.rowset; + +import io.deephaven.util.SafeCloseable; +import io.deephaven.util.datastructures.LongSizedDataStructure; + +public interface RowSet extends RowSequence, LongSizedDataStructure, SafeCloseable { + RowSet copy(); + long get(long rowPosition); + + WritableRowSet intersect(RowSet rowSet); +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/RowSetBuilderSequential.java b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/RowSetBuilderSequential.java new file mode 100644 index 00000000000..e14ffa21d9f --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/RowSetBuilderSequential.java @@ -0,0 +1,11 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.rowset; + +import io.deephaven.util.datastructures.LongRangeConsumer; + +public interface RowSetBuilderSequential extends LongRangeConsumer { + void appendRange(long rangeFirstRowKey, long rangeLastRowKey); + RowSet build(); +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/RowSetFactory.java b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/RowSetFactory.java new file mode 100644 index 00000000000..4dfe25e44e6 --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/RowSetFactory.java @@ -0,0 +1,23 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.rowset; + +import io.deephaven.engine.rowset.RowSetBuilderSequential; +import io.deephaven.web.shared.data.RangeSet; + +public class RowSetFactory { + + public static RowSet empty() { + return new WebRowSetImpl(RangeSet.empty()); + } + public static RowSetBuilderSequential builderSequential() { + return new WebRowSetBuilderSequentialImpl(); + } + public static RowSet fromRange(long first, long last) { + return new WebRowSetImpl(RangeSet.ofRange(first, last)); + } + public static RowSet flat(long size) { + return size <= 0 ? empty() : new WebRowSetImpl(RangeSet.ofRange(0, size - 1)); + } +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/WebRowSetBuilderSequentialImpl.java b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/WebRowSetBuilderSequentialImpl.java new file mode 100644 index 00000000000..7b2673269d0 --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/WebRowSetBuilderSequentialImpl.java @@ -0,0 +1,21 @@ +package io.deephaven.engine.rowset; + +import io.deephaven.web.shared.data.Range; +import io.deephaven.web.shared.data.RangeSet; + +final class WebRowSetBuilderSequentialImpl implements RowSetBuilderSequential { + private final RangeSet rangeSet = new RangeSet(); + @Override + public void appendRange(long rangeFirstRowKey, long rangeLastRowKey) { + rangeSet.addRange(new Range(rangeFirstRowKey, rangeLastRowKey)); + } + + @Override + public void accept(long first, long last) { + appendRange(first, last); + } + @Override + public RowSet build() { + return new WebRowSetImpl(rangeSet); + } +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/WebRowSetImpl.java b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/WebRowSetImpl.java new file mode 100644 index 00000000000..abd578086a6 --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/WebRowSetImpl.java @@ -0,0 +1,89 @@ +package io.deephaven.engine.rowset; + +import io.deephaven.util.datastructures.LongAbortableConsumer; +import io.deephaven.util.datastructures.LongRangeConsumer; +import io.deephaven.web.shared.data.RangeSet; + +import java.util.PrimitiveIterator; + +final class WebRowSetImpl implements RowSet, WritableRowSet { + private final RangeSet rangeSet; + + WebRowSetImpl(RangeSet rangeSet) { + this.rangeSet = rangeSet; + } + + @Override + public boolean isEmpty() { + return rangeSet.isEmpty(); + } + + @Override + public long lastRowKey() { + return rangeSet.getLastRow(); + } + + @Override + public boolean forEachRowKey(LongAbortableConsumer lac) { + PrimitiveIterator.OfLong iter = rangeSet.indexIterator(); + while (iter.hasNext()) { + long key = iter.nextLong(); + if (!lac.accept(key)) { + return false; + } + } + return true; + } + + @Override + public void forAllRowKeyRanges(LongRangeConsumer lrc) { + rangeSet.rangeIterator().forEachRemaining(r -> { + lrc.accept(r.getFirst(), r.getLast()); + }); + } + + @Override + public long get(long position) { + return rangeSet.get(position); + } + @Override + public WritableRowSet intersect(RowSet rowSet) { + throw new UnsupportedOperationException("intersect"); + } + @Override + public WritableRowSet shift(long shiftAmount) { + throw new UnsupportedOperationException("shift"); + } + + + @Override + public long size() { + return rangeSet.size(); + } + + @Override + public void close() { + + } + + @Override + public RowSet copy() { + return new WebRowSetImpl(rangeSet.copy()); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof WebRowSetImpl)) { + return false; + } + return rangeSet.equals(((WebRowSetImpl) obj).rangeSet); + } + + @Override + public int hashCode() { + return rangeSet.hashCode(); + } +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/WritableRowSet.java b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/WritableRowSet.java new file mode 100644 index 00000000000..85572b5939c --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/engine/rowset/WritableRowSet.java @@ -0,0 +1,8 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.rowset; + +public interface WritableRowSet extends RowSet { + WritableRowSet shift(long shiftAmount); +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/extensions/barrage/chunk/array/ArrayReflectUtil.java b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/extensions/barrage/chunk/array/ArrayReflectUtil.java new file mode 100644 index 00000000000..4b0dd298944 --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/extensions/barrage/chunk/array/ArrayReflectUtil.java @@ -0,0 +1,18 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.extensions.barrage.chunk.array; + +import java.lang.reflect.Array; + +/** + * Browser impl, that only returns a plain Object[]. + */ +class ArrayReflectUtil { + /** + * Always returns a plain Object[]. + */ + static Object newInstance(Class componentType, int length) { + return new Object[length]; + } +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/util/MultiException.java b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/util/MultiException.java new file mode 100644 index 00000000000..25c80ae422b --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/util/MultiException.java @@ -0,0 +1,74 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.util; + +import java.util.List; + +/** + * Simplified version of MultiException, which doesn't require PrintWriter. + */ +public class MultiException extends Exception { + + public static final Throwable[] ZERO_LENGTH_THROWABLE_ARRAY = new Throwable[0]; + private final Throwable[] causes; + + /** + * Create a MultiException from an array of Throwable causes. + * + * @param description the message to use + * @param causes a list of causes + */ + public MultiException(String description, Throwable... causes) { + super(description, getFirstCause(causes)); + this.causes = causes == null ? ZERO_LENGTH_THROWABLE_ARRAY : causes; + } + + + /** + * If there is a single exception, return that exception; otherwise wrap the causes into a MultiException. + * + * @param description the description for the MultiException + * @param causes the list of causes + * @return a MultiException or the single Throwable + */ + public static Throwable maybeWrapInMultiException(String description, List causes) { + if (causes.size() == 1) { + return causes.get(0); + } + return new MultiException(description, causes.toArray(ZERO_LENGTH_THROWABLE_ARRAY)); + } + + private static Throwable getFirstCause(Throwable[] causes) { + if (causes == null || causes.length == 0) { + return null; + } + + return causes[0]; + } + + /** + * @return all of the exceptions that resulted in this one. + */ + public Throwable[] getCauses() { + return causes; + } + + @Override + public String getMessage() { + StringBuilder sb = new StringBuilder(); + sb.append(super.getMessage()).append(":\n"); + for (int i = 0; i < causes.length; i++) { + sb.append("Cause ").append(i).append(": "); + sb.append(causes[i].toString()); + sb.append('\n'); + } + + return sb.toString(); + } + + @Override + public String toString() { + return getMessage(); + } +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/util/referencecounting/ReferenceCounted.java b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/util/referencecounting/ReferenceCounted.java new file mode 100644 index 00000000000..f33bd8c6483 --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/io/deephaven/util/referencecounting/ReferenceCounted.java @@ -0,0 +1,193 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.util.referencecounting; + +import org.jetbrains.annotations.NotNull; + +/** + * Implements a recurring reference counting pattern - a concurrent reference count that should refuse to go below zero, + * and invokes {@link #onReferenceCountAtZero()} exactly once when the count returns to zero. + */ +public abstract class ReferenceCounted { + /** + * This constant represents a "zero" reference count value that doesn't prevent increasing the reference count. + */ + private static final int INITIAL_ZERO_VALUE = -1; + + /** + * Since we've reserved -1 as our initial reference count value, our maximum is really one less. + */ + private static final int MAXIMUM_VALUE = -3; + + /** + * This is our "one" reference count value. + */ + private static final int ONE_VALUE = 1; + + /** + * This is our normal "zero" reference count value (terminal state). + */ + private static final int NORMAL_TERMINAL_ZERO_VALUE = 0; + + /** + * This is a marker "zero" reference count value (terminal state), signifying that a reference count was set to zero + * under exceptional circumstances, and additional attempts to drop the reference count should be treated as + * successful so as not to violate constraints. + */ + private static final int FORCED_TERMINAL_ZERO_VALUE = -2; + + /** + * The actual value of our reference count. + */ + private int referenceCount; + + protected ReferenceCounted() { + this(0); + } + + /** + * @param initialValue The initial value for the reference count, taken as an unsigned integer. Must not be one of + * the reserved values {@value #INITIAL_ZERO_VALUE} or {@value #FORCED_TERMINAL_ZERO_VALUE}. + */ + @SuppressWarnings("WeakerAccess") + protected ReferenceCounted(final int initialValue) { + initializeReferenceCount(initialValue); + } + + private void initializeReferenceCount(final int initialValue) { + if (initialValue == INITIAL_ZERO_VALUE || initialValue == FORCED_TERMINAL_ZERO_VALUE) { + throw new IllegalArgumentException("Invalid initial reference count " + initialValue); + } + referenceCount = initialValue == 0 ? INITIAL_ZERO_VALUE : initialValue; + } + + public static String getReferenceCountDebug(Object maybeReferenceCounted) { + return maybeReferenceCounted instanceof ReferenceCounted + ? Integer.toString(((ReferenceCounted) maybeReferenceCounted).getCurrentReferenceCount()) + : "not reference counted"; + } + + private int getCurrentReferenceCount() { + return referenceCount; + } + + private boolean tryUpdateReferenceCount(final int expected, final int update) { + if (referenceCount != expected) { + return false; + } + referenceCount = update; + return true; + } + + /** + * Reset this reference count to its initial state for reuse. + */ + public final void resetReferenceCount() { + if (!tryUpdateReferenceCount(NORMAL_TERMINAL_ZERO_VALUE, INITIAL_ZERO_VALUE) + && !tryUpdateReferenceCount(FORCED_TERMINAL_ZERO_VALUE, INITIAL_ZERO_VALUE)) { + throw new IllegalStateException("reference count is non-zero and cannot be reset"); + } + } + + private static boolean isInitialZero(final int countValue) { + return countValue == INITIAL_ZERO_VALUE; + } + + private static boolean isTerminalZero(final int countValue) { + return countValue == NORMAL_TERMINAL_ZERO_VALUE || countValue == FORCED_TERMINAL_ZERO_VALUE; + } + + private static boolean isZero(final int countValue) { + return isInitialZero(countValue) || isTerminalZero(countValue); + } + + /** + * Increment the reference count by 1, if it has not already been decreased to 0. + * + * @return Whether the reference count was successfully incremented + * @throws IllegalStateException If the reference count is already at its maximum referenceCount + */ + public final boolean tryIncrementReferenceCount() { + int currentReferenceCount; + while (!isTerminalZero(currentReferenceCount = getCurrentReferenceCount())) { + if (currentReferenceCount == MAXIMUM_VALUE) { + throw new IllegalStateException("reference count cannot exceed maximum value"); + } + if (tryUpdateReferenceCount(currentReferenceCount, + isInitialZero(currentReferenceCount) ? ONE_VALUE : currentReferenceCount + 1)) { + return true; + } + } + return false; + } + + /** + * Increment the reference count by one, if it has not already been decreased to zero. + * + * @throws IllegalStateException If the reference count was not successfully incremented + */ + public final void incrementReferenceCount() { + if (!tryIncrementReferenceCount()) { + throw new IllegalStateException("reference count has already reached zero"); + } + } + + /** + * Decrement the reference count by one, if it has ever been increased and has not already been decreased to zero. + * Invokes the implementation's {@link #onReferenceCountAtZero()} method if decrementing to zero. + * + * @return Whether the reference count was successfully decremented + */ + @SuppressWarnings({"WeakerAccess", "BooleanMethodIsAlwaysInverted"}) + public final boolean tryDecrementReferenceCount() { + int currentReferenceCount; + while (!isZero(currentReferenceCount = getCurrentReferenceCount())) { + if (tryUpdateReferenceCount(currentReferenceCount, currentReferenceCount - 1)) { + if (currentReferenceCount == ONE_VALUE) { // Did we just CAS from 1 to 0? + onReferenceCountAtZero(); + } + return true; + } + } + return currentReferenceCount == FORCED_TERMINAL_ZERO_VALUE; + } + + /** + * Force the reference count to zero. If it was non-zero, this will have the same side effects as returning to zero + * normally, but subsequent invocations of {@link #decrementReferenceCount()} and + * {@link #tryDecrementReferenceCount()} will act as if the reference count was successfully decremented until + * {@link #resetReferenceCount()} is invoked. + * + * @return Whether this invocation actually forced the reference count to zero (and invoked + * {@link #onReferenceCountAtZero()}. {@code false} means that this ReferenceCounted reached a zero through + * other means. + */ + public final boolean forceReferenceCountToZero() { + int currentReferenceCount; + while (!isZero(currentReferenceCount = getCurrentReferenceCount())) { + if (tryUpdateReferenceCount(currentReferenceCount, FORCED_TERMINAL_ZERO_VALUE)) { + onReferenceCountAtZero(); + return true; + } + } + return false; + } + + /** + * Decrement the reference count by one, if it has ever been increased and has not already been decreased to zero. + * Invokes the implementation's {@link #onReferenceCountAtZero()} method if decrementing to zero. + * + * @throws IllegalStateException If the reference count was not successfully decremented + */ + public final void decrementReferenceCount() { + if (!tryDecrementReferenceCount()) { + throw new IllegalStateException("reference count has been decreased more than increased"); + } + } + + /** + * Callback method that will be invoked when the reference count returns to zero. + */ + protected abstract void onReferenceCountAtZero(); +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/java/io/DataInput.java b/web/client-api/src/main/resources/io/deephaven/web/super/java/io/DataInput.java new file mode 100644 index 00000000000..8b8eb52520e --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/java/io/DataInput.java @@ -0,0 +1,19 @@ +package java.io; + +public interface DataInput { + boolean readBoolean() throws IOException; + byte readByte() throws IOException; + char readChar() throws IOException; + double readDouble() throws IOException; + float readFloat() throws IOException; + void readFully(byte[] b) throws IOException; + void readFully(byte[] b, int off, int len) throws IOException; + int readInt() throws IOException; + String readLine() throws IOException; + long readLong() throws IOException; + short readShort() throws IOException; + int readUnsignedByte() throws IOException; + int readUnsignedShort() throws IOException; + String readUTF() throws IOException; + int skipBytes(int n) throws IOException; +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/java/io/DataOutput.java b/web/client-api/src/main/resources/io/deephaven/web/super/java/io/DataOutput.java new file mode 100644 index 00000000000..0534189f69a --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/java/io/DataOutput.java @@ -0,0 +1,18 @@ +package java.io; + +public interface DataOutput { + void write(byte[] b) throws IOException; + void write(byte[] b, int off, int len) throws IOException; + void write(int b) throws IOException; + void writeBoolean(boolean v) throws IOException; + void writeByte(int v) throws IOException; + void writeBytes(String s) throws IOException; + void writeChar(int v) throws IOException; + void writeChars(String s) throws IOException; + void writeDouble(double v) throws IOException; + void writeFloat(float v) throws IOException; + void writeInt(int v) throws IOException; + void writeLong(long v) throws IOException; + void writeShort(int v) throws IOException; + void writeUTF(String s) throws IOException; +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/java/io/DataOutputStream.java b/web/client-api/src/main/resources/io/deephaven/web/super/java/io/DataOutputStream.java new file mode 100644 index 00000000000..29f380a4a12 --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/java/io/DataOutputStream.java @@ -0,0 +1,104 @@ +package java.io; + +import org.gwtproject.nio.Numbers; + +import java.nio.charset.StandardCharsets; + +public class DataOutputStream extends FilterOutputStream implements DataOutput { + protected int written; + + public DataOutputStream(OutputStream out) { + super(out); + } + + @Override + public void write(int b) throws IOException { + super.write(b); + written++; + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + super.write(b, off, len); + written += len; + } + + @Override + public void writeBoolean(boolean b) throws IOException { + write(b ? 1 : 0); + } + + @Override + public void writeByte(int i) throws IOException { + write(i); + } + + @Override + public void writeShort(int i) throws IOException { + super.write((i >> 8) & 0xFF); + super.write((i >> 0) & 0xFF); + written += 2; + } + + @Override + public void writeChar(int i) throws IOException { + super.write((i >> 8) & 0xFF); + super.write((i >> 0) & 0xFF); + written += 2; + } + + @Override + public void writeInt(int i) throws IOException { + super.write((i >> 24) & 0xFF); + super.write((i >> 16) & 0xFF); + super.write((i >> 8) & 0xFF); + super.write((i >> 0) & 0xFF); + written += 4; + } + + @Override + public void writeLong(long l) throws IOException { + super.write((int) (l >> 56) & 0xFF); + super.write((int) (l >> 48) & 0xFF); + super.write((int) (l >> 40) & 0xFF); + super.write((int) (l >> 32) & 0xFF); + super.write((int) (l >> 24) & 0xFF); + super.write((int) (l >> 16) & 0xFF); + super.write((int) (l >> 8) & 0xFF); + super.write((int) (l >> 0) & 0xFF); + written += 8; + } + + @Override + public void writeFloat(float v) throws IOException { + writeInt(Numbers.floatToIntBits(v)); + } + + @Override + public void writeDouble(double v) throws IOException { + writeLong(Numbers.doubleToRawLongBits(v)); + } + + @Override + public void writeBytes(String s) throws IOException { + for (int i = 0; i < s.length(); i++) { + super.write(s.charAt(i) & 0xFF); + } + written += s.length(); + } + + @Override + public void writeChars(String s) throws IOException { + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + super.write((c >> 8) & 0xFF); + super.write(c & 0xFF); + } + written += s.length() * 2; + } + + @Override + public void writeUTF(String s) throws IOException { + throw new UnsupportedOperationException("modified utf-8"); + } +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/java/io/EOFException.java b/web/client-api/src/main/resources/io/deephaven/web/super/java/io/EOFException.java new file mode 100644 index 00000000000..ac8a7959205 --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/java/io/EOFException.java @@ -0,0 +1,10 @@ +package java.io; + +public class EOFException extends IOException { + public EOFException() { + } + + public EOFException(String s) { + super(s); + } +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/java/io/UTFDataFormatException.java b/web/client-api/src/main/resources/io/deephaven/web/super/java/io/UTFDataFormatException.java new file mode 100644 index 00000000000..e2c6a3d3bd8 --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/java/io/UTFDataFormatException.java @@ -0,0 +1,11 @@ +package java.io; + +public class UTFDataFormatException extends IOException { + public UTFDataFormatException() { + super(); + } + + public UTFDataFormatException(String message) { + super(message); + } +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/java/lang/ref/Reference.java b/web/client-api/src/main/resources/io/deephaven/web/super/java/lang/ref/Reference.java new file mode 100644 index 00000000000..2f9dc276b00 --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/java/lang/ref/Reference.java @@ -0,0 +1,44 @@ +package java.lang.ref; + +import io.deephaven.web.client.fu.JsWeakRef; + +public abstract class Reference { + + private JsWeakRef jsWeakRef; + + Reference(T referent) { + this(referent, (ReferenceQueue) null); + } + + Reference(T referent, ReferenceQueue queue) { + if (referent != null) { + jsWeakRef = new JsWeakRef<>(referent); + } + } + + public T get() { + if (jsWeakRef == null) { + return null; + } + return jsWeakRef.deref(); + } + + public void clear() { + if (jsWeakRef != null) { + jsWeakRef = null; + } + } + + public boolean isEnqueued() { + return false; + } + + public boolean enqueue() { + throw new IllegalStateException("never called when emulated"); + } + + protected Object clone() throws CloneNotSupportedException { + throw new CloneNotSupportedException(); + } + +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/java/lang/ref/ReferenceQueue.java b/web/client-api/src/main/resources/io/deephaven/web/super/java/lang/ref/ReferenceQueue.java new file mode 100644 index 00000000000..1e2dfd25483 --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/java/lang/ref/ReferenceQueue.java @@ -0,0 +1,5 @@ +package java.lang.ref; + +public class ReferenceQueue { + +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/java/lang/ref/SoftReference.java b/web/client-api/src/main/resources/io/deephaven/web/super/java/lang/ref/SoftReference.java new file mode 100644 index 00000000000..f18650366ea --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/java/lang/ref/SoftReference.java @@ -0,0 +1,10 @@ +package java.lang.ref; + +public class SoftReference extends Reference { + public SoftReference(T referent) { + super(referent); + } + public SoftReference(T referent, ReferenceQueue q) { + super(referent, q); + } +} diff --git a/web/client-api/src/main/resources/io/deephaven/web/super/java/math/BigInteger.java b/web/client-api/src/main/resources/io/deephaven/web/super/java/math/BigInteger.java new file mode 100644 index 00000000000..500451f2f96 --- /dev/null +++ b/web/client-api/src/main/resources/io/deephaven/web/super/java/math/BigInteger.java @@ -0,0 +1,1599 @@ +/* + * Copyright 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * INCLUDES MODIFICATIONS BY RICHARD ZSCHECH AS WELL AS GOOGLE. + */ +package java.math; + +import static javaemul.internal.Coercions.ensureInt; +import static javaemul.internal.InternalPreconditions.checkCriticalArgument; +import static javaemul.internal.InternalPreconditions.checkNotNull; + +import java.io.Serializable; +import java.util.Random; +import javaemul.internal.LongUtils; + +/** + * This class represents immutable integer numbers of arbitrary length. Large + * numbers are typically used in security applications and therefore BigIntegers + * offer dedicated functionality like the generation of large prime numbers or + * the computation of modular inverse. + *

+ * Since the class was modeled to offer all the functionality as the + * {@link Integer} class does, it provides even methods that operate bitwise on + * a two's complement representation of large integers. Note however that the + * implementations favors an internal representation where magnitude and sign + * are treated separately. Hence such operations are inefficient and should be + * discouraged. In simple words: Do NOT implement any bit fields based on + * BigInteger. + */ +public class BigInteger extends Number implements Comparable, + Serializable { + + /** + * The {@code BigInteger} constant 1. + */ + public static final BigInteger ONE = new BigInteger(1, 1); + + /* Fields used for the internal representation. */ + + /** + * The {@code BigInteger} constant 10. + */ + public static final BigInteger TEN = new BigInteger(1, 10); + + /** + * The {@code BigInteger} constant 0. + */ + public static final BigInteger ZERO = new BigInteger(0, 0); + + /** + * The {@code BigInteger} constant 0 used for comparison. + */ + static final int EQUALS = 0; + + /** + * The {@code BigInteger} constant 1 used for comparison. + */ + static final int GREATER = 1; + + /** + * The {@code BigInteger} constant -1 used for comparison. + */ + static final int LESS = -1; + + /** + * The {@code BigInteger} constant -1. + */ + static final BigInteger MINUS_ONE = new BigInteger(-1, 1); + + /** + * All the {@code BigInteger} numbers in the range [0,10] are cached. + */ + static final BigInteger[] SMALL_VALUES = { + ZERO, ONE, new BigInteger(1, 2), new BigInteger(1, 3), + new BigInteger(1, 4), new BigInteger(1, 5), new BigInteger(1, 6), + new BigInteger(1, 7), new BigInteger(1, 8), new BigInteger(1, 9), TEN}; + + static final BigInteger[] TWO_POWS; + + /** + * This is the serialVersionUID used by the sun implementation. + */ + private static final long serialVersionUID = -8287574255936472291L; + + static { + TWO_POWS = new BigInteger[32]; + for (int i = 0; i < TWO_POWS.length; i++) { + TWO_POWS[i] = BigInteger.valueOf(1L << i); + } + } + + /** + * Returns a random positive {@code BigInteger} instance in the range [0, + * 2^(bitLength)-1] which is probably prime. The probability that the returned + * {@code BigInteger} is prime is beyond (1-1/2^80). + *

+ * Implementation Note: Currently {@code rnd} is ignored. + * + * @param bitLength length of the new {@code BigInteger} in bits. + * @param rnd random generator used to generate the new {@code BigInteger}. + * @return probably prime random {@code BigInteger} instance. + * @throws ArithmeticException if {@code bitLength < 2}. + */ + public static BigInteger probablePrime(int bitLength, Random rnd) { + return new BigInteger(bitLength, 100, rnd); + } + + public static BigInteger valueOf(long val) { + return val >= 0 ? BigInteger.fromBits(val) : BigInteger.fromBits(-val).negate(); + } + + private static BigInteger fromBits(long bits) { + int lowBits = (int) bits; + int highBits = LongUtils.getHighBits(bits); + if (highBits != 0) { + return new BigInteger(1, lowBits, highBits); + } + if (lowBits > 10 || lowBits < 0) { + return new BigInteger(1, lowBits); + } + return SMALL_VALUES[lowBits]; + } + + static BigInteger getPowerOfTwo(int exp) { + if (exp < TWO_POWS.length) { + return TWO_POWS[exp]; + } + int intCount = exp >> 5; + int bitN = exp & 31; + int resDigits[] = new int[intCount + 1]; + resDigits[intCount] = 1 << bitN; + return new BigInteger(1, intCount + 1, resDigits); + } + + /** + * @see BigInteger#BigInteger(String, int) + */ + private static void setFromString(BigInteger bi, String val, int radix) { + int sign; + int[] digits; + int numberLength; + int stringLength = val.length(); + int startChar; + int endChar = stringLength; + + if (val.charAt(0) == '-') { + sign = -1; + startChar = 1; + stringLength--; + } else { + sign = 1; + startChar = 0; + } + /* + * We use the following algorithm: split a string into portions of n + * characters and convert each portion to an integer according to the radix. + * Then convert an exp(radix, n) based number to binary using the + * multiplication method. See D. Knuth, The Art of Computer Programming, + * vol. 2. + */ + + int charsPerInt = Conversion.digitFitInInt[radix]; + int bigRadixDigitsLength = stringLength / charsPerInt; + int topChars = stringLength % charsPerInt; + + if (topChars != 0) { + bigRadixDigitsLength++; + } + digits = new int[bigRadixDigitsLength]; + // Get the maximal power of radix that fits in int + int bigRadix = Conversion.bigRadices[radix - 2]; + // Parse an input string and accumulate the BigInteger's magnitude + int digitIndex = 0; // index of digits array + int substrEnd = startChar + ((topChars == 0) ? charsPerInt : topChars); + int newDigit; + + for (int substrStart = startChar; substrStart < endChar; substrStart = substrEnd, substrEnd = substrStart + + charsPerInt) { + int bigRadixDigit = Integer.parseInt( + val.substring(substrStart, substrEnd), radix); + newDigit = Multiplication.multiplyByInt(digits, digitIndex, bigRadix); + newDigit += Elementary.inplaceAdd(digits, digitIndex, bigRadixDigit); + digits[digitIndex++] = newDigit; + } + numberLength = digitIndex; + bi.sign = sign; + bi.numberLength = numberLength; + bi.digits = digits; + bi.cutOffLeadingZeroes(); + } + + /** + * The magnitude of this big integer. This array is in little endian order and + * each "digit" is a 32-bit unsigned integer. For example: {@code 13} is + * represented as [ 13 ] {@code -13} is represented as [ 13 ] {@code 2^32 + + * 13} is represented as [ 13, 1 ] {@code 2^64 + 13} is represented as [ 13, + * 0, 1 ] {@code 2^31} is represented as [ Integer.MIN_VALUE ] The magnitude + * array may be longer than strictly necessary, which results in additional + * trailing zeros. + * + *

TODO(jat): consider changing to 24-bit integers for better performance + * in browsers. + */ + transient int digits[]; + + /** + * The length of this in measured in ints. Can be less than digits.length(). + */ + transient int numberLength; + + /** + * The sign of this. + */ + transient int sign; + + private transient int firstNonzeroDigit = -2; + + /** + * Cache for the hash code. + */ + private transient int hashCode = 0; + + /** + * Constructs a new {@code BigInteger} from the given two's complement + * representation. The most significant byte is the entry at index 0. The most + * significant bit of this entry determines the sign of the new {@code + * BigInteger} instance. The given array must not be empty. + * + * @param val two's complement representation of the new {@code BigInteger}. + * @throws NullPointerException if {@code val == null}. + * @throws NumberFormatException if the length of {@code val} is zero. + */ + public BigInteger(byte[] val) { + this(val, 0, val.length); + } + + /** + * Constructs a new {@code BigInteger} from the given two's complement + * representation. The most significant byte is the entry at index 0. The most + * significant bit of this entry determines the sign of the new {@code + * BigInteger} instance. The given array must not be empty. + * + * @param val two's complement representation of the new {@code BigInteger}. + * @param offset the start offset of the binary representation. + * @param length the number of bytes to use. + * @throws NullPointerException if {@code val == null}. + * @throws NumberFormatException if the length of {@code val} is zero. + */ + public BigInteger(byte[] val, int offset, int length) { + if (val.length == 0) { + // math.12=Zero length BigInteger + throw new NumberFormatException("Zero length BigInteger"); //$NON-NLS-1$ + } + if (length < 0 || offset < 0 || length > val.length - offset) { + throw new IndexOutOfBoundsException("Range check failed: offset=" + offset + ", length=" + + length + ", val.length=" + val.length); + } + if (length == 0) { + sign = 0; + numberLength = 1; + digits = new int[] {0}; + return; + } + if (val[offset] < 0) { + sign = -1; + putBytesNegativeToIntegers(val, offset, length); + } else { + sign = 1; + putBytesPositiveToIntegers(val, offset, length); + } + cutOffLeadingZeroes(); + } + + /** + * Constructs a new {@code BigInteger} instance with the given sign and the + * given magnitude. The sign is given as an integer (-1 for negative, 0 for + * zero, 1 for positive). The magnitude is specified as a byte array. The most + * significant byte is the entry at index 0. + * + * @param signum sign of the new {@code BigInteger} (-1 for negative, 0 for + * zero, 1 for positive). + * @param magnitude magnitude of the new {@code BigInteger} with the most + * significant byte first. + * @throws NullPointerException if {@code magnitude == null}. + * @throws NumberFormatException if the sign is not one of -1, 0, 1 or if the + * sign is zero and the magnitude contains non-zero entries. + */ + public BigInteger(int signum, byte[] magnitude) { + this(signum, magnitude, 0, magnitude.length); + } + + /** + * Constructs a new {@code BigInteger} instance with the given sign and the + * given magnitude. The sign is given as an integer (-1 for negative, 0 for + * zero, 1 for positive). The magnitude is specified as a byte array. The most + * significant byte is the entry at index 0. + * + * @param signum sign of the new {@code BigInteger} (-1 for negative, 0 for + * zero, 1 for positive). + * @param magnitude magnitude of the new {@code BigInteger} with the most + * significant byte first. + * @param offset the start offset of the binary representation. + * @param length the number of bytes to use. + * @throws NullPointerException if {@code magnitude == null}. + * @throws NumberFormatException if the sign is not one of -1, 0, 1 or if the + * sign is zero and the magnitude contains non-zero entries. + */ + public BigInteger(int signum, byte[] magnitude, int offset, int length) { + checkNotNull(magnitude); + + if (length < 0 || offset < 0 || length > magnitude.length - offset) { + throw new IndexOutOfBoundsException("Range check failed: offset=" + offset + ", length=" + + length + ", val.length=" + magnitude.length); + } + + if ((signum < -1) || (signum > 1)) { + // math.13=Invalid signum value + throw new NumberFormatException("Invalid signum value"); //$NON-NLS-1$ + } + if (signum == 0) { + for (int index = offset; index < offset + length; index++) { + byte element = magnitude[index]; + if (element != 0) { + // math.14=signum-magnitude mismatch + throw new NumberFormatException("signum-magnitude mismatch"); //$NON-NLS-1$ + } + } + } + if (length == 0) { + sign = 0; + numberLength = 1; + digits = new int[] {0}; + } else { + sign = signum; + putBytesPositiveToIntegers(magnitude, offset, length); + cutOffLeadingZeroes(); + } + } + + /** + * Constructs a random {@code BigInteger} instance in the range [0, + * 2^(bitLength)-1] which is probably prime. The probability that the returned + * {@code BigInteger} is prime is beyond (1-1/2^certainty). + * + * @param bitLength length of the new {@code BigInteger} in bits. + * @param certainty tolerated primality uncertainty. + * @param rnd is an optional random generator to be used. + * @throws ArithmeticException if {@code bitLength} < 2. + */ + public BigInteger(int bitLength, int certainty, Random rnd) { + if (bitLength < 2) { + // math.1C=bitLength < 2 + throw new ArithmeticException("bitLength < 2"); //$NON-NLS-1$ + } + BigInteger me = Primality.consBigInteger(bitLength, certainty, rnd); + sign = me.sign; + numberLength = me.numberLength; + digits = me.digits; + } + + /** + * Constructs a random non-negative {@code BigInteger} instance in the range + * [0, 2^(numBits)-1]. + * + * @param numBits maximum length of the new {@code BigInteger} in bits. + * @param rnd is an optional random generator to be used. + * @throws IllegalArgumentException if {@code numBits} < 0. + */ + public BigInteger(int numBits, Random rnd) { + checkCriticalArgument(numBits >= 0, "numBits must be non-negative"); + + if (numBits == 0) { + sign = 0; + numberLength = 1; + digits = new int[] {0}; + } else { + sign = 1; + numberLength = (numBits + 31) >> 5; + digits = new int[numberLength]; + for (int i = 0; i < numberLength; i++) { + digits[i] = rnd.nextInt(); + } + // Using only the necessary bits + digits[numberLength - 1] >>>= (-numBits) & 31; + cutOffLeadingZeroes(); + } + } + + /** + * Constructs a new {@code BigInteger} instance from the string + * representation. The string representation consists of an optional minus + * sign followed by a non-empty sequence of decimal digits. + * + * @param val string representation of the new {@code BigInteger}. + * @throws NullPointerException if {@code val == null}. + * @throws NumberFormatException if {@code val} is not a valid representation + * of a {@code BigInteger}. + */ + public BigInteger(String val) { + this(val, 10); + } + + /** + * Constructs a new {@code BigInteger} instance from the string + * representation. The string representation consists of an optional minus + * sign followed by a non-empty sequence of digits in the specified radix. For + * the conversion the method {@code Character.digit(char, radix)} is used. + * + * @param val string representation of the new {@code BigInteger}. + * @param radix the base to be used for the conversion. + * @throws NullPointerException if {@code val == null}. + * @throws NumberFormatException if {@code val} is not a valid representation + * of a {@code BigInteger} or if {@code radix < Character.MIN_RADIX} + * or {@code radix > Character.MAX_RADIX}. + */ + public BigInteger(String val, int radix) { + checkNotNull(val); + + if ((radix < Character.MIN_RADIX) || (radix > Character.MAX_RADIX)) { + // math.11=Radix out of range + throw new NumberFormatException("Radix out of range"); //$NON-NLS-1$ + } + if (val.isEmpty()) { + // math.12=Zero length BigInteger + throw new NumberFormatException("Zero length BigInteger"); //$NON-NLS-1$ + } + setFromString(this, val, radix); + } + + /** + * Constructs a number which array is of size 1. + * + * @param sign the sign of the number + * @param value the only one digit of array + */ + BigInteger(int sign, int bits) { + this(sign, 1, new int[] {bits}); + } + + BigInteger(int sign, int lowBits, int highBits) { + this(sign, 2, new int[] {lowBits, highBits}); + } + + /** + * Creates a new {@code BigInteger} with the given sign and magnitude. This constructor does not + * create a copy, so any changes to the reference will affect the new number. + * + * @param signum The sign of the number represented by {@code digits} + * @param digits The magnitude of the number + */ + BigInteger(int signum, int digits[]) { + if (digits.length == 0) { + sign = 0; + numberLength = 1; + this.digits = new int[] {0}; + } else { + sign = signum; + numberLength = digits.length; + this.digits = digits; + cutOffLeadingZeroes(); + } + } + + /** + * Constructs a number without to create new space. This construct should be used only if the + * three fields of representation are known. + * + * @param sign the sign of the number + * @param numberLength the length of the internal array + * @param digits a reference of some array created before + */ + BigInteger(int sign, int numberLength, int[] digits) { + this.sign = sign; + this.numberLength = numberLength; + this.digits = digits; + } + + /** + * Returns a (new) {@code BigInteger} whose value is the absolute value of + * {@code this}. + * + * @return {@code abs(this)}. + */ + public BigInteger abs() { + return sign < 0 ? negate() : this; + } + + /** + * Returns a new {@code BigInteger} whose value is {@code this + val}. + * + * @param val value to be added to {@code this}. + * @return {@code this + val}. + * @throws NullPointerException if {@code val == null}. + */ + public BigInteger add(BigInteger val) { + return Elementary.add(this, val); + } + + /** + * Returns a new {@code BigInteger} whose value is {@code this & val}. + *

+ * Implementation Note: Usage of this method is not recommended as the + * current implementation is not efficient. + * + * @param val value to be and'ed with {@code this}. + * @return {@code this & val}. + * @throws NullPointerException if {@code val == null}. + */ + public BigInteger and(BigInteger val) { + return Logical.and(this, val); + } + + /** + * Returns a new {@code BigInteger} whose value is {@code this & ~val}. + * Evaluating {@code x.andNot(val)} returns the same result as {@code + * x.and(val.not())}. + *

+ * Implementation Note: Usage of this method is not recommended as the + * current implementation is not efficient. + * + * @param val value to be not'ed and then and'ed with {@code this}. + * @return {@code this & ~val}. + * @throws NullPointerException if {@code val == null}. + */ + public BigInteger andNot(BigInteger val) { + return Logical.andNot(this, val); + } + + /** + * Use {@code bitLength(0)} if you want to know the length of the binary value + * in bits. + *

+ * Returns the number of bits in the binary representation of {@code this} + * which differ from the sign bit. If {@code this} is positive the result is + * equivalent to the number of bits set in the binary representation of + * {@code this}. If {@code this} is negative the result is equivalent to the + * number of bits set in the binary representation of {@code -this-1}. + *

+ * Implementation Note: Usage of this method is not recommended as the + * current implementation is not efficient. + * + * @return number of bits in the binary representation of {@code this} which + * differ from the sign bit + */ + public int bitCount() { + return BitLevel.bitCount(this); + } + + /** + * Returns the length of the value's two's complement representation without + * leading zeros for positive numbers / without leading ones for negative + * values. + *

+ * The two's complement representation of {@code this} will be at least + * {@code bitLength() + 1} bits long. + *

+ * The value will fit into an {@code int} if {@code bitLength() < 32} or into + * a {@code long} if {@code bitLength() < 64}. + * + * @return the length of the minimal two's complement representation for + * {@code this} without the sign bit. + */ + public int bitLength() { + return BitLevel.bitLength(this); + } + + /** + * Converts value of this {@code BigInteger} to a {@code byte} if it fits it, + * otherwise {@code ArithmeticException} is thrown. + * + * @return this {@code BigInteger} converted to a {@code byte}. + * @throws ArithmeticException if the value of this {@code BigInteger} + * does not fit in a {@code byte}. + */ + public byte byteValueExact() { + if (numberLength <= 1 && bitLength() < Byte.SIZE) { + return byteValue(); + } + throw new ArithmeticException("out of byte range"); + } + + /** + * Returns a new {@code BigInteger} which has the same binary representation + * as {@code this} but with the bit at position n cleared. The result is + * equivalent to {@code this & ~(2^n)}. + *

+ * Implementation Note: Usage of this method is not recommended as the + * current implementation is not efficient. + * + * @param n position where the bit in {@code this} has to be cleared. + * @return {@code this & ~(2^n)}. + * @throws ArithmeticException if {@code n < 0}. + */ + public BigInteger clearBit(int n) { + if (testBit(n)) { + return BitLevel.flipBit(this, n); + } + return this; + } + + /** + * Compares this {@code BigInteger} with {@code val}. Returns one of the three + * values 1, 0, or -1. + * + * @param val value to be compared with {@code this}. + * @return {@code 1} if {@code this > val}, {@code -1} if {@code this < val} , + * {@code 0} if {@code this == val}. + * @throws NullPointerException if {@code val == null}. + */ + @Override + public int compareTo(BigInteger val) { + if (sign > val.sign) { + return GREATER; + } + if (sign < val.sign) { + return LESS; + } + if (numberLength > val.numberLength) { + return sign; + } + if (numberLength < val.numberLength) { + return -val.sign; + } + // Equal sign and equal numberLength + return (sign * Elementary.compareArrays(digits, val.digits, numberLength)); + } + + /** + * Returns a new {@code BigInteger} whose value is {@code this / divisor}. + * + * @param divisor value by which {@code this} is divided. + * @return {@code this / divisor}. + * @throws NullPointerException if {@code divisor == null}. + * @throws ArithmeticException if {@code divisor == 0}. + */ + public BigInteger divide(BigInteger divisor) { + if (divisor.sign == 0) { + // math.17=BigInteger divide by zero + throw new ArithmeticException("BigInteger divide by zero"); //$NON-NLS-1$ + } + int divisorSign = divisor.sign; + if (divisor.isOne()) { + return ((divisor.sign > 0) ? this : this.negate()); + } + int thisSign = sign; + int thisLen = numberLength; + int divisorLen = divisor.numberLength; + if (thisLen + divisorLen == 2) { + long val = (digits[0] & 0xFFFFFFFFL) / (divisor.digits[0] & 0xFFFFFFFFL); + if (thisSign != divisorSign) { + val = -val; + } + return valueOf(val); + } + int cmp = ((thisLen != divisorLen) ? ((thisLen > divisorLen) ? 1 : -1) + : Elementary.compareArrays(digits, divisor.digits, thisLen)); + if (cmp == EQUALS) { + return ((thisSign == divisorSign) ? ONE : MINUS_ONE); + } + if (cmp == LESS) { + return ZERO; + } + int resLength = thisLen - divisorLen + 1; + int resDigits[] = new int[resLength]; + int resSign = ((thisSign == divisorSign) ? 1 : -1); + if (divisorLen == 1) { + Division.divideArrayByInt(resDigits, digits, thisLen, divisor.digits[0]); + } else { + Division.divide(resDigits, resLength, digits, thisLen, divisor.digits, + divisorLen); + } + BigInteger result = new BigInteger(resSign, resLength, resDigits); + result.cutOffLeadingZeroes(); + return result; + } + + /** + * Returns a {@code BigInteger} array which contains {@code this / divisor} at + * index 0 and {@code this % divisor} at index 1. + * + * @param divisor value by which {@code this} is divided. + * @return {@code [this / divisor, this % divisor]}. + * @throws NullPointerException if {@code divisor == null}. + * @throws ArithmeticException if {@code divisor == 0}. + * @see #divide + * @see #remainder + */ + public BigInteger[] divideAndRemainder(BigInteger divisor) { + int divisorSign = divisor.sign; + if (divisorSign == 0) { + // math.17=BigInteger divide by zero + throw new ArithmeticException("BigInteger divide by zero"); //$NON-NLS-1$ + } + int divisorLen = divisor.numberLength; + int[] divisorDigits = divisor.digits; + if (divisorLen == 1) { + return Division.divideAndRemainderByInteger(this, divisorDigits[0], + divisorSign); + } + // res[0] is a quotient and res[1] is a remainder: + int[] thisDigits = digits; + int thisLen = numberLength; + int cmp = (thisLen != divisorLen) ? ((thisLen > divisorLen) ? 1 : -1) + : Elementary.compareArrays(thisDigits, divisorDigits, thisLen); + if (cmp < 0) { + return new BigInteger[] {ZERO, this}; + } + int thisSign = sign; + int quotientLength = thisLen - divisorLen + 1; + int remainderLength = divisorLen; + int quotientSign = ((thisSign == divisorSign) ? 1 : -1); + int quotientDigits[] = new int[quotientLength]; + int remainderDigits[] = Division.divide(quotientDigits, quotientLength, + thisDigits, thisLen, divisorDigits, divisorLen); + BigInteger result0 = new BigInteger(quotientSign, quotientLength, + quotientDigits); + BigInteger result1 = new BigInteger(thisSign, remainderLength, + remainderDigits); + result0.cutOffLeadingZeroes(); + result1.cutOffLeadingZeroes(); + return new BigInteger[] {result0, result1}; + } + + /** + * Returns this {@code BigInteger} as an double value. If {@code this} is too + * big to be represented as an double, then {@code Double.POSITIVE_INFINITY} + * or {@code Double.NEGATIVE_INFINITY} is returned. Note, that not all + * integers x in the range [-Double.MAX_VALUE, Double.MAX_VALUE] can be + * represented as a double. The double representation has a mantissa of length + * 53. For example, 2^53+1 = 9007199254740993 is returned as double + * 9007199254740992.0. + * + * @return this {@code BigInteger} as a double value + */ + @Override + public double doubleValue() { + return Double.parseDouble(this.toString()); + } + + /** + * Returns {@code true} if {@code x} is a BigInteger instance and if this + * instance is equal to this {@code BigInteger}. + * + * @param x object to be compared with {@code this}. + * @return true if {@code x} is a BigInteger and {@code this == x}, {@code + * false} otherwise. + */ + @Override + public boolean equals(Object x) { + if (this == x) { + return true; + } + if (x instanceof BigInteger) { + BigInteger x1 = (BigInteger) x; + return sign == x1.sign && numberLength == x1.numberLength + && equalsArrays(x1.digits); + } + return false; + } + + /** + * Returns a new {@code BigInteger} which has the same binary representation + * as {@code this} but with the bit at position n flipped. The result is + * equivalent to {@code this ^ 2^n}. + *

+ * Implementation Note: Usage of this method is not recommended as the + * current implementation is not efficient. + * + * @param n position where the bit in {@code this} has to be flipped. + * @return {@code this ^ 2^n}. + * @throws ArithmeticException if {@code n < 0}. + */ + public BigInteger flipBit(int n) { + if (n < 0) { + // math.15=Negative bit address + throw new ArithmeticException("Negative bit address"); //$NON-NLS-1$ + } + return BitLevel.flipBit(this, n); + } + + /** + * Returns this {@code BigInteger} as an float value. If {@code this} is too + * big to be represented as an float, then {@code Float.POSITIVE_INFINITY} or + * {@code Float.NEGATIVE_INFINITY} is returned. Note, that not all integers x + * in the range [-Float.MAX_VALUE, Float.MAX_VALUE] can be represented as a + * float. The float representation has a mantissa of length 24. For example, + * 2^24+1 = 16777217 is returned as float 16777216.0. + * + * @return this {@code BigInteger} as a float value. + */ + @Override + public float floatValue() { + return Float.parseFloat(this.toString()); + } + + /** + * Returns a new {@code BigInteger} whose value is greatest common divisor of + * {@code this} and {@code val}. If {@code this==0} and {@code val==0} then + * zero is returned, otherwise the result is positive. + * + * @param val value with which the greatest common divisor is computed. + * @return {@code gcd(this, val)}. + * @throws NullPointerException if {@code val == null}. + */ + public BigInteger gcd(BigInteger val) { + BigInteger val1 = this.abs(); + BigInteger val2 = val.abs(); + // To avoid a possible division by zero + if (val1.signum() == 0) { + return val2; + } else if (val2.signum() == 0) { + return val1; + } + + // Optimization for small operands + // (op2.bitLength() < 64) and (op1.bitLength() < 64) + if (((val1.numberLength == 1) || ((val1.numberLength == 2) && (val1.digits[1] > 0))) + && (val2.numberLength == 1 || (val2.numberLength == 2 && val2.digits[1] > 0))) { + return BigInteger.valueOf(Division.gcdBinary(val1.longValue(), + val2.longValue())); + } + + return Division.gcdBinary(val1.copy(), val2.copy()); + } + + /** + * Returns the position of the lowest set bit in the two's complement + * representation of this {@code BigInteger}. If all bits are zero (this=0) + * then -1 is returned as result. + *

+ * Implementation Note: Usage of this method is not recommended as the + * current implementation is not efficient. + * + * @return position of lowest bit if {@code this != 0}, {@code -1} otherwise + */ + public int getLowestSetBit() { + if (sign == 0) { + return -1; + } + // (sign != 0) implies that exists some non zero digit + int i = getFirstNonzeroDigit(); + return ((i << 5) + Integer.numberOfTrailingZeros(digits[i])); + } + + /** + * Returns a hash code for this {@code BigInteger}. + * + * @return hash code for {@code this}. + */ + @Override + public int hashCode() { + if (hashCode != 0) { + return hashCode; + } + for (int i = 0; i < digits.length; i++) { + hashCode = (hashCode * 33 + (digits[i] & 0xffffffff)); + } + hashCode = hashCode * sign; + return hashCode; + } + + /** + * Returns this {@code BigInteger} as an int value. If {@code this} is too big + * to be represented as an int, then {@code this} % 2^32 is returned. + * + * @return this {@code BigInteger} as an int value. + */ + @Override + public int intValue() { + int i = digits[0]; + // i is always positive except for Integer.MIN_VALUE because of int overflow + return sign > 0 ? i : ensureInt(-i); + } + + /** + * Converts value of this {@code BigInteger} to an {@code int} if it fits it, + * otherwise {@code ArithmeticException} is thrown. + * + * @return this {@code BigInteger} converted to an {@code int}. + * @throws ArithmeticException if the value of this {@code BigInteger} + * does not fit in an {@code int}. + */ + public int intValueExact() { + if (numberLength <= 1 && bitLength() < Integer.SIZE) { + return intValue(); + } + throw new ArithmeticException("out of int range"); + } + + /** + * Tests whether this {@code BigInteger} is probably prime. If {@code true} is + * returned, then this is prime with a probability beyond (1-1/2^certainty). + * If {@code false} is returned, then this is definitely composite. If the + * argument {@code certainty} <= 0, then this method returns true. + * + * @param certainty tolerated primality uncertainty. + * @return {@code true}, if {@code this} is probably prime, {@code false} + * otherwise. + */ + public boolean isProbablePrime(int certainty) { + return Primality.isProbablePrime(abs(), certainty); + } + + /** + * Returns this {@code BigInteger} as an long value. If {@code this} is too + * big to be represented as an long, then {@code this} % 2^64 is returned. + * + * @return this {@code BigInteger} as a long value. + */ + @Override + public long longValue() { + long value = + numberLength > 1 + ? LongUtils.fromBits(digits[0], digits[1]) + : LongUtils.fromBits(digits[0], 0); + return sign > 0 ? value : -value; + } + + /** + * Converts value of this {@code BigInteger} to a {@code long} if it fits it, + * otherwise {@code ArithmeticException} is thrown. + * + * @return this {@code BigInteger} converted to a {@code long}. + * @throws ArithmeticException if the value of this {@code BigInteger} + * does not fit in a {@code long}. + */ + public long longValueExact() { + if (numberLength <= 2 && bitLength() < Long.SIZE) { + return longValue(); + } + throw new ArithmeticException("out of long range"); + } + + /** + * Returns the maximum of this {@code BigInteger} and {@code val}. + * + * @param val value to be used to compute the maximum with {@code this} + * @return {@code max(this, val)} + * @throws NullPointerException if {@code val == null} + */ + public BigInteger max(BigInteger val) { + return ((this.compareTo(val) == GREATER) ? this : val); + } + + /** + * Returns the minimum of this {@code BigInteger} and {@code val}. + * + * @param val value to be used to compute the minimum with {@code this}. + * @return {@code min(this, val)}. + * @throws NullPointerException if {@code val == null}. + */ + public BigInteger min(BigInteger val) { + return ((this.compareTo(val) == LESS) ? this : val); + } + + /** + * Returns a new {@code BigInteger} whose value is {@code this mod m}. The + * modulus {@code m} must be positive. The result is guaranteed to be in the + * interval {@code [0, m)} (0 inclusive, m exclusive). The behavior of this + * function is not equivalent to the behavior of the % operator defined for + * the built-in {@code int}'s. + * + * @param m the modulus. + * @return {@code this mod m}. + * @throws NullPointerException if {@code m == null}. + * @throws ArithmeticException if {@code m < 0}. + */ + public BigInteger mod(BigInteger m) { + if (m.sign <= 0) { + // math.18=BigInteger: modulus not positive + throw new ArithmeticException("BigInteger: modulus not positive"); //$NON-NLS-1$ + } + BigInteger rem = remainder(m); + return ((rem.sign < 0) ? rem.add(m) : rem); + } + + // @Override + // public double doubleValue() { + // return Conversion.bigInteger2Double(this); + // } + + /** + * Returns a new {@code BigInteger} whose value is {@code 1/this mod m}. The + * modulus {@code m} must be positive. The result is guaranteed to be in the + * interval {@code [0, m)} (0 inclusive, m exclusive). If {@code this} is not + * relatively prime to m, then an exception is thrown. + * + * @param m the modulus. + * @return {@code 1/this mod m}. + * @throws NullPointerException if {@code m == null} + * @throws ArithmeticException if {@code m < 0 or} if {@code this} is not + * relatively prime to {@code m} + */ + public BigInteger modInverse(BigInteger m) { + if (m.sign <= 0) { + // math.18=BigInteger: modulus not positive + throw new ArithmeticException("BigInteger: modulus not positive"); //$NON-NLS-1$ + } + // If both are even, no inverse exists + if (!(testBit(0) || m.testBit(0))) { + // math.19=BigInteger not invertible. + throw new ArithmeticException("BigInteger not invertible."); //$NON-NLS-1$ + } + if (m.isOne()) { + return ZERO; + } + + // From now on: (m > 1) + BigInteger res = Division.modInverseMontgomery(abs().mod(m), m); + if (res.sign == 0) { + // math.19=BigInteger not invertible. + throw new ArithmeticException("BigInteger not invertible."); //$NON-NLS-1$ + } + + res = ((sign < 0) ? m.subtract(res) : res); + return res; + } + + /** + * Returns a new {@code BigInteger} whose value is {@code this^exponent mod m} + * . The modulus {@code m} must be positive. The result is guaranteed to be in + * the interval {@code [0, m)} (0 inclusive, m exclusive). If the exponent is + * negative, then {@code this.modInverse(m)^(-exponent) mod m)} is computed. + * The inverse of this only exists if {@code this} is relatively prime to m, + * otherwise an exception is thrown. + * + * @param exponent the exponent. + * @param m the modulus. + * @return {@code this^exponent mod val}. + * @throws NullPointerException if {@code m == null} or {@code exponent == + * null}. + * @throws ArithmeticException if {@code m < 0} or if {@code exponent<0} and + * this is not relatively prime to {@code m}. + */ + public BigInteger modPow(BigInteger exponent, BigInteger m) { + if (m.sign <= 0) { + // math.18=BigInteger: modulus not positive + throw new ArithmeticException("BigInteger: modulus not positive"); //$NON-NLS-1$ + } + BigInteger base = this; + + if (m.isOne() | (exponent.sign > 0 & base.sign == 0)) { + return BigInteger.ZERO; + } + if (base.sign == 0 && exponent.sign == 0) { + return BigInteger.ONE; + } + if (exponent.sign < 0) { + base = modInverse(m); + exponent = exponent.negate(); + } + // From now on: (m > 0) and (exponent >= 0) + BigInteger res = (m.testBit(0)) ? Division.oddModPow(base.abs(), exponent, + m) : Division.evenModPow(base.abs(), exponent, m); + if ((base.sign < 0) && exponent.testBit(0)) { + // -b^e mod m == ((-1 mod m) * (b^e mod m)) mod m + res = m.subtract(BigInteger.ONE).multiply(res).mod(m); + } + // else exponent is even, so base^exp is positive + return res; + } + + /** + * Returns a new {@code BigInteger} whose value is {@code this * val}. + * + * @param val value to be multiplied with {@code this}. + * @return {@code this * val}. + * @throws NullPointerException if {@code val == null}. + */ + public BigInteger multiply(BigInteger val) { + // This let us to throw NullPointerException when val == null + if (val.sign == 0 || sign == 0) { + return ZERO; + } + return Multiplication.multiply(this, val); + } + + /** + * Returns a new {@code BigInteger} whose value is the {@code -this}. + * + * @return {@code -this}. + */ + public BigInteger negate() { + return sign == 0 ? this : new BigInteger(-sign, numberLength, digits); + } + + /** + * Returns the smallest integer x > {@code this} which is probably prime as a + * {@code BigInteger} instance. The probability that the returned {@code + * BigInteger} is prime is beyond (1-1/2^80). + * + * @return smallest integer > {@code this} which is robably prime. + * @throws ArithmeticException if {@code this < 0}. + */ + public BigInteger nextProbablePrime() { + if (sign < 0) { + // math.1A=start < 0: {0} + throw new ArithmeticException("start < 0: " + this); //$NON-NLS-1$ + } + return Primality.nextProbablePrime(this); + } + + /** + * Returns a new {@code BigInteger} whose value is {@code ~this}. The result + * of this operation is {@code -this-1}. + *

+ * Implementation Note: Usage of this method is not recommended as the + * current implementation is not efficient. + * + * @return {@code ~this}. + */ + public BigInteger not() { + return Logical.not(this); + } + + /** + * Returns a new {@code BigInteger} whose value is {@code this | val}. + *

+ * Implementation Note: Usage of this method is not recommended as the + * current implementation is not efficient. + * + * @param val value to be or'ed with {@code this}. + * @return {@code this | val}. + * @throws NullPointerException if {@code val == null}. + */ + public BigInteger or(BigInteger val) { + return Logical.or(this, val); + } + + /** + * Returns a new {@code BigInteger} whose value is {@code this ^ exp}. + * + * @param exp exponent to which {@code this} is raised. + * @return {@code this ^ exp}. + * @throws ArithmeticException if {@code exp < 0}. + */ + public BigInteger pow(int exp) { + if (exp < 0) { + // math.16=Negative exponent + throw new ArithmeticException("Negative exponent"); //$NON-NLS-1$ + } + if (exp == 0) { + return ONE; + } else if (exp == 1 || equals(ONE) || equals(ZERO)) { + return this; + } + + // if even take out 2^x factor which we can + // calculate by shifting. + if (!testBit(0)) { + int x = 1; + while (!testBit(x)) { + x++; + } + return getPowerOfTwo(x * exp).multiply(this.shiftRight(x).pow(exp)); + } + return Multiplication.pow(this, exp); + } + + /** + * Returns a new {@code BigInteger} whose value is {@code this % divisor}. + * Regarding signs this methods has the same behavior as the % operator on + * int's, i.e. the sign of the remainder is the same as the sign of this. + * + * @param divisor value by which {@code this} is divided. + * @return {@code this % divisor}. + * @throws NullPointerException if {@code divisor == null}. + * @throws ArithmeticException if {@code divisor == 0}. + */ + public BigInteger remainder(BigInteger divisor) { + if (divisor.sign == 0) { + // math.17=BigInteger divide by zero + throw new ArithmeticException("BigInteger divide by zero"); //$NON-NLS-1$ + } + int thisLen = numberLength; + int divisorLen = divisor.numberLength; + if (((thisLen != divisorLen) ? ((thisLen > divisorLen) ? 1 : -1) + : Elementary.compareArrays(digits, divisor.digits, thisLen)) == LESS) { + return this; + } + int resLength = divisorLen; + int resDigits[] = new int[resLength]; + if (resLength == 1) { + resDigits[0] = Division.remainderArrayByInt(digits, thisLen, + divisor.digits[0]); + } else { + int qLen = thisLen - divisorLen + 1; + resDigits = Division.divide(null, qLen, digits, thisLen, divisor.digits, + divisorLen); + } + BigInteger result = new BigInteger(sign, resLength, resDigits); + result.cutOffLeadingZeroes(); + return result; + } + + /** + * Returns a new {@code BigInteger} which has the same binary representation + * as {@code this} but with the bit at position n set. The result is + * equivalent to {@code this | 2^n}. + *

+ * Implementation Note: Usage of this method is not recommended as the + * current implementation is not efficient. + * + * @param n position where the bit in {@code this} has to be set. + * @return {@code this | 2^n}. + * @throws ArithmeticException if {@code n < 0}. + */ + public BigInteger setBit(int n) { + if (!testBit(n)) { + return BitLevel.flipBit(this, n); + } + return this; + } + + /** + * Returns a new {@code BigInteger} whose value is {@code this << n}. The + * result is equivalent to {@code this * 2^n} if n >= 0. The shift distance + * may be negative which means that {@code this} is shifted right. The result + * then corresponds to {@code floor(this / 2^(-n))}. + *

+ * Implementation Note: Usage of this method on negative values is not + * recommended as the current implementation is not efficient. + * + * @param n shift distance. + * @return {@code this << n} if {@code n >= 0}; {@code this >> (-n)}. + * otherwise + */ + public BigInteger shiftLeft(int n) { + if ((n == 0) || (sign == 0)) { + return this; + } + return ((n > 0) ? BitLevel.shiftLeft(this, n) : BitLevel.shiftRight(this, + -n)); + } + + /** + * Returns a new {@code BigInteger} whose value is {@code this >> n}. For + * negative arguments, the result is also negative. The shift distance may be + * negative which means that {@code this} is shifted left. + *

+ * Implementation Note: Usage of this method on negative values is not + * recommended as the current implementation is not efficient. + * + * @param n shift distance + * @return {@code this >> n} if {@code n >= 0}; {@code this << (-n)} otherwise + */ + public BigInteger shiftRight(int n) { + if ((n == 0) || (sign == 0)) { + return this; + } + return ((n > 0) ? BitLevel.shiftRight(this, n) : BitLevel.shiftLeft(this, + -n)); + } + + /** + * Converts value of this {@code BigInteger} to a {@code short} if it fits it, + * otherwise {@code ArithmeticException} is thrown. + * + * @return this {@code BigInteger} converted to a {@code short}. + * @throws ArithmeticException if the value of this {@code BigInteger} + * does not fit in a {@code short}. + */ + public short shortValueExact() { + if (numberLength <= 1 && bitLength() < Short.SIZE) { + return shortValue(); + } + throw new ArithmeticException("out of short range"); + } + + /** + * Returns the sign of this {@code BigInteger}. + * + * @return {@code -1} if {@code this < 0}, {@code 0} if {@code this == 0}, + * {@code 1} if {@code this > 0}. + */ + public int signum() { + return sign; + } + + /** + * Returns a new {@code BigInteger} whose value is {@code this - val}. + * + * @param val value to be subtracted from {@code this}. + * @return {@code this - val}. + * @throws NullPointerException if {@code val == null}. + */ + public BigInteger subtract(BigInteger val) { + return Elementary.subtract(this, val); + } + + /** + * Tests whether the bit at position n in {@code this} is set. The result is + * equivalent to {@code this & (2^n) != 0}. + *

+ * Implementation Note: Usage of this method is not recommended as the + * current implementation is not efficient. + * + * @param n position where the bit in {@code this} has to be inspected. + * @return {@code this & (2^n) != 0}. + * @throws ArithmeticException if {@code n < 0}. + */ + public boolean testBit(int n) { + if (n == 0) { + return ((digits[0] & 1) != 0); + } + if (n < 0) { + // math.15=Negative bit address + throw new ArithmeticException("Negative bit address"); //$NON-NLS-1$ + } + int intCount = n >> 5; + if (intCount >= numberLength) { + return (sign < 0); + } + int digit = digits[intCount]; + n = (1 << (n & 31)); // int with 1 set to the needed position + if (sign < 0) { + int firstNonZeroDigit = getFirstNonzeroDigit(); + if (intCount < firstNonZeroDigit) { + return false; + } else if (firstNonZeroDigit == intCount) { + digit = -digit; + } else { + digit = ~digit; + } + } + return ((digit & n) != 0); + } + + /** + * Returns the two's complement representation of this BigInteger in a byte + * array. + * + * @return two's complement representation of {@code this}. + */ + public byte[] toByteArray() { + if (this.sign == 0) { + return new byte[] {0}; + } + BigInteger temp = this; + int bitLen = bitLength(); + int iThis = getFirstNonzeroDigit(); + int bytesLen = (bitLen >> 3) + 1; + /* + * Puts the little-endian int array representing the magnitude of this + * BigInteger into the big-endian byte array. + */ + byte[] bytes = new byte[bytesLen]; + int firstByteNumber = 0; + int highBytes; + int digitIndex = 0; + int bytesInInteger = 4; + int digit; + int hB; + + if (bytesLen - (numberLength << 2) == 1) { + bytes[0] = (byte) ((sign < 0) ? -1 : 0); + highBytes = 4; + firstByteNumber++; + } else { + hB = bytesLen & 3; + highBytes = (hB == 0) ? 4 : hB; + } + + digitIndex = iThis; + bytesLen -= iThis << 2; + + if (sign < 0) { + digit = -temp.digits[digitIndex]; + digitIndex++; + if (digitIndex == numberLength) { + bytesInInteger = highBytes; + } + for (int i = 0; i < bytesInInteger; i++, digit >>= 8) { + bytes[--bytesLen] = (byte) digit; + } + while (bytesLen > firstByteNumber) { + digit = ~temp.digits[digitIndex]; + digitIndex++; + if (digitIndex == numberLength) { + bytesInInteger = highBytes; + } + for (int i = 0; i < bytesInInteger; i++, digit >>= 8) { + bytes[--bytesLen] = (byte) digit; + } + } + } else { + while (bytesLen > firstByteNumber) { + digit = temp.digits[digitIndex]; + digitIndex++; + if (digitIndex == numberLength) { + bytesInInteger = highBytes; + } + for (int i = 0; i < bytesInInteger; i++, digit >>= 8) { + bytes[--bytesLen] = (byte) digit; + } + } + } + return bytes; + } + + /** + * Returns a string representation of this {@code BigInteger} in decimal form. + * + * @return a string representation of {@code this} in decimal form. + */ + @Override + public String toString() { + return Conversion.toDecimalScaledString(this, 0); + } + + /** + * Returns a string containing a string representation of this {@code + * BigInteger} with base radix. If {@code radix} is less than + * {@link Character#MIN_RADIX} or greater than {@link Character#MAX_RADIX} + * then a decimal representation is returned. The characters of the string + * representation are generated with method {@link Character#forDigit}. + * + * @param radix base to be used for the string representation. + * @return a string representation of this with radix 10. + */ + public String toString(int radix) { + return Conversion.bigInteger2String(this, radix); + } + + /** + * Returns a new {@code BigInteger} whose value is {@code this ^ val}. + *

+ * Implementation Note: Usage of this method is not recommended as the + * current implementation is not efficient. + * + * @param val value to be xor'ed with {@code this} + * @return {@code this ^ val} + * @throws NullPointerException if {@code val == null} + */ + public BigInteger xor(BigInteger val) { + return Logical.xor(this, val); + } + + /* + * Returns a copy of the current instance to achieve immutability + */ + BigInteger copy() { + int[] copyDigits = new int[numberLength]; + System.arraycopy(digits, 0, copyDigits, 0, numberLength); + return new BigInteger(sign, numberLength, copyDigits); + } + + /* Private Methods */ + + /** + * Decreases {@code numberLength} if there are zero high elements. + */ + final void cutOffLeadingZeroes() { + while ((numberLength > 0) && (digits[--numberLength] == 0)) { + // Empty + } + if (digits[numberLength++] == 0) { + sign = 0; + } + } + + boolean equalsArrays(final int[] b) { + int i; + for (i = numberLength - 1; (i >= 0) && (digits[i] == b[i]); i--) { + // Empty + } + return i < 0; + } + + int getFirstNonzeroDigit() { + if (firstNonzeroDigit == -2) { + int i; + if (this.sign == 0) { + i = -1; + } else { + for (i = 0; digits[i] == 0; i++) { + // Empty + } + } + firstNonzeroDigit = i; + } + return firstNonzeroDigit; + } + + /** + * Tests if {@code this.abs()} is equals to {@code ONE}. + */ + boolean isOne() { + return ((numberLength == 1) && (digits[0] == 1)); + } + + BigInteger shiftLeftOneBit() { + return (sign == 0) ? this : BitLevel.shiftLeftOneBit(this); + } + + void unCache() { + firstNonzeroDigit = -2; + } + + /** + * Puts a big-endian byte array into a little-endian applying two complement. + */ + private void putBytesNegativeToIntegers(byte[] byteValues, int offset, int length) { + int bytesLen = length; + int highBytes = bytesLen & 3; + numberLength = (bytesLen >> 2) + ((highBytes == 0) ? 0 : 1); + digits = new int[numberLength]; + int i = 0; + // Setting the sign + digits[numberLength - 1] = -1; + // Put bytes to the int array starting from the end of the byte array + while (bytesLen > highBytes) { + digits[i] = (byteValues[--bytesLen + offset] & 0xFF) + | (byteValues[--bytesLen + offset] & 0xFF) << 8 + | (byteValues[--bytesLen + offset] & 0xFF) << 16 + | (byteValues[--bytesLen + offset] & 0xFF) << 24; + if (digits[i] != 0) { + digits[i] = -digits[i]; + firstNonzeroDigit = i; + i++; + while (bytesLen > highBytes) { + digits[i] = (byteValues[--bytesLen + offset] & 0xFF) + | (byteValues[--bytesLen + offset] & 0xFF) << 8 + | (byteValues[--bytesLen + offset] & 0xFF) << 16 + | (byteValues[--bytesLen + offset] & 0xFF) << 24; + digits[i] = ~digits[i]; + i++; + } + break; + } + i++; + } + if (highBytes != 0) { + // Put the first bytes in the highest element of the int array + if (firstNonzeroDigit != -2) { + for (int j = offset; j < bytesLen + offset; j++) { + digits[i] = (digits[i] << 8) | (byteValues[j] & 0xFF); + } + digits[i] = ~digits[i]; + } else { + for (int j = offset; j < bytesLen + offset; j++) { + digits[i] = (digits[i] << 8) | (byteValues[j] & 0xFF); + } + digits[i] = -digits[i]; + } + } + } + + /** + * Puts a big-endian byte array into a little-endian int array. + */ + private void putBytesPositiveToIntegers(byte[] byteValues, int offset, int length) { + int bytesLen = length; + int highBytes = bytesLen & 3; + numberLength = (bytesLen >> 2) + ((highBytes == 0) ? 0 : 1); + digits = new int[numberLength]; + int i = 0; + // Put bytes to the int array starting from the end of the byte array + while (bytesLen > highBytes) { + digits[i++] = (byteValues[--bytesLen + offset] & 0xFF) + | (byteValues[--bytesLen + offset] & 0xFF) << 8 + | (byteValues[--bytesLen + offset] & 0xFF) << 16 + | (byteValues[--bytesLen + offset] & 0xFF) << 24; + } + // Put the first bytes in the highest element of the int array + for (int j = offset; j < bytesLen + offset; j++) { + digits[i] = (digits[i] << 8) | (byteValues[j] & 0xFF); + } + } +} diff --git a/web/client-api/src/main/resources/io/grpc/Grpc.gwt.xml b/web/client-api/src/main/resources/io/grpc/Grpc.gwt.xml new file mode 100644 index 00000000000..187ac264524 --- /dev/null +++ b/web/client-api/src/main/resources/io/grpc/Grpc.gwt.xml @@ -0,0 +1,3 @@ + + + diff --git a/web/client-api/src/main/resources/org/apache/arrow/flatbuf/FlightFlatbufFormat.gwt.xml b/web/client-api/src/main/resources/org/apache/arrow/flatbuf/FlightFlatbufFormat.gwt.xml new file mode 100644 index 00000000000..9fd0a379083 --- /dev/null +++ b/web/client-api/src/main/resources/org/apache/arrow/flatbuf/FlightFlatbufFormat.gwt.xml @@ -0,0 +1,4 @@ + + + + diff --git a/web/client-api/src/main/resources/org/immutables/value/Immutables.gwt.xml b/web/client-api/src/main/resources/org/immutables/value/Immutables.gwt.xml new file mode 100644 index 00000000000..c78151ef021 --- /dev/null +++ b/web/client-api/src/main/resources/org/immutables/value/Immutables.gwt.xml @@ -0,0 +1,3 @@ + + + diff --git a/web/client-api/src/test/java/io/deephaven/web/client/api/HierarchicalTableTestGwt.java b/web/client-api/src/test/java/io/deephaven/web/client/api/HierarchicalTableTestGwt.java index 3e244ba9cd8..191b8d73f72 100644 --- a/web/client-api/src/test/java/io/deephaven/web/client/api/HierarchicalTableTestGwt.java +++ b/web/client-api/src/test/java/io/deephaven/web/client/api/HierarchicalTableTestGwt.java @@ -4,20 +4,27 @@ package io.deephaven.web.client.api; import elemental2.dom.CustomEvent; +import elemental2.promise.Promise; import io.deephaven.web.client.api.tree.JsTreeTable; +import io.deephaven.web.client.api.tree.TreeViewportData; public class HierarchicalTableTestGwt extends AbstractAsyncGwtTestCase { + private static final Format red = new Format(0x1ff000001e0e0e0L, 0, null, null); + private static final Format green = new Format(0x100800001e0e0e0L, 0, null, null); + @Override public String getModuleName() { return "io.deephaven.web.DeephavenIntegrationTest"; } private final TableSourceBuilder tables = new TableSourceBuilder() - .script("from deephaven import empty_table, time_table") + .script("from deephaven import empty_table, time_table, agg") .script("static_tree", - "empty_table(1000).update(['ID=i', 'Parent=i == 0 ? null : (int)(i/10)']).tree('ID', 'Parent')") + "empty_table(1000).update(['ID=i', 'Parent=i == 0 ? null : (int)(i/10)']).format_columns(['ID=ID>0 ? GREEN : RED']).tree('ID', 'Parent')") .script("ticking_tree", - "time_table('PT0.1s').update(['ID=i', 'Parent=i == 0 ? null : (int)(i/10)']).tree('ID', 'Parent')"); + "time_table('PT0.1s').update(['ID=i', 'Parent=i == 0 ? null : (int)(i/10)']).format_columns(['ID=ID>0 ? GREEN : RED']).tree('ID', 'Parent')") + .script("ticking_rollup", + "time_table('PT0.1s').update(['Y=Math.sin(i/3)', 'X=i%3']).format_columns(['Y=Y>0 ? GREEN : RED']).rollup(aggs=[agg.first('Y')],by=['X'],include_constituents=True)"); public void testStaticTreeTable() { connect(tables) @@ -33,19 +40,32 @@ public void testStaticTreeTable() { assertEquals("Parent", treeTable.getColumns().getAt(1).getName()); treeTable.setViewport(0, 99, treeTable.getColumns(), null); - return treeTable.getViewportData().then(data -> { - assertEquals(1d, data.getTreeSize()); + return treeTable.getViewportData() + .then(data -> Promise.resolve((TreeViewportData) data)) + .then(data -> { + assertEquals(1d, data.getTreeSize()); + + treeTable.expand(JsTreeTable.RowReferenceUnion.of(0), null); + return treeTable.nextEvent( + JsTreeTable.EVENT_UPDATED, 2001d); + }).then(event -> { + assertEquals(10d, event.detail.getTreeSize()); + assertEquals(10, event.detail.getRows().length); + - treeTable.expand(JsTreeTable.RowReferenceUnion.of(0), null); - return treeTable.nextEvent(JsTreeTable.EVENT_UPDATED, 2001d); - }).then(event -> { - assertEquals(10d, event.detail.getTreeSize()); + // move the viewport and try again + treeTable.setViewport(5, 50, treeTable.getColumns(), null); + return treeTable.nextEvent( + JsTreeTable.EVENT_UPDATED, 2002d); + }).then(event -> { + assertEquals(10d, event.detail.getTreeSize()); + assertEquals(5, event.detail.getRows().length); - treeTable.close(); + treeTable.close(); - assertTrue(treeTable.isClosed()); - return null; - }); + assertTrue(treeTable.isClosed()); + return null; + }); }) .then(this::finish).catch_(this::report); } @@ -68,17 +88,73 @@ public void testRefreshingTreeTable() { treeTable.setViewport(0, 99, treeTable.getColumns(), null); // Wait for the table to tick such that the first row has children + // Read values from the one returned row return waitForEventWhere(treeTable, JsTreeTable.EVENT_UPDATED, - (CustomEvent d) -> d.detail.getTreeSize() == 1 - && d.detail.getRows().getAt(0).hasChildren(), - 10001).then(data -> { + (CustomEvent d) -> d.detail + .getTreeSize() == 1 + && d.detail.getRows().getAtAsAny(0).cast() + .hasChildren(), + 10001) + .then(JsTreeTable::getViewportData) + .then(data -> Promise.resolve((TreeViewportData) data)) + .then(data -> { + assertEquals(1.0, data.getTreeSize()); + TreeViewportData.TreeRow row1 = (TreeViewportData.TreeRow) data.getRows().getAt(0); + Column timestampCol = treeTable.findColumn("Timestamp"); + assertEquals(Format.EMPTY, data.getFormat(0, timestampCol)); + assertEquals(Format.EMPTY, row1.getFormat(timestampCol)); + assertEquals(Format.EMPTY, timestampCol.getFormat(row1)); + + Column idCol = treeTable.findColumn("ID"); + assertEquals(0, data.getData(0, idCol).asInt()); + assertEquals(0, row1.get(idCol).asInt()); + assertEquals(0, idCol.get(row1).asInt()); + + assertEquals(red, data.getFormat(0, idCol)); + assertEquals(red, row1.getFormat(idCol)); + assertEquals(red, idCol.getFormat(row1)); + + assertNotNull(data.getData(0, timestampCol)); + assertNotNull(row1.get(timestampCol)); + assertNotNull(timestampCol.get(row1)); + treeTable.expand(JsTreeTable.RowReferenceUnion.of(0), null); // Wait for the expand to occur and table to show all 10 rows return waitForEventWhere(treeTable, JsTreeTable.EVENT_UPDATED, - (CustomEvent d) -> d.detail.getTreeSize() == 10, + (CustomEvent d) -> d.detail.getTreeSize() == 10, 14004); + }) + .then(JsTreeTable::getViewportData) + .then(data -> Promise.resolve((TreeViewportData) data)) + .then(data -> { + TreeViewportData.TreeRow row2 = (TreeViewportData.TreeRow) data.getRows().getAt(1); + + Column timestampCol = treeTable.findColumn("Timestamp"); + assertEquals(Format.EMPTY, data.getFormat(1, timestampCol)); + assertEquals(Format.EMPTY, row2.getFormat(timestampCol)); + assertEquals(Format.EMPTY, timestampCol.getFormat(row2)); + + Column idCol = treeTable.findColumn("ID"); + assertEquals(1, data.getData(1, idCol).asInt()); + assertEquals(1, row2.get(idCol).asInt()); + assertEquals(1, idCol.get(row2).asInt()); + + assertEquals(green, data.getFormat(1, idCol)); + assertEquals(green, row2.getFormat(idCol)); + assertEquals(green, idCol.getFormat(row2)); + + // Move the viewport and make sure we get the correct data + treeTable.setViewport(5, 49, treeTable.getColumns(), null); + return treeTable.nextEvent( + JsTreeTable.EVENT_UPDATED, 2002d); }).then(event -> { + assertEquals(10d, event.detail.getTreeSize()); + assertEquals(5, event.detail.getRows().length); + + return Promise.resolve(treeTable); + }) + .then(event -> { treeTable.close(); assertTrue(treeTable.isClosed()); return null; @@ -86,4 +162,110 @@ public void testRefreshingTreeTable() { }) .then(this::finish).catch_(this::report); } + + public void testTickingRollup() { + connect(tables) + .then(treeTable("ticking_rollup")) + .then(rollup -> { + // Very large timeout, 3.5s is enough that we see failures on this regularly + delayTestFinish(20_001); + assertTrue(rollup.isRefreshing()); + assertFalse(rollup.isClosed()); + assertTrue(rollup.isIncludeConstituents()); + + assertEquals(2, rollup.getColumns().length); + assertEquals("X", rollup.getColumns().getAt(0).getName()); + assertEquals("Y", rollup.getColumns().getAt(1).getName()); + + rollup.setViewport(0, 99, rollup.getColumns(), null); + + Column xCol = rollup.findColumn("X"); + Column yCol = rollup.findColumn("Y"); + + // Wait for the table to tick such that we have at least 4 rows (root, three children) + return waitForEventWhere(rollup, JsTreeTable.EVENT_UPDATED, + (CustomEvent d) -> d.detail.getTreeSize() == 4, + 10002) + .then(JsTreeTable::getViewportData) + .then(data -> Promise.resolve((TreeViewportData) data)) + .then(data -> { + TreeViewportData.TreeRow row1 = (TreeViewportData.TreeRow) data.getRows().getAt(0); + + assertEquals(Format.EMPTY, data.getFormat(0, xCol)); + assertEquals(Format.EMPTY, row1.getFormat(xCol)); + assertEquals(Format.EMPTY, xCol.getFormat(row1)); + + assertNull(data.getData(0, xCol)); + assertNull(row1.get(xCol)); + assertNull(xCol.get(row1)); + + assertEquals(Format.EMPTY, data.getFormat(0, yCol)); + assertEquals(Format.EMPTY, row1.getFormat(yCol)); + assertEquals(Format.EMPTY, yCol.getFormat(row1)); + + assertEquals(0d, data.getData(0, yCol).asDouble()); + assertEquals(0d, row1.get(yCol).asDouble()); + assertEquals(0d, yCol.get(row1).asDouble()); + + TreeViewportData.TreeRow row2 = (TreeViewportData.TreeRow) data.getRows().getAt(1); + assertEquals(Format.EMPTY, data.getFormat(1, xCol)); + assertEquals(Format.EMPTY, row2.getFormat(xCol)); + assertEquals(Format.EMPTY, xCol.getFormat(row2)); + + assertEquals(0d, data.getData(1, xCol).asDouble()); + assertEquals(0d, row2.get(xCol).asDouble()); + assertEquals(0d, xCol.get(row2).asDouble()); + + assertEquals(Format.EMPTY, data.getFormat(1, yCol)); + assertEquals(Format.EMPTY, row2.getFormat(yCol)); + assertEquals(Format.EMPTY, yCol.getFormat(row2)); + + assertEquals(0d, data.getData(1, yCol).asDouble()); + assertEquals(0d, row2.get(yCol).asDouble()); + assertEquals(0d, yCol.get(row2).asDouble()); + + // Expand row 2 + rollup.expand(JsTreeTable.RowReferenceUnion.of(1), null); + + // Wait for the expand to occur and table to show all 10 rows + return waitForEventWhere(rollup, JsTreeTable.EVENT_UPDATED, + (CustomEvent d) -> d.detail.getTreeSize() > 4, + 14008); + }) + .then(JsTreeTable::getViewportData) + .then(data -> Promise.resolve((TreeViewportData) data)) + .then(data -> { + TreeViewportData.TreeRow row3 = (TreeViewportData.TreeRow) data.getRows().getAt(2); + + assertEquals(Format.EMPTY, data.getFormat(2, xCol)); + assertEquals(Format.EMPTY, row3.getFormat(xCol)); + assertEquals(Format.EMPTY, xCol.getFormat(row3)); + + assertEquals(0d, data.getData(2, yCol).asDouble()); + assertEquals(0d, row3.get(yCol).asDouble()); + assertEquals(0d, yCol.get(row3).asDouble()); + + assertEquals(red, data.getFormat(2, yCol)); + assertEquals(red, row3.getFormat(yCol)); + assertEquals(red, yCol.getFormat(row3)); + + assertEquals(0d, data.getData(2, yCol).asDouble()); + assertEquals(0d, row3.get(yCol).asDouble()); + assertEquals(0d, yCol.get(row3).asDouble()); + + // Collapse row 2, wait until back to 4 rows + rollup.collapse(JsTreeTable.RowReferenceUnion.of(1)); + return waitForEventWhere(rollup, JsTreeTable.EVENT_UPDATED, + (CustomEvent d) -> d.detail.getTreeSize() == 4, + 14009); + }) + .then(event -> { + rollup.close(); + assertTrue(rollup.isClosed()); + return null; + }); + }) + .then(this::finish).catch_(this::report); + } + } diff --git a/web/client-api/src/test/java/io/deephaven/web/client/api/NullValueTestGwt.java b/web/client-api/src/test/java/io/deephaven/web/client/api/NullValueTestGwt.java index 4699a9f3de5..03ec2a2382b 100644 --- a/web/client-api/src/test/java/io/deephaven/web/client/api/NullValueTestGwt.java +++ b/web/client-api/src/test/java/io/deephaven/web/client/api/NullValueTestGwt.java @@ -7,7 +7,6 @@ import com.google.gwt.junit.Platform; import elemental2.core.JsArray; import elemental2.promise.Promise; -import io.deephaven.web.client.api.subscription.ViewportRow; import jsinterop.base.Js; import java.math.BigDecimal; @@ -86,8 +85,8 @@ public void testTableWithSomeNulls() { .then(table -> { table.setViewport(0, 1, null); return assertUpdateReceived(table, viewport -> { - JsArray rows = viewport.getRows(); - ViewportRow nullRow = rows.getAt(0); + JsArray rows = viewport.getRows(); + TableData.Row nullRow = rows.getAt(0); JsArray columns = table.getColumns(); for (int i = 0; i < columns.length; i++) { @@ -95,7 +94,7 @@ public void testTableWithSomeNulls() { assertNull(nullRow.get(columns.getAt(i))); } - ViewportRow valueRow = rows.getAt(1); + TableData.Row valueRow = rows.getAt(1); assertEquals(1, valueRow.get(table.findColumn("MyInt")).asInt()); assertEquals((long) 1, valueRow.get(table.findColumn("MyLong")).cast().getWrapped()); @@ -157,12 +156,12 @@ public void testTableWithAllNulls() { .then(table -> { table.setViewport(0, 1, null); return assertUpdateReceived(table, viewport -> { - JsArray rows = viewport.getRows(); + JsArray rows = viewport.getRows(); JsArray columns = table.getColumns(); for (int i = 0; i < columns.length; i++) { for (int j = 0; j < rows.length; j++) { - ViewportRow row = rows.getAt(j); + TableData.Row row = rows.getAt(j); assertFalse(Js.isTripleEqual(Js.undefined(), row.get(columns.getAt(i)))); assertNull(columns.getAt(i).getName(), row.get(columns.getAt(i))); } diff --git a/web/client-api/src/test/java/io/deephaven/web/client/api/PartitionedTableTestGwt.java b/web/client-api/src/test/java/io/deephaven/web/client/api/PartitionedTableTestGwt.java index 0d3cf8edbb3..d0b9c4fcc40 100644 --- a/web/client-api/src/test/java/io/deephaven/web/client/api/PartitionedTableTestGwt.java +++ b/web/client-api/src/test/java/io/deephaven/web/client/api/PartitionedTableTestGwt.java @@ -5,8 +5,6 @@ import elemental2.core.JsArray; import elemental2.dom.CustomEvent; -import io.deephaven.web.client.api.subscription.ViewportData; -import io.deephaven.web.client.api.tree.JsTreeTable; public class PartitionedTableTestGwt extends AbstractAsyncGwtTestCase { @Override diff --git a/web/client-api/src/test/java/io/deephaven/web/client/api/TotalsTableTestGwt.java b/web/client-api/src/test/java/io/deephaven/web/client/api/TotalsTableTestGwt.java index 6a6c2d3e706..857cad855c5 100644 --- a/web/client-api/src/test/java/io/deephaven/web/client/api/TotalsTableTestGwt.java +++ b/web/client-api/src/test/java/io/deephaven/web/client/api/TotalsTableTestGwt.java @@ -72,7 +72,7 @@ public void testQueryDefinedConfigs() { .then(totals -> { assertEquals(3, totals.getColumns().length); assertEquals(1, totals.getSize(), DELTA); - totals.setViewport(0, 100, null, null); + totals.setViewport(0, 100, null, null, null); return waitForEvent(totals, JsTable.EVENT_UPDATED, checkTotals(totals, 5, 6., 0, "a1"), 2508); @@ -81,7 +81,7 @@ public void testQueryDefinedConfigs() { .then(totals -> { assertEquals(3, totals.getColumns().length); assertEquals(1, totals.getSize(), DELTA); - totals.setViewport(0, 100, null, null); + totals.setViewport(0, 100, null, null, null); return waitForEvent(totals, JsTable.EVENT_UPDATED, checkTotals(totals, 5, 6.0, 0., "a2"), 2509); @@ -112,7 +112,7 @@ public void ignore_testTotalsOnFilteredTable() { totalTables[0] = totals; assertEquals(3, totals.getColumns().length); assertEquals(1, totals.getSize(), DELTA); - totals.setViewport(0, 100, null, null); + totals.setViewport(0, 100, null, null, null); // confirm the normal totals match the filtered data return waitForEvent(totals, JsTable.EVENT_UPDATED, @@ -123,7 +123,7 @@ public void ignore_testTotalsOnFilteredTable() { totalTables[1] = totals; assertEquals(3, totals.getColumns().length); assertEquals(1, totals.getSize(), DELTA); - totals.setViewport(0, 100, null, null); + totals.setViewport(0, 100, null, null, null); // confirm the grand totals are unchanged return waitForEvent(totals, JsTable.EVENT_UPDATED, @@ -238,7 +238,7 @@ public void ignore_testFilteringTotalsTable() { totalTables[0] = totals; assertEquals(4, totals.getColumns().length); assertEquals(2, totals.getSize(), DELTA); - totals.setViewport(0, 100, null, null); + totals.setViewport(0, 100, null, null, null); // confirm the normal totals match the filtered data return waitForEvent(totals, JsTable.EVENT_UPDATED, checkTotals(totals, "a1", @@ -250,7 +250,7 @@ public void ignore_testFilteringTotalsTable() { totalTables[1] = totals; assertEquals(4, totals.getColumns().length); assertEquals(2, totals.getSize(), DELTA); - totals.setViewport(0, 100, null, null); + totals.setViewport(0, 100, null, null, null); // confirm the grand totals include the missing row... return waitForEvent(totals, JsTable.EVENT_UPDATED, checkTotals(totals, "a2", @@ -269,8 +269,8 @@ public void ignore_testFilteringTotalsTable() { totalTables[1].applyFilter(new FilterCondition[] { totalTables[1].findColumn("J__Avg").filter().eq(FilterValue.ofNumber(5.0)) }); - totalTables[0].setViewport(0, 100, null, null); - totalTables[1].setViewport(0, 100, null, null); + totalTables[0].setViewport(0, 100, null, null, null); + totalTables[1].setViewport(0, 100, null, null, null); return promiseAllThen(table, totalPromises[0] = waitForEvent(totalTables[0], JsTable.EVENT_UPDATED, @@ -327,7 +327,7 @@ public void testGroupedTotals() { (JsTotalsTable totals) -> { assertEquals(4, totals.getColumns().length); assertEquals(2, totals.getSize(), DELTA); - totals.setViewport(0, 100, null, null); + totals.setViewport(0, 100, null, null, null); // confirm the grand totals are unchanged return waitForEvent(totals, JsTable.EVENT_UPDATED, update -> { diff --git a/web/client-api/src/test/java/io/deephaven/web/client/api/subscription/ViewportTestGwt.java b/web/client-api/src/test/java/io/deephaven/web/client/api/subscription/ViewportTestGwt.java index 19e1a1ddf4f..2d9648c7b08 100644 --- a/web/client-api/src/test/java/io/deephaven/web/client/api/subscription/ViewportTestGwt.java +++ b/web/client-api/src/test/java/io/deephaven/web/client/api/subscription/ViewportTestGwt.java @@ -12,6 +12,7 @@ import io.deephaven.web.client.api.Column; import io.deephaven.web.client.api.HasEventHandling; import io.deephaven.web.client.api.JsTable; +import io.deephaven.web.client.api.TableData; import io.deephaven.web.client.api.filter.FilterCondition; import io.deephaven.web.client.api.filter.FilterValue; import io.deephaven.web.shared.fu.RemoverFn; @@ -40,7 +41,8 @@ public class ViewportTestGwt extends AbstractAsyncGwtTestCase { .script("from datetime import datetime, timedelta") .script("growingForward", "time_table(period=\"PT00:00:01\", start_time=datetime.now() - timedelta(minutes=1)).update([\"I=i\", \"J=i*i\", \"K=0\"])") - .script("growingBackward", "growingForward.sort_descending(\"Timestamp\")") + .script("growingBackward", + "growingForward.sort_descending(\"Timestamp\").format_columns(['I=I>2 ? GREEN : RED'])") .script("blinkOne", "time_table(\"PT00:00:01\").update([\"I=i\", \"J=1\"]).last_by(by=\"J\").where(\"I%2 != 0\")"); @@ -59,28 +61,28 @@ public void testViewportOnStaticTable() { // table has 100 rows, go through each page of 25, make sure the offset and length is sane table.setViewport(0, 24, null); return assertUpdateReceived(table, viewport -> { - assertEquals(0, (long) viewport.getOffset()); + assertEquals(0d, viewport.getOffset()); assertEquals(25, viewport.getRows().length); }, 2100); }) .then(table -> { table.setViewport(25, 49, null); return assertUpdateReceived(table, viewport -> { - assertEquals(25, (long) viewport.getOffset()); + assertEquals(25d, viewport.getOffset()); assertEquals(25, viewport.getRows().length); }, 2101); }) .then(table -> { table.setViewport(50, 74, null); return assertUpdateReceived(table, viewport -> { - assertEquals(50, (long) viewport.getOffset()); + assertEquals(50d, viewport.getOffset()); assertEquals(25, viewport.getRows().length); }, 2102); }) .then(table -> { table.setViewport(75, 99, null); return assertUpdateReceived(table, viewport -> { - assertEquals(75, (long) viewport.getOffset()); + assertEquals(75d, viewport.getOffset()); assertEquals(25, viewport.getRows().length); }, 2103); }) @@ -179,10 +181,16 @@ public void testViewportSubsetOfColumns() { assertEquals(0, indexOf(viewport.getColumns(), table.findColumn("I"))); assertEquals(1, viewport.getRows().length); - assertNotNull(viewport.getRows().getAt(0).get(table.findColumn("I"))); - assertThrowsException(() -> viewport.getRows().getAt(0).get(table.findColumn("J"))); - assertThrowsException(() -> viewport.getRows().getAt(0).get(table.findColumn("K"))); - + TableData.Row row1 = viewport.getRows().getAt(0); + assertNotNull(viewport.getData(0, table.findColumn("I"))); + assertNotNull(row1.get(table.findColumn("I"))); + assertNotNull(table.findColumn("I").get(row1)); + assertNotNull(row1.getFormat(table.findColumn("I"))); + assertNotNull(table.findColumn("I").getFormat(row1)); + assertNotNull(viewport.getFormat(0, table.findColumn("I"))); + + assertThrowsException(() -> row1.get(table.findColumn("J"))); + assertThrowsException(() -> row1.get(table.findColumn("K"))); }, 2501); }) .then(table -> { @@ -361,7 +369,7 @@ public void testRapidChangingViewport() { table.setViewport(0, 10, null); table.setViewport(5, 14, null); return assertUpdateReceived(table, viewport -> { - assertEquals(5, (int) viewport.getOffset()); + assertEquals(5d, viewport.getOffset()); assertEquals(10, (int) viewport.getRows().length); }, 1008); }) @@ -373,8 +381,8 @@ public void testRapidChangingViewport() { .then(table -> { table.setViewport(6, 14, null); return assertUpdateReceived(table, viewport -> { - assertEquals(6, (int) viewport.getOffset()); - assertEquals(9, (int) viewport.getRows().length); + assertEquals(6d, viewport.getOffset()); + assertEquals(9, viewport.getRows().length); }, 1009); }) .then(table -> { @@ -387,12 +395,9 @@ public void testRapidChangingViewport() { table.setViewport(7, 17, null); return assertUpdateReceived(table, ignored -> { }, 1010) - .then(waitFor(JsTable.DEBOUNCE_TIME * 2)) .then(t -> { - // force the debounce to be processed - t.processSnapshot(); t.getViewportData().then(vp -> { - // assertEquals(7, (int) vp.getOffset()); + assertEquals(7d, vp.getOffset()); assertEquals(11, (int) vp.getRows().length); return Promise.resolve(vp); }); diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/Flatbuf.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/Flatbuf.java deleted file mode 100644 index d32b397e89b..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/Flatbuf.java +++ /dev/null @@ -1,300 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow; - -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf.DictionaryBatch; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf.RecordBatch; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Schema; -import jsinterop.annotations.JsFunction; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Message_generated.org.apache.arrow.flatbuf", - namespace = JsPackage.GLOBAL) -public class Flatbuf { - @JsFunction - public interface UnionListToMessageHeaderAccessorFn { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface P1UnionType { - @JsOverlay - static Flatbuf.UnionListToMessageHeaderAccessorFn.P1UnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default DictionaryBatch asDictionaryBatch() { - return Js.cast(this); - } - - @JsOverlay - default RecordBatch asRecordBatch() { - return Js.cast(this); - } - - @JsOverlay - default Schema asSchema() { - return Js.cast(this); - } - - @JsOverlay - default boolean isDictionaryBatch() { - return (Object) this instanceof DictionaryBatch; - } - - @JsOverlay - default boolean isRecordBatch() { - return (Object) this instanceof RecordBatch; - } - - @JsOverlay - default boolean isSchema() { - return (Object) this instanceof Schema; - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface UnionType { - @JsOverlay - static Flatbuf.UnionListToMessageHeaderAccessorFn.UnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default DictionaryBatch asDictionaryBatch() { - return Js.cast(this); - } - - @JsOverlay - default RecordBatch asRecordBatch() { - return Js.cast(this); - } - - @JsOverlay - default Schema asSchema() { - return Js.cast(this); - } - - @JsOverlay - default boolean isDictionaryBatch() { - return (Object) this instanceof DictionaryBatch; - } - - @JsOverlay - default boolean isRecordBatch() { - return (Object) this instanceof RecordBatch; - } - - @JsOverlay - default boolean isSchema() { - return (Object) this instanceof Schema; - } - } - - @JsOverlay - default Flatbuf.UnionListToMessageHeaderAccessorFn.UnionType onInvoke( - double p0, DictionaryBatch p1) { - return onInvoke( - p0, Js.uncheckedCast(p1)); - } - - Flatbuf.UnionListToMessageHeaderAccessorFn.UnionType onInvoke( - double p0, Flatbuf.UnionListToMessageHeaderAccessorFn.P1UnionType p1); - - @JsOverlay - default Flatbuf.UnionListToMessageHeaderAccessorFn.UnionType onInvoke( - double p0, RecordBatch p1) { - return onInvoke( - p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToMessageHeaderAccessorFn.UnionType onInvoke(double p0, Schema p1) { - return onInvoke( - p0, Js.uncheckedCast(p1)); - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface UnionListToMessageHeaderUnionType { - @JsOverlay - static Flatbuf.UnionListToMessageHeaderUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default DictionaryBatch asDictionaryBatch() { - return Js.cast(this); - } - - @JsOverlay - default RecordBatch asRecordBatch() { - return Js.cast(this); - } - - @JsOverlay - default Schema asSchema() { - return Js.cast(this); - } - - @JsOverlay - default boolean isDictionaryBatch() { - return (Object) this instanceof DictionaryBatch; - } - - @JsOverlay - default boolean isRecordBatch() { - return (Object) this instanceof RecordBatch; - } - - @JsOverlay - default boolean isSchema() { - return (Object) this instanceof Schema; - } - } - - @JsFunction - public interface UnionToMessageHeaderAccessorFn { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface P0UnionType { - @JsOverlay - static Flatbuf.UnionToMessageHeaderAccessorFn.P0UnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default DictionaryBatch asDictionaryBatch() { - return Js.cast(this); - } - - @JsOverlay - default RecordBatch asRecordBatch() { - return Js.cast(this); - } - - @JsOverlay - default Schema asSchema() { - return Js.cast(this); - } - - @JsOverlay - default boolean isDictionaryBatch() { - return (Object) this instanceof DictionaryBatch; - } - - @JsOverlay - default boolean isRecordBatch() { - return (Object) this instanceof RecordBatch; - } - - @JsOverlay - default boolean isSchema() { - return (Object) this instanceof Schema; - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface UnionType { - @JsOverlay - static Flatbuf.UnionToMessageHeaderAccessorFn.UnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default DictionaryBatch asDictionaryBatch() { - return Js.cast(this); - } - - @JsOverlay - default RecordBatch asRecordBatch() { - return Js.cast(this); - } - - @JsOverlay - default Schema asSchema() { - return Js.cast(this); - } - - @JsOverlay - default boolean isDictionaryBatch() { - return (Object) this instanceof DictionaryBatch; - } - - @JsOverlay - default boolean isRecordBatch() { - return (Object) this instanceof RecordBatch; - } - - @JsOverlay - default boolean isSchema() { - return (Object) this instanceof Schema; - } - } - - @JsOverlay - default Flatbuf.UnionToMessageHeaderAccessorFn.UnionType onInvoke(DictionaryBatch p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - Flatbuf.UnionToMessageHeaderAccessorFn.UnionType onInvoke( - Flatbuf.UnionToMessageHeaderAccessorFn.P0UnionType p0); - - @JsOverlay - default Flatbuf.UnionToMessageHeaderAccessorFn.UnionType onInvoke(RecordBatch p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToMessageHeaderAccessorFn.UnionType onInvoke(Schema p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface UnionToMessageHeaderUnionType { - @JsOverlay - static Flatbuf.UnionToMessageHeaderUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default DictionaryBatch asDictionaryBatch() { - return Js.cast(this); - } - - @JsOverlay - default RecordBatch asRecordBatch() { - return Js.cast(this); - } - - @JsOverlay - default Schema asSchema() { - return Js.cast(this); - } - - @JsOverlay - default boolean isDictionaryBatch() { - return (Object) this instanceof DictionaryBatch; - } - - @JsOverlay - default boolean isRecordBatch() { - return (Object) this instanceof RecordBatch; - } - - @JsOverlay - default boolean isSchema() { - return (Object) this instanceof Schema; - } - } - - public static native Flatbuf.UnionListToMessageHeaderUnionType unionListToMessageHeader( - int type, Flatbuf.UnionListToMessageHeaderAccessorFn accessor, double index); - - public static native Flatbuf.UnionToMessageHeaderUnionType unionToMessageHeader( - int type, Flatbuf.UnionToMessageHeaderAccessorFn accessor); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/BodyCompression.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/BodyCompression.java deleted file mode 100644 index 77a91516d9c..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/BodyCompression.java +++ /dev/null @@ -1,44 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Message_generated.org.apache.arrow.flatbuf.BodyCompression", - namespace = JsPackage.GLOBAL) -public class BodyCompression { - public static native void addCodec(Builder builder, int codec); - - public static native void addMethod(Builder builder, int method); - - public static native double createBodyCompression( - Builder builder, int codec, int method); - - public static native double endBodyCompression(Builder builder); - - public static native BodyCompression getRootAsBodyCompression(ByteBuffer bb, BodyCompression obj); - - public static native BodyCompression getRootAsBodyCompression(ByteBuffer bb); - - public static native BodyCompression getSizePrefixedRootAsBodyCompression( - ByteBuffer bb, BodyCompression obj); - - public static native BodyCompression getSizePrefixedRootAsBodyCompression(ByteBuffer bb); - - public static native void startBodyCompression(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native BodyCompression __init(double i, ByteBuffer bb); - - public native int codec(); - - public native int method(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/BodyCompressionMethod.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/BodyCompressionMethod.java deleted file mode 100644 index 0fa3c6919e4..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/BodyCompressionMethod.java +++ /dev/null @@ -1,15 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf; - -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Message_generated.org.apache.arrow.flatbuf.BodyCompressionMethod", - namespace = JsPackage.GLOBAL) -public class BodyCompressionMethod { - public static int BUFFER; -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/CompressionType.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/CompressionType.java deleted file mode 100644 index a395a0317e3..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/CompressionType.java +++ /dev/null @@ -1,16 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf; - -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Message_generated.org.apache.arrow.flatbuf.CompressionType", - namespace = JsPackage.GLOBAL) -public class CompressionType { - public static int LZ4_FRAME, - ZSTD; -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/DictionaryBatch.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/DictionaryBatch.java deleted file mode 100644 index b063716b3c4..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/DictionaryBatch.java +++ /dev/null @@ -1,48 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Long; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Message_generated.org.apache.arrow.flatbuf.DictionaryBatch", - namespace = JsPackage.GLOBAL) -public class DictionaryBatch { - public static native void addData(Builder builder, double dataOffset); - - public static native void addId(Builder builder, Long id); - - public static native void addIsDelta(Builder builder, boolean isDelta); - - public static native double endDictionaryBatch(Builder builder); - - public static native DictionaryBatch getRootAsDictionaryBatch(ByteBuffer bb, DictionaryBatch obj); - - public static native DictionaryBatch getRootAsDictionaryBatch(ByteBuffer bb); - - public static native DictionaryBatch getSizePrefixedRootAsDictionaryBatch( - ByteBuffer bb, DictionaryBatch obj); - - public static native DictionaryBatch getSizePrefixedRootAsDictionaryBatch(ByteBuffer bb); - - public static native void startDictionaryBatch(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native DictionaryBatch __init(double i, ByteBuffer bb); - - public native RecordBatch data(); - - public native RecordBatch data(RecordBatch obj); - - public native Long id(); - - public native boolean isDelta(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/FieldNode.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/FieldNode.java deleted file mode 100644 index e3751057417..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/FieldNode.java +++ /dev/null @@ -1,29 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Long; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Message_generated.org.apache.arrow.flatbuf.FieldNode", - namespace = JsPackage.GLOBAL) -public class FieldNode { - public static native double createFieldNode(Builder builder, Long length, Long null_count); - - public static native double sizeOf(); - - public ByteBuffer bb; - public double bb_pos; - - public native FieldNode __init(double i, ByteBuffer bb); - - public native Long length(); - - public native Long nullCount(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/Message.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/Message.java deleted file mode 100644 index 691e7947369..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/Message.java +++ /dev/null @@ -1,82 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf; - -import elemental2.core.JsArray; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.KeyValue; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Long; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Message_generated.org.apache.arrow.flatbuf.Message", - namespace = JsPackage.GLOBAL) -public class Message { - public static native void addBodyLength(Builder builder, Long bodyLength); - - public static native void addCustomMetadata(Builder builder, double customMetadataOffset); - - public static native void addHeader(Builder builder, double headerOffset); - - public static native void addHeaderType(Builder builder, int headerType); - - public static native void addVersion(Builder builder, int version); - - public static native double createCustomMetadataVector(Builder builder, JsArray data); - - @JsOverlay - public static final double createCustomMetadataVector(Builder builder, double[] data) { - return createCustomMetadataVector(builder, Js.>uncheckedCast(data)); - } - - public static native double createMessage( - Builder builder, - int version, - int headerType, - double headerOffset, - Long bodyLength, - double customMetadataOffset); - - public static native double endMessage(Builder builder); - - public static native void finishMessageBuffer(Builder builder, double offset); - - public static native void finishSizePrefixedMessageBuffer(Builder builder, double offset); - - public static native Message getRootAsMessage(ByteBuffer bb, Message obj); - - public static native Message getRootAsMessage(ByteBuffer bb); - - public static native Message getSizePrefixedRootAsMessage(ByteBuffer bb, Message obj); - - public static native Message getSizePrefixedRootAsMessage(ByteBuffer bb); - - public static native void startCustomMetadataVector(Builder builder, double numElems); - - public static native void startMessage(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native Message __init(double i, ByteBuffer bb); - - public native Long bodyLength(); - - public native KeyValue customMetadata(double index, KeyValue obj); - - public native KeyValue customMetadata(double index); - - public native double customMetadataLength(); - - public native T header(T obj); - - public native int headerType(); - - public native int version(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/MessageHeader.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/MessageHeader.java deleted file mode 100644 index 48bae3c723d..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/MessageHeader.java +++ /dev/null @@ -1,18 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf; - -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Message_generated.org.apache.arrow.flatbuf.MessageHeader", - namespace = JsPackage.GLOBAL) -public class MessageHeader { - public static int DictionaryBatch, - NONE, - RecordBatch, - Schema; -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/RecordBatch.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/RecordBatch.java deleted file mode 100644 index 619fbee84cb..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/message_generated/org/apache/arrow/flatbuf/RecordBatch.java +++ /dev/null @@ -1,64 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.message_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Buffer; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Long; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Message_generated.org.apache.arrow.flatbuf.RecordBatch", - namespace = JsPackage.GLOBAL) -public class RecordBatch { - public static native void addBuffers(Builder builder, double buffersOffset); - - public static native void addCompression(Builder builder, double compressionOffset); - - public static native void addLength(Builder builder, Long length); - - public static native void addNodes(Builder builder, double nodesOffset); - - public static native double endRecordBatch(Builder builder); - - public static native RecordBatch getRootAsRecordBatch(ByteBuffer bb, RecordBatch obj); - - public static native RecordBatch getRootAsRecordBatch(ByteBuffer bb); - - public static native RecordBatch getSizePrefixedRootAsRecordBatch(ByteBuffer bb, RecordBatch obj); - - public static native RecordBatch getSizePrefixedRootAsRecordBatch(ByteBuffer bb); - - public static native void startBuffersVector(Builder builder, double numElems); - - public static native void startNodesVector(Builder builder, double numElems); - - public static native void startRecordBatch(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native RecordBatch __init(double i, ByteBuffer bb); - - public native Buffer buffers(double index, Buffer obj); - - public native Buffer buffers(double index); - - public native double buffersLength(); - - public native BodyCompression compression(); - - public native BodyCompression compression(BodyCompression obj); - - public native Long length(); - - public native FieldNode nodes(double index, FieldNode obj); - - public native FieldNode nodes(double index); - - public native double nodesLength(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/Flatbuf.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/Flatbuf.java deleted file mode 100644 index c74661f7d31..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/Flatbuf.java +++ /dev/null @@ -1,1572 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow; - -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Binary; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Bool; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Date; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Decimal; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Duration; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.FixedSizeBinary; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.FixedSizeList; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.FloatingPoint; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Int; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Interval; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.LargeBinary; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.LargeList; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.LargeUtf8; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.List; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Map; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Null; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Struct_; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Time; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Timestamp; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Union; -import io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf.Utf8; -import jsinterop.annotations.JsFunction; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf", - namespace = JsPackage.GLOBAL) -public class Flatbuf { - @JsFunction - public interface UnionListToTypeAccessorFn { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface P1UnionType { - @JsOverlay - static Flatbuf.UnionListToTypeAccessorFn.P1UnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Binary asBinary() { - return Js.cast(this); - } - - @JsOverlay - default Bool asBool() { - return Js.cast(this); - } - - @JsOverlay - default Date asDate() { - return Js.cast(this); - } - - @JsOverlay - default Decimal asDecimal() { - return Js.cast(this); - } - - @JsOverlay - default Duration asDuration() { - return Js.cast(this); - } - - @JsOverlay - default FixedSizeBinary asFixedSizeBinary() { - return Js.cast(this); - } - - @JsOverlay - default FixedSizeList asFixedSizeList() { - return Js.cast(this); - } - - @JsOverlay - default FloatingPoint asFloatingPoint() { - return Js.cast(this); - } - - @JsOverlay - default Int asInt() { - return Js.cast(this); - } - - @JsOverlay - default Interval asInterval() { - return Js.cast(this); - } - - @JsOverlay - default LargeBinary asLargeBinary() { - return Js.cast(this); - } - - @JsOverlay - default LargeList asLargeList() { - return Js.cast(this); - } - - @JsOverlay - default LargeUtf8 asLargeUtf8() { - return Js.cast(this); - } - - @JsOverlay - default List asList() { - return Js.cast(this); - } - - @JsOverlay - default Map asMap() { - return Js.cast(this); - } - - @JsOverlay - default Null asNull() { - return Js.cast(this); - } - - @JsOverlay - default Struct_ asStruct_() { - return Js.cast(this); - } - - @JsOverlay - default Time asTime() { - return Js.cast(this); - } - - @JsOverlay - default Timestamp asTimestamp() { - return Js.cast(this); - } - - @JsOverlay - default Union asUnion() { - return Js.cast(this); - } - - @JsOverlay - default Utf8 asUtf8() { - return Js.cast(this); - } - - @JsOverlay - default boolean isBinary() { - return (Object) this instanceof Binary; - } - - @JsOverlay - default boolean isBool() { - return (Object) this instanceof Bool; - } - - @JsOverlay - default boolean isDate() { - return (Object) this instanceof Date; - } - - @JsOverlay - default boolean isDecimal() { - return (Object) this instanceof Decimal; - } - - @JsOverlay - default boolean isDuration() { - return (Object) this instanceof Duration; - } - - @JsOverlay - default boolean isFixedSizeBinary() { - return (Object) this instanceof FixedSizeBinary; - } - - @JsOverlay - default boolean isFixedSizeList() { - return (Object) this instanceof FixedSizeList; - } - - @JsOverlay - default boolean isFloatingPoint() { - return (Object) this instanceof FloatingPoint; - } - - @JsOverlay - default boolean isInt() { - return (Object) this instanceof Int; - } - - @JsOverlay - default boolean isInterval() { - return (Object) this instanceof Interval; - } - - @JsOverlay - default boolean isLargeBinary() { - return (Object) this instanceof LargeBinary; - } - - @JsOverlay - default boolean isLargeList() { - return (Object) this instanceof LargeList; - } - - @JsOverlay - default boolean isLargeUtf8() { - return (Object) this instanceof LargeUtf8; - } - - @JsOverlay - default boolean isList() { - return (Object) this instanceof List; - } - - @JsOverlay - default boolean isMap() { - return (Object) this instanceof Map; - } - - @JsOverlay - default boolean isNull() { - return (Object) this instanceof Null; - } - - @JsOverlay - default boolean isStruct_() { - return (Object) this instanceof Struct_; - } - - @JsOverlay - default boolean isTime() { - return (Object) this instanceof Time; - } - - @JsOverlay - default boolean isTimestamp() { - return (Object) this instanceof Timestamp; - } - - @JsOverlay - default boolean isUnion() { - return (Object) this instanceof Union; - } - - @JsOverlay - default boolean isUtf8() { - return (Object) this instanceof Utf8; - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface UnionType { - @JsOverlay - static Flatbuf.UnionListToTypeAccessorFn.UnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Binary asBinary() { - return Js.cast(this); - } - - @JsOverlay - default Bool asBool() { - return Js.cast(this); - } - - @JsOverlay - default Date asDate() { - return Js.cast(this); - } - - @JsOverlay - default Decimal asDecimal() { - return Js.cast(this); - } - - @JsOverlay - default Duration asDuration() { - return Js.cast(this); - } - - @JsOverlay - default FixedSizeBinary asFixedSizeBinary() { - return Js.cast(this); - } - - @JsOverlay - default FixedSizeList asFixedSizeList() { - return Js.cast(this); - } - - @JsOverlay - default FloatingPoint asFloatingPoint() { - return Js.cast(this); - } - - @JsOverlay - default Int asInt() { - return Js.cast(this); - } - - @JsOverlay - default Interval asInterval() { - return Js.cast(this); - } - - @JsOverlay - default LargeBinary asLargeBinary() { - return Js.cast(this); - } - - @JsOverlay - default LargeList asLargeList() { - return Js.cast(this); - } - - @JsOverlay - default LargeUtf8 asLargeUtf8() { - return Js.cast(this); - } - - @JsOverlay - default List asList() { - return Js.cast(this); - } - - @JsOverlay - default Map asMap() { - return Js.cast(this); - } - - @JsOverlay - default Null asNull() { - return Js.cast(this); - } - - @JsOverlay - default Struct_ asStruct_() { - return Js.cast(this); - } - - @JsOverlay - default Time asTime() { - return Js.cast(this); - } - - @JsOverlay - default Timestamp asTimestamp() { - return Js.cast(this); - } - - @JsOverlay - default Union asUnion() { - return Js.cast(this); - } - - @JsOverlay - default Utf8 asUtf8() { - return Js.cast(this); - } - - @JsOverlay - default boolean isBinary() { - return (Object) this instanceof Binary; - } - - @JsOverlay - default boolean isBool() { - return (Object) this instanceof Bool; - } - - @JsOverlay - default boolean isDate() { - return (Object) this instanceof Date; - } - - @JsOverlay - default boolean isDecimal() { - return (Object) this instanceof Decimal; - } - - @JsOverlay - default boolean isDuration() { - return (Object) this instanceof Duration; - } - - @JsOverlay - default boolean isFixedSizeBinary() { - return (Object) this instanceof FixedSizeBinary; - } - - @JsOverlay - default boolean isFixedSizeList() { - return (Object) this instanceof FixedSizeList; - } - - @JsOverlay - default boolean isFloatingPoint() { - return (Object) this instanceof FloatingPoint; - } - - @JsOverlay - default boolean isInt() { - return (Object) this instanceof Int; - } - - @JsOverlay - default boolean isInterval() { - return (Object) this instanceof Interval; - } - - @JsOverlay - default boolean isLargeBinary() { - return (Object) this instanceof LargeBinary; - } - - @JsOverlay - default boolean isLargeList() { - return (Object) this instanceof LargeList; - } - - @JsOverlay - default boolean isLargeUtf8() { - return (Object) this instanceof LargeUtf8; - } - - @JsOverlay - default boolean isList() { - return (Object) this instanceof List; - } - - @JsOverlay - default boolean isMap() { - return (Object) this instanceof Map; - } - - @JsOverlay - default boolean isNull() { - return (Object) this instanceof Null; - } - - @JsOverlay - default boolean isStruct_() { - return (Object) this instanceof Struct_; - } - - @JsOverlay - default boolean isTime() { - return (Object) this instanceof Time; - } - - @JsOverlay - default boolean isTimestamp() { - return (Object) this instanceof Timestamp; - } - - @JsOverlay - default boolean isUnion() { - return (Object) this instanceof Union; - } - - @JsOverlay - default boolean isUtf8() { - return (Object) this instanceof Utf8; - } - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, Binary p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, Bool p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, Date p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, Decimal p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, Duration p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, FixedSizeBinary p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, FixedSizeList p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, FloatingPoint p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, Int p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, Interval p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, LargeBinary p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, LargeList p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, LargeUtf8 p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, List p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, Map p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, Null p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke( - double p0, Flatbuf.UnionListToTypeAccessorFn.P1UnionType p1); - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, Struct_ p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, Time p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, Timestamp p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, Union p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - - @JsOverlay - default Flatbuf.UnionListToTypeAccessorFn.UnionType onInvoke(double p0, Utf8 p1) { - return onInvoke(p0, Js.uncheckedCast(p1)); - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface UnionListToTypeUnionType { - @JsOverlay - static Flatbuf.UnionListToTypeUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Binary asBinary() { - return Js.cast(this); - } - - @JsOverlay - default Bool asBool() { - return Js.cast(this); - } - - @JsOverlay - default Date asDate() { - return Js.cast(this); - } - - @JsOverlay - default Decimal asDecimal() { - return Js.cast(this); - } - - @JsOverlay - default Duration asDuration() { - return Js.cast(this); - } - - @JsOverlay - default FixedSizeBinary asFixedSizeBinary() { - return Js.cast(this); - } - - @JsOverlay - default FixedSizeList asFixedSizeList() { - return Js.cast(this); - } - - @JsOverlay - default FloatingPoint asFloatingPoint() { - return Js.cast(this); - } - - @JsOverlay - default Int asInt() { - return Js.cast(this); - } - - @JsOverlay - default Interval asInterval() { - return Js.cast(this); - } - - @JsOverlay - default LargeBinary asLargeBinary() { - return Js.cast(this); - } - - @JsOverlay - default LargeList asLargeList() { - return Js.cast(this); - } - - @JsOverlay - default LargeUtf8 asLargeUtf8() { - return Js.cast(this); - } - - @JsOverlay - default List asList() { - return Js.cast(this); - } - - @JsOverlay - default Map asMap() { - return Js.cast(this); - } - - @JsOverlay - default Null asNull() { - return Js.cast(this); - } - - @JsOverlay - default Struct_ asStruct_() { - return Js.cast(this); - } - - @JsOverlay - default Time asTime() { - return Js.cast(this); - } - - @JsOverlay - default Timestamp asTimestamp() { - return Js.cast(this); - } - - @JsOverlay - default Union asUnion() { - return Js.cast(this); - } - - @JsOverlay - default Utf8 asUtf8() { - return Js.cast(this); - } - - @JsOverlay - default boolean isBinary() { - return (Object) this instanceof Binary; - } - - @JsOverlay - default boolean isBool() { - return (Object) this instanceof Bool; - } - - @JsOverlay - default boolean isDate() { - return (Object) this instanceof Date; - } - - @JsOverlay - default boolean isDecimal() { - return (Object) this instanceof Decimal; - } - - @JsOverlay - default boolean isDuration() { - return (Object) this instanceof Duration; - } - - @JsOverlay - default boolean isFixedSizeBinary() { - return (Object) this instanceof FixedSizeBinary; - } - - @JsOverlay - default boolean isFixedSizeList() { - return (Object) this instanceof FixedSizeList; - } - - @JsOverlay - default boolean isFloatingPoint() { - return (Object) this instanceof FloatingPoint; - } - - @JsOverlay - default boolean isInt() { - return (Object) this instanceof Int; - } - - @JsOverlay - default boolean isInterval() { - return (Object) this instanceof Interval; - } - - @JsOverlay - default boolean isLargeBinary() { - return (Object) this instanceof LargeBinary; - } - - @JsOverlay - default boolean isLargeList() { - return (Object) this instanceof LargeList; - } - - @JsOverlay - default boolean isLargeUtf8() { - return (Object) this instanceof LargeUtf8; - } - - @JsOverlay - default boolean isList() { - return (Object) this instanceof List; - } - - @JsOverlay - default boolean isMap() { - return (Object) this instanceof Map; - } - - @JsOverlay - default boolean isNull() { - return (Object) this instanceof Null; - } - - @JsOverlay - default boolean isStruct_() { - return (Object) this instanceof Struct_; - } - - @JsOverlay - default boolean isTime() { - return (Object) this instanceof Time; - } - - @JsOverlay - default boolean isTimestamp() { - return (Object) this instanceof Timestamp; - } - - @JsOverlay - default boolean isUnion() { - return (Object) this instanceof Union; - } - - @JsOverlay - default boolean isUtf8() { - return (Object) this instanceof Utf8; - } - } - - @JsFunction - public interface UnionToTypeAccessorFn { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface P0UnionType { - @JsOverlay - static Flatbuf.UnionToTypeAccessorFn.P0UnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Binary asBinary() { - return Js.cast(this); - } - - @JsOverlay - default Bool asBool() { - return Js.cast(this); - } - - @JsOverlay - default Date asDate() { - return Js.cast(this); - } - - @JsOverlay - default Decimal asDecimal() { - return Js.cast(this); - } - - @JsOverlay - default Duration asDuration() { - return Js.cast(this); - } - - @JsOverlay - default FixedSizeBinary asFixedSizeBinary() { - return Js.cast(this); - } - - @JsOverlay - default FixedSizeList asFixedSizeList() { - return Js.cast(this); - } - - @JsOverlay - default FloatingPoint asFloatingPoint() { - return Js.cast(this); - } - - @JsOverlay - default Int asInt() { - return Js.cast(this); - } - - @JsOverlay - default Interval asInterval() { - return Js.cast(this); - } - - @JsOverlay - default LargeBinary asLargeBinary() { - return Js.cast(this); - } - - @JsOverlay - default LargeList asLargeList() { - return Js.cast(this); - } - - @JsOverlay - default LargeUtf8 asLargeUtf8() { - return Js.cast(this); - } - - @JsOverlay - default List asList() { - return Js.cast(this); - } - - @JsOverlay - default Map asMap() { - return Js.cast(this); - } - - @JsOverlay - default Null asNull() { - return Js.cast(this); - } - - @JsOverlay - default Struct_ asStruct_() { - return Js.cast(this); - } - - @JsOverlay - default Time asTime() { - return Js.cast(this); - } - - @JsOverlay - default Timestamp asTimestamp() { - return Js.cast(this); - } - - @JsOverlay - default Union asUnion() { - return Js.cast(this); - } - - @JsOverlay - default Utf8 asUtf8() { - return Js.cast(this); - } - - @JsOverlay - default boolean isBinary() { - return (Object) this instanceof Binary; - } - - @JsOverlay - default boolean isBool() { - return (Object) this instanceof Bool; - } - - @JsOverlay - default boolean isDate() { - return (Object) this instanceof Date; - } - - @JsOverlay - default boolean isDecimal() { - return (Object) this instanceof Decimal; - } - - @JsOverlay - default boolean isDuration() { - return (Object) this instanceof Duration; - } - - @JsOverlay - default boolean isFixedSizeBinary() { - return (Object) this instanceof FixedSizeBinary; - } - - @JsOverlay - default boolean isFixedSizeList() { - return (Object) this instanceof FixedSizeList; - } - - @JsOverlay - default boolean isFloatingPoint() { - return (Object) this instanceof FloatingPoint; - } - - @JsOverlay - default boolean isInt() { - return (Object) this instanceof Int; - } - - @JsOverlay - default boolean isInterval() { - return (Object) this instanceof Interval; - } - - @JsOverlay - default boolean isLargeBinary() { - return (Object) this instanceof LargeBinary; - } - - @JsOverlay - default boolean isLargeList() { - return (Object) this instanceof LargeList; - } - - @JsOverlay - default boolean isLargeUtf8() { - return (Object) this instanceof LargeUtf8; - } - - @JsOverlay - default boolean isList() { - return (Object) this instanceof List; - } - - @JsOverlay - default boolean isMap() { - return (Object) this instanceof Map; - } - - @JsOverlay - default boolean isNull() { - return (Object) this instanceof Null; - } - - @JsOverlay - default boolean isStruct_() { - return (Object) this instanceof Struct_; - } - - @JsOverlay - default boolean isTime() { - return (Object) this instanceof Time; - } - - @JsOverlay - default boolean isTimestamp() { - return (Object) this instanceof Timestamp; - } - - @JsOverlay - default boolean isUnion() { - return (Object) this instanceof Union; - } - - @JsOverlay - default boolean isUtf8() { - return (Object) this instanceof Utf8; - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface UnionType { - @JsOverlay - static Flatbuf.UnionToTypeAccessorFn.UnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Binary asBinary() { - return Js.cast(this); - } - - @JsOverlay - default Bool asBool() { - return Js.cast(this); - } - - @JsOverlay - default Date asDate() { - return Js.cast(this); - } - - @JsOverlay - default Decimal asDecimal() { - return Js.cast(this); - } - - @JsOverlay - default Duration asDuration() { - return Js.cast(this); - } - - @JsOverlay - default FixedSizeBinary asFixedSizeBinary() { - return Js.cast(this); - } - - @JsOverlay - default FixedSizeList asFixedSizeList() { - return Js.cast(this); - } - - @JsOverlay - default FloatingPoint asFloatingPoint() { - return Js.cast(this); - } - - @JsOverlay - default Int asInt() { - return Js.cast(this); - } - - @JsOverlay - default Interval asInterval() { - return Js.cast(this); - } - - @JsOverlay - default LargeBinary asLargeBinary() { - return Js.cast(this); - } - - @JsOverlay - default LargeList asLargeList() { - return Js.cast(this); - } - - @JsOverlay - default LargeUtf8 asLargeUtf8() { - return Js.cast(this); - } - - @JsOverlay - default List asList() { - return Js.cast(this); - } - - @JsOverlay - default Map asMap() { - return Js.cast(this); - } - - @JsOverlay - default Null asNull() { - return Js.cast(this); - } - - @JsOverlay - default Struct_ asStruct_() { - return Js.cast(this); - } - - @JsOverlay - default Time asTime() { - return Js.cast(this); - } - - @JsOverlay - default Timestamp asTimestamp() { - return Js.cast(this); - } - - @JsOverlay - default Union asUnion() { - return Js.cast(this); - } - - @JsOverlay - default Utf8 asUtf8() { - return Js.cast(this); - } - - @JsOverlay - default boolean isBinary() { - return (Object) this instanceof Binary; - } - - @JsOverlay - default boolean isBool() { - return (Object) this instanceof Bool; - } - - @JsOverlay - default boolean isDate() { - return (Object) this instanceof Date; - } - - @JsOverlay - default boolean isDecimal() { - return (Object) this instanceof Decimal; - } - - @JsOverlay - default boolean isDuration() { - return (Object) this instanceof Duration; - } - - @JsOverlay - default boolean isFixedSizeBinary() { - return (Object) this instanceof FixedSizeBinary; - } - - @JsOverlay - default boolean isFixedSizeList() { - return (Object) this instanceof FixedSizeList; - } - - @JsOverlay - default boolean isFloatingPoint() { - return (Object) this instanceof FloatingPoint; - } - - @JsOverlay - default boolean isInt() { - return (Object) this instanceof Int; - } - - @JsOverlay - default boolean isInterval() { - return (Object) this instanceof Interval; - } - - @JsOverlay - default boolean isLargeBinary() { - return (Object) this instanceof LargeBinary; - } - - @JsOverlay - default boolean isLargeList() { - return (Object) this instanceof LargeList; - } - - @JsOverlay - default boolean isLargeUtf8() { - return (Object) this instanceof LargeUtf8; - } - - @JsOverlay - default boolean isList() { - return (Object) this instanceof List; - } - - @JsOverlay - default boolean isMap() { - return (Object) this instanceof Map; - } - - @JsOverlay - default boolean isNull() { - return (Object) this instanceof Null; - } - - @JsOverlay - default boolean isStruct_() { - return (Object) this instanceof Struct_; - } - - @JsOverlay - default boolean isTime() { - return (Object) this instanceof Time; - } - - @JsOverlay - default boolean isTimestamp() { - return (Object) this instanceof Timestamp; - } - - @JsOverlay - default boolean isUnion() { - return (Object) this instanceof Union; - } - - @JsOverlay - default boolean isUtf8() { - return (Object) this instanceof Utf8; - } - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(Binary p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(Bool p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(Date p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(Decimal p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(Duration p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(FixedSizeBinary p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(FixedSizeList p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(FloatingPoint p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(Int p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(Interval p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(LargeBinary p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(LargeList p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(LargeUtf8 p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(List p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(Map p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(Null p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(Flatbuf.UnionToTypeAccessorFn.P0UnionType p0); - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(Struct_ p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(Time p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(Timestamp p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(Union p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - - @JsOverlay - default Flatbuf.UnionToTypeAccessorFn.UnionType onInvoke(Utf8 p0) { - return onInvoke(Js.uncheckedCast(p0)); - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface UnionToTypeUnionType { - @JsOverlay - static Flatbuf.UnionToTypeUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Binary asBinary() { - return Js.cast(this); - } - - @JsOverlay - default Bool asBool() { - return Js.cast(this); - } - - @JsOverlay - default Date asDate() { - return Js.cast(this); - } - - @JsOverlay - default Decimal asDecimal() { - return Js.cast(this); - } - - @JsOverlay - default Duration asDuration() { - return Js.cast(this); - } - - @JsOverlay - default FixedSizeBinary asFixedSizeBinary() { - return Js.cast(this); - } - - @JsOverlay - default FixedSizeList asFixedSizeList() { - return Js.cast(this); - } - - @JsOverlay - default FloatingPoint asFloatingPoint() { - return Js.cast(this); - } - - @JsOverlay - default Int asInt() { - return Js.cast(this); - } - - @JsOverlay - default Interval asInterval() { - return Js.cast(this); - } - - @JsOverlay - default LargeBinary asLargeBinary() { - return Js.cast(this); - } - - @JsOverlay - default LargeList asLargeList() { - return Js.cast(this); - } - - @JsOverlay - default LargeUtf8 asLargeUtf8() { - return Js.cast(this); - } - - @JsOverlay - default List asList() { - return Js.cast(this); - } - - @JsOverlay - default Map asMap() { - return Js.cast(this); - } - - @JsOverlay - default Null asNull() { - return Js.cast(this); - } - - @JsOverlay - default Struct_ asStruct_() { - return Js.cast(this); - } - - @JsOverlay - default Time asTime() { - return Js.cast(this); - } - - @JsOverlay - default Timestamp asTimestamp() { - return Js.cast(this); - } - - @JsOverlay - default Union asUnion() { - return Js.cast(this); - } - - @JsOverlay - default Utf8 asUtf8() { - return Js.cast(this); - } - - @JsOverlay - default boolean isBinary() { - return (Object) this instanceof Binary; - } - - @JsOverlay - default boolean isBool() { - return (Object) this instanceof Bool; - } - - @JsOverlay - default boolean isDate() { - return (Object) this instanceof Date; - } - - @JsOverlay - default boolean isDecimal() { - return (Object) this instanceof Decimal; - } - - @JsOverlay - default boolean isDuration() { - return (Object) this instanceof Duration; - } - - @JsOverlay - default boolean isFixedSizeBinary() { - return (Object) this instanceof FixedSizeBinary; - } - - @JsOverlay - default boolean isFixedSizeList() { - return (Object) this instanceof FixedSizeList; - } - - @JsOverlay - default boolean isFloatingPoint() { - return (Object) this instanceof FloatingPoint; - } - - @JsOverlay - default boolean isInt() { - return (Object) this instanceof Int; - } - - @JsOverlay - default boolean isInterval() { - return (Object) this instanceof Interval; - } - - @JsOverlay - default boolean isLargeBinary() { - return (Object) this instanceof LargeBinary; - } - - @JsOverlay - default boolean isLargeList() { - return (Object) this instanceof LargeList; - } - - @JsOverlay - default boolean isLargeUtf8() { - return (Object) this instanceof LargeUtf8; - } - - @JsOverlay - default boolean isList() { - return (Object) this instanceof List; - } - - @JsOverlay - default boolean isMap() { - return (Object) this instanceof Map; - } - - @JsOverlay - default boolean isNull() { - return (Object) this instanceof Null; - } - - @JsOverlay - default boolean isStruct_() { - return (Object) this instanceof Struct_; - } - - @JsOverlay - default boolean isTime() { - return (Object) this instanceof Time; - } - - @JsOverlay - default boolean isTimestamp() { - return (Object) this instanceof Timestamp; - } - - @JsOverlay - default boolean isUnion() { - return (Object) this instanceof Union; - } - - @JsOverlay - default boolean isUtf8() { - return (Object) this instanceof Utf8; - } - } - - public static native Flatbuf.UnionListToTypeUnionType unionListToType( - int type, Flatbuf.UnionListToTypeAccessorFn accessor, double index); - - public static native Flatbuf.UnionToTypeUnionType unionToType( - int type, Flatbuf.UnionToTypeAccessorFn accessor); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Binary.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Binary.java deleted file mode 100644 index 9174b46ad62..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Binary.java +++ /dev/null @@ -1,34 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Binary", - namespace = JsPackage.GLOBAL) -public class Binary { - public static native double createBinary(Builder builder); - - public static native double endBinary(Builder builder); - - public static native Binary getRootAsBinary(ByteBuffer bb, Binary obj); - - public static native Binary getRootAsBinary(ByteBuffer bb); - - public static native Binary getSizePrefixedRootAsBinary(ByteBuffer bb, Binary obj); - - public static native Binary getSizePrefixedRootAsBinary(ByteBuffer bb); - - public static native void startBinary(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native Binary __init(double i, ByteBuffer bb); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Bool.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Bool.java deleted file mode 100644 index a995e0e6a1d..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Bool.java +++ /dev/null @@ -1,34 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Bool", - namespace = JsPackage.GLOBAL) -public class Bool { - public static native double createBool(Builder builder); - - public static native double endBool(Builder builder); - - public static native Bool getRootAsBool(ByteBuffer bb, Bool obj); - - public static native Bool getRootAsBool(ByteBuffer bb); - - public static native Bool getSizePrefixedRootAsBool(ByteBuffer bb, Bool obj); - - public static native Bool getSizePrefixedRootAsBool(ByteBuffer bb); - - public static native void startBool(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native Bool __init(double i, ByteBuffer bb); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Buffer.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Buffer.java deleted file mode 100644 index 9cb4e6bc3c6..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Buffer.java +++ /dev/null @@ -1,29 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Long; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Buffer", - namespace = JsPackage.GLOBAL) -public class Buffer { - public static native double createBuffer(Builder builder, Long offset, Long length); - - public static native double sizeOf(); - - public ByteBuffer bb; - public double bb_pos; - - public native Buffer __init(double i, ByteBuffer bb); - - public native Long length(); - - public native Long offset(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Date.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Date.java deleted file mode 100644 index 37483e70bcc..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Date.java +++ /dev/null @@ -1,38 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Date", - namespace = JsPackage.GLOBAL) -public class Date { - public static native void addUnit(Builder builder, int unit); - - public static native double createDate(Builder builder, int unit); - - public static native double endDate(Builder builder); - - public static native Date getRootAsDate(ByteBuffer bb, Date obj); - - public static native Date getRootAsDate(ByteBuffer bb); - - public static native Date getSizePrefixedRootAsDate(ByteBuffer bb, Date obj); - - public static native Date getSizePrefixedRootAsDate(ByteBuffer bb); - - public static native void startDate(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native Date __init(double i, ByteBuffer bb); - - public native int unit(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/DateUnit.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/DateUnit.java deleted file mode 100644 index b5edb152cb7..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/DateUnit.java +++ /dev/null @@ -1,16 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.DateUnit", - namespace = JsPackage.GLOBAL) -public class DateUnit { - public static int DAY, - MILLISECOND; -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Decimal.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Decimal.java deleted file mode 100644 index fa3ee35026b..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Decimal.java +++ /dev/null @@ -1,47 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Decimal", - namespace = JsPackage.GLOBAL) -public class Decimal { - public static native void addBitWidth(Builder builder, double bitWidth); - - public static native void addPrecision(Builder builder, double precision); - - public static native void addScale(Builder builder, double scale); - - public static native double createDecimal( - Builder builder, double precision, double scale, double bitWidth); - - public static native double endDecimal(Builder builder); - - public static native Decimal getRootAsDecimal(ByteBuffer bb, Decimal obj); - - public static native Decimal getRootAsDecimal(ByteBuffer bb); - - public static native Decimal getSizePrefixedRootAsDecimal(ByteBuffer bb, Decimal obj); - - public static native Decimal getSizePrefixedRootAsDecimal(ByteBuffer bb); - - public static native void startDecimal(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native Decimal __init(double i, ByteBuffer bb); - - public native double bitWidth(); - - public native double precision(); - - public native double scale(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/DictionaryEncoding.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/DictionaryEncoding.java deleted file mode 100644 index 9be0497b0ec..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/DictionaryEncoding.java +++ /dev/null @@ -1,53 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Long; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.DictionaryEncoding", - namespace = JsPackage.GLOBAL) -public class DictionaryEncoding { - public static native void addDictionaryKind(Builder builder, int dictionaryKind); - - public static native void addId(Builder builder, Long id); - - public static native void addIndexType(Builder builder, double indexTypeOffset); - - public static native void addIsOrdered(Builder builder, boolean isOrdered); - - public static native double endDictionaryEncoding(Builder builder); - - public static native DictionaryEncoding getRootAsDictionaryEncoding( - ByteBuffer bb, DictionaryEncoding obj); - - public static native DictionaryEncoding getRootAsDictionaryEncoding(ByteBuffer bb); - - public static native DictionaryEncoding getSizePrefixedRootAsDictionaryEncoding( - ByteBuffer bb, DictionaryEncoding obj); - - public static native DictionaryEncoding getSizePrefixedRootAsDictionaryEncoding(ByteBuffer bb); - - public static native void startDictionaryEncoding(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native DictionaryEncoding __init(double i, ByteBuffer bb); - - public native int dictionaryKind(); - - public native Long id(); - - public native Int indexType(); - - public native Int indexType(Int obj); - - public native boolean isOrdered(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/DictionaryKind.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/DictionaryKind.java deleted file mode 100644 index 0a78b7f3036..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/DictionaryKind.java +++ /dev/null @@ -1,15 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.DictionaryKind", - namespace = JsPackage.GLOBAL) -public class DictionaryKind { - public static int DenseArray; -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Duration.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Duration.java deleted file mode 100644 index ae2bc2aee9d..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Duration.java +++ /dev/null @@ -1,38 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Duration", - namespace = JsPackage.GLOBAL) -public class Duration { - public static native void addUnit(Builder builder, int unit); - - public static native double createDuration(Builder builder, int unit); - - public static native double endDuration(Builder builder); - - public static native Duration getRootAsDuration(ByteBuffer bb, Duration obj); - - public static native Duration getRootAsDuration(ByteBuffer bb); - - public static native Duration getSizePrefixedRootAsDuration(ByteBuffer bb, Duration obj); - - public static native Duration getSizePrefixedRootAsDuration(ByteBuffer bb); - - public static native void startDuration(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native Duration __init(double i, ByteBuffer bb); - - public native int unit(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Endianness.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Endianness.java deleted file mode 100644 index e29338b855b..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Endianness.java +++ /dev/null @@ -1,16 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Endianness", - namespace = JsPackage.GLOBAL) -public class Endianness { - public static int Big, - Little; -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Feature.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Feature.java deleted file mode 100644 index 5b81178a43e..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Feature.java +++ /dev/null @@ -1,17 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Feature", - namespace = JsPackage.GLOBAL) -public class Feature { - public static int COMPRESSED_BODY, - DICTIONARY_REPLACEMENT, - UNUSED; -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Field.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Field.java deleted file mode 100644 index 6cdc1706df3..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Field.java +++ /dev/null @@ -1,123 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import elemental2.core.JsArray; -import elemental2.core.Uint8Array; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Encoding; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Field", - namespace = JsPackage.GLOBAL) -public class Field { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface NameUnionType { - @JsOverlay - static Field.NameUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default String asString() { - return Js.asString(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isString() { - return (Object) this instanceof String; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - public static native void addChildren(Builder builder, double childrenOffset); - - public static native void addCustomMetadata(Builder builder, double customMetadataOffset); - - public static native void addDictionary(Builder builder, double dictionaryOffset); - - public static native void addName(Builder builder, double nameOffset); - - public static native void addNullable(Builder builder, boolean nullable); - - public static native void addType(Builder builder, double typeOffset); - - public static native void addTypeType(Builder builder, int typeType); - - public static native double createChildrenVector(Builder builder, JsArray data); - - @JsOverlay - public static final double createChildrenVector(Builder builder, double[] data) { - return createChildrenVector(builder, Js.>uncheckedCast(data)); - } - - public static native double createCustomMetadataVector(Builder builder, JsArray data); - - @JsOverlay - public static final double createCustomMetadataVector(Builder builder, double[] data) { - return createCustomMetadataVector(builder, Js.>uncheckedCast(data)); - } - - public static native double endField(Builder builder); - - public static native Field getRootAsField(ByteBuffer bb, Field obj); - - public static native Field getRootAsField(ByteBuffer bb); - - public static native Field getSizePrefixedRootAsField(ByteBuffer bb, Field obj); - - public static native Field getSizePrefixedRootAsField(ByteBuffer bb); - - public static native void startChildrenVector(Builder builder, double numElems); - - public static native void startCustomMetadataVector(Builder builder, double numElems); - - public static native void startField(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native Field __init(double i, ByteBuffer bb); - - public native Field children(double index, Field obj); - - public native Field children(double index); - - public native double childrenLength(); - - public native KeyValue customMetadata(double index, KeyValue obj); - - public native KeyValue customMetadata(double index); - - public native double customMetadataLength(); - - public native DictionaryEncoding dictionary(); - - public native DictionaryEncoding dictionary(DictionaryEncoding obj); - - public native Field.NameUnionType name(); - - public native Field.NameUnionType name(Encoding optionalEncoding); - - public native boolean nullable(); - - public native T type(T obj); - - public native int typeType(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/FixedSizeBinary.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/FixedSizeBinary.java deleted file mode 100644 index 701078f258f..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/FixedSizeBinary.java +++ /dev/null @@ -1,39 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.FixedSizeBinary", - namespace = JsPackage.GLOBAL) -public class FixedSizeBinary { - public static native void addByteWidth(Builder builder, double byteWidth); - - public static native double createFixedSizeBinary(Builder builder, double byteWidth); - - public static native double endFixedSizeBinary(Builder builder); - - public static native FixedSizeBinary getRootAsFixedSizeBinary(ByteBuffer bb, FixedSizeBinary obj); - - public static native FixedSizeBinary getRootAsFixedSizeBinary(ByteBuffer bb); - - public static native FixedSizeBinary getSizePrefixedRootAsFixedSizeBinary( - ByteBuffer bb, FixedSizeBinary obj); - - public static native FixedSizeBinary getSizePrefixedRootAsFixedSizeBinary(ByteBuffer bb); - - public static native void startFixedSizeBinary(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native FixedSizeBinary __init(double i, ByteBuffer bb); - - public native double byteWidth(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/FixedSizeList.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/FixedSizeList.java deleted file mode 100644 index 6e7f302b85d..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/FixedSizeList.java +++ /dev/null @@ -1,39 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.FixedSizeList", - namespace = JsPackage.GLOBAL) -public class FixedSizeList { - public static native void addListSize(Builder builder, double listSize); - - public static native double createFixedSizeList(Builder builder, double listSize); - - public static native double endFixedSizeList(Builder builder); - - public static native FixedSizeList getRootAsFixedSizeList(ByteBuffer bb, FixedSizeList obj); - - public static native FixedSizeList getRootAsFixedSizeList(ByteBuffer bb); - - public static native FixedSizeList getSizePrefixedRootAsFixedSizeList( - ByteBuffer bb, FixedSizeList obj); - - public static native FixedSizeList getSizePrefixedRootAsFixedSizeList(ByteBuffer bb); - - public static native void startFixedSizeList(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native FixedSizeList __init(double i, ByteBuffer bb); - - public native double listSize(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/FloatingPoint.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/FloatingPoint.java deleted file mode 100644 index 739c3f455d6..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/FloatingPoint.java +++ /dev/null @@ -1,39 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.FloatingPoint", - namespace = JsPackage.GLOBAL) -public class FloatingPoint { - public static native void addPrecision(Builder builder, int precision); - - public static native double createFloatingPoint(Builder builder, int precision); - - public static native double endFloatingPoint(Builder builder); - - public static native FloatingPoint getRootAsFloatingPoint(ByteBuffer bb, FloatingPoint obj); - - public static native FloatingPoint getRootAsFloatingPoint(ByteBuffer bb); - - public static native FloatingPoint getSizePrefixedRootAsFloatingPoint( - ByteBuffer bb, FloatingPoint obj); - - public static native FloatingPoint getSizePrefixedRootAsFloatingPoint(ByteBuffer bb); - - public static native void startFloatingPoint(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native FloatingPoint __init(double i, ByteBuffer bb); - - public native int precision(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Int.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Int.java deleted file mode 100644 index c1441852b02..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Int.java +++ /dev/null @@ -1,42 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Int", - namespace = JsPackage.GLOBAL) -public class Int { - public static native void addBitWidth(Builder builder, double bitWidth); - - public static native void addIsSigned(Builder builder, boolean isSigned); - - public static native double createInt(Builder builder, double bitWidth, boolean isSigned); - - public static native double endInt(Builder builder); - - public static native Int getRootAsInt(ByteBuffer bb, Int obj); - - public static native Int getRootAsInt(ByteBuffer bb); - - public static native Int getSizePrefixedRootAsInt(ByteBuffer bb, Int obj); - - public static native Int getSizePrefixedRootAsInt(ByteBuffer bb); - - public static native void startInt(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native Int __init(double i, ByteBuffer bb); - - public native double bitWidth(); - - public native boolean isSigned(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Interval.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Interval.java deleted file mode 100644 index c844a0f89d2..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Interval.java +++ /dev/null @@ -1,38 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Interval", - namespace = JsPackage.GLOBAL) -public class Interval { - public static native void addUnit(Builder builder, int unit); - - public static native double createInterval(Builder builder, int unit); - - public static native double endInterval(Builder builder); - - public static native Interval getRootAsInterval(ByteBuffer bb, Interval obj); - - public static native Interval getRootAsInterval(ByteBuffer bb); - - public static native Interval getSizePrefixedRootAsInterval(ByteBuffer bb, Interval obj); - - public static native Interval getSizePrefixedRootAsInterval(ByteBuffer bb); - - public static native void startInterval(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native Interval __init(double i, ByteBuffer bb); - - public native int unit(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/IntervalUnit.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/IntervalUnit.java deleted file mode 100644 index aee19255211..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/IntervalUnit.java +++ /dev/null @@ -1,16 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.IntervalUnit", - namespace = JsPackage.GLOBAL) -public class IntervalUnit { - public static int DAY_TIME, - YEAR_MONTH; -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/KeyValue.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/KeyValue.java deleted file mode 100644 index 3a7511d91eb..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/KeyValue.java +++ /dev/null @@ -1,106 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import elemental2.core.Uint8Array; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Encoding; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.KeyValue", - namespace = JsPackage.GLOBAL) -public class KeyValue { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface KeyUnionType { - @JsOverlay - static KeyValue.KeyUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default String asString() { - return Js.asString(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isString() { - return (Object) this instanceof String; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface ValueUnionType { - @JsOverlay - static KeyValue.ValueUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default String asString() { - return Js.asString(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isString() { - return (Object) this instanceof String; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - public static native void addKey(Builder builder, double keyOffset); - - public static native void addValue(Builder builder, double valueOffset); - - public static native double createKeyValue(Builder builder, double keyOffset, double valueOffset); - - public static native double endKeyValue(Builder builder); - - public static native KeyValue getRootAsKeyValue(ByteBuffer bb, KeyValue obj); - - public static native KeyValue getRootAsKeyValue(ByteBuffer bb); - - public static native KeyValue getSizePrefixedRootAsKeyValue(ByteBuffer bb, KeyValue obj); - - public static native KeyValue getSizePrefixedRootAsKeyValue(ByteBuffer bb); - - public static native void startKeyValue(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native KeyValue __init(double i, ByteBuffer bb); - - public native KeyValue.KeyUnionType key(); - - public native KeyValue.KeyUnionType key(Encoding optionalEncoding); - - public native KeyValue.ValueUnionType value(); - - public native KeyValue.ValueUnionType value(Encoding optionalEncoding); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/LargeBinary.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/LargeBinary.java deleted file mode 100644 index 98b3ca5480b..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/LargeBinary.java +++ /dev/null @@ -1,34 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.LargeBinary", - namespace = JsPackage.GLOBAL) -public class LargeBinary { - public static native double createLargeBinary(Builder builder); - - public static native double endLargeBinary(Builder builder); - - public static native LargeBinary getRootAsLargeBinary(ByteBuffer bb, LargeBinary obj); - - public static native LargeBinary getRootAsLargeBinary(ByteBuffer bb); - - public static native LargeBinary getSizePrefixedRootAsLargeBinary(ByteBuffer bb, LargeBinary obj); - - public static native LargeBinary getSizePrefixedRootAsLargeBinary(ByteBuffer bb); - - public static native void startLargeBinary(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native LargeBinary __init(double i, ByteBuffer bb); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/LargeList.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/LargeList.java deleted file mode 100644 index c7581db121c..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/LargeList.java +++ /dev/null @@ -1,34 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.LargeList", - namespace = JsPackage.GLOBAL) -public class LargeList { - public static native double createLargeList(Builder builder); - - public static native double endLargeList(Builder builder); - - public static native LargeList getRootAsLargeList(ByteBuffer bb, LargeList obj); - - public static native LargeList getRootAsLargeList(ByteBuffer bb); - - public static native LargeList getSizePrefixedRootAsLargeList(ByteBuffer bb, LargeList obj); - - public static native LargeList getSizePrefixedRootAsLargeList(ByteBuffer bb); - - public static native void startLargeList(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native LargeList __init(double i, ByteBuffer bb); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/LargeUtf8.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/LargeUtf8.java deleted file mode 100644 index 55031aa3e29..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/LargeUtf8.java +++ /dev/null @@ -1,34 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.LargeUtf8", - namespace = JsPackage.GLOBAL) -public class LargeUtf8 { - public static native double createLargeUtf8(Builder builder); - - public static native double endLargeUtf8(Builder builder); - - public static native LargeUtf8 getRootAsLargeUtf8(ByteBuffer bb, LargeUtf8 obj); - - public static native LargeUtf8 getRootAsLargeUtf8(ByteBuffer bb); - - public static native LargeUtf8 getSizePrefixedRootAsLargeUtf8(ByteBuffer bb, LargeUtf8 obj); - - public static native LargeUtf8 getSizePrefixedRootAsLargeUtf8(ByteBuffer bb); - - public static native void startLargeUtf8(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native LargeUtf8 __init(double i, ByteBuffer bb); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/List.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/List.java deleted file mode 100644 index f4a0f64ba0e..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/List.java +++ /dev/null @@ -1,34 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.List", - namespace = JsPackage.GLOBAL) -public class List { - public static native double createList(Builder builder); - - public static native double endList(Builder builder); - - public static native List getRootAsList(ByteBuffer bb, List obj); - - public static native List getRootAsList(ByteBuffer bb); - - public static native List getSizePrefixedRootAsList(ByteBuffer bb, List obj); - - public static native List getSizePrefixedRootAsList(ByteBuffer bb); - - public static native void startList(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native List __init(double i, ByteBuffer bb); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Map.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Map.java deleted file mode 100644 index 1a2e54e0fc6..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Map.java +++ /dev/null @@ -1,38 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Map", - namespace = JsPackage.GLOBAL) -public class Map { - public static native void addKeysSorted(Builder builder, boolean keysSorted); - - public static native double createMap(Builder builder, boolean keysSorted); - - public static native double endMap(Builder builder); - - public static native Map getRootAsMap(ByteBuffer bb, Map obj); - - public static native Map getRootAsMap(ByteBuffer bb); - - public static native Map getSizePrefixedRootAsMap(ByteBuffer bb, Map obj); - - public static native Map getSizePrefixedRootAsMap(ByteBuffer bb); - - public static native void startMap(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native Map __init(double i, ByteBuffer bb); - - public native boolean keysSorted(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/MetadataVersion.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/MetadataVersion.java deleted file mode 100644 index 6e26ef3e21b..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/MetadataVersion.java +++ /dev/null @@ -1,19 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.MetadataVersion", - namespace = JsPackage.GLOBAL) -public class MetadataVersion { - public static int V1, - V2, - V3, - V4, - V5; -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Null.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Null.java deleted file mode 100644 index a4c761da693..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Null.java +++ /dev/null @@ -1,34 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Null", - namespace = JsPackage.GLOBAL) -public class Null { - public static native double createNull(Builder builder); - - public static native double endNull(Builder builder); - - public static native Null getRootAsNull(ByteBuffer bb, Null obj); - - public static native Null getRootAsNull(ByteBuffer bb); - - public static native Null getSizePrefixedRootAsNull(ByteBuffer bb, Null obj); - - public static native Null getSizePrefixedRootAsNull(ByteBuffer bb); - - public static native void startNull(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native Null __init(double i, ByteBuffer bb); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Precision.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Precision.java deleted file mode 100644 index 0c7f6cad0e3..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Precision.java +++ /dev/null @@ -1,17 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Precision", - namespace = JsPackage.GLOBAL) -public class Precision { - public static int DOUBLE, - HALF, - SINGLE; -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Schema.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Schema.java deleted file mode 100644 index 23ddc811ef7..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Schema.java +++ /dev/null @@ -1,100 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import elemental2.core.JsArray; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Long; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Schema", - namespace = JsPackage.GLOBAL) -public class Schema { - public static native void addCustomMetadata(Builder builder, double customMetadataOffset); - - public static native void addEndianness(Builder builder, int endianness); - - public static native void addFeatures(Builder builder, double featuresOffset); - - public static native void addFields(Builder builder, double fieldsOffset); - - public static native double createCustomMetadataVector(Builder builder, JsArray data); - - @JsOverlay - public static final double createCustomMetadataVector(Builder builder, double[] data) { - return createCustomMetadataVector(builder, Js.>uncheckedCast(data)); - } - - public static native double createFeaturesVector(Builder builder, JsArray data); - - @JsOverlay - public static final double createFeaturesVector(Builder builder, Object[] data) { - return createFeaturesVector(builder, Js.>uncheckedCast(data)); - } - - public static native double createFieldsVector(Builder builder, JsArray data); - - @JsOverlay - public static final double createFieldsVector(Builder builder, double[] data) { - return createFieldsVector(builder, Js.>uncheckedCast(data)); - } - - public static native double createSchema( - Builder builder, - int endianness, - double fieldsOffset, - double customMetadataOffset, - double featuresOffset); - - public static native double endSchema(Builder builder); - - public static native void finishSchemaBuffer(Builder builder, double offset); - - public static native void finishSizePrefixedSchemaBuffer(Builder builder, double offset); - - public static native Schema getRootAsSchema(ByteBuffer bb, Schema obj); - - public static native Schema getRootAsSchema(ByteBuffer bb); - - public static native Schema getSizePrefixedRootAsSchema(ByteBuffer bb, Schema obj); - - public static native Schema getSizePrefixedRootAsSchema(ByteBuffer bb); - - public static native void startCustomMetadataVector(Builder builder, double numElems); - - public static native void startFeaturesVector(Builder builder, double numElems); - - public static native void startFieldsVector(Builder builder, double numElems); - - public static native void startSchema(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native Schema __init(double i, ByteBuffer bb); - - public native KeyValue customMetadata(double index, KeyValue obj); - - public native KeyValue customMetadata(double index); - - public native double customMetadataLength(); - - public native int endianness(); - - public native Long features(double index); - - public native double featuresLength(); - - public native Field fields(double index, Field obj); - - public native Field fields(double index); - - public native double fieldsLength(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Struct_.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Struct_.java deleted file mode 100644 index 62c88e3f8ae..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Struct_.java +++ /dev/null @@ -1,34 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Struct_", - namespace = JsPackage.GLOBAL) -public class Struct_ { - public static native double createStruct_(Builder builder); - - public static native double endStruct_(Builder builder); - - public static native Struct_ getRootAsStruct_(ByteBuffer bb, Struct_ obj); - - public static native Struct_ getRootAsStruct_(ByteBuffer bb); - - public static native Struct_ getSizePrefixedRootAsStruct_(ByteBuffer bb, Struct_ obj); - - public static native Struct_ getSizePrefixedRootAsStruct_(ByteBuffer bb); - - public static native void startStruct_(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native Struct_ __init(double i, ByteBuffer bb); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Time.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Time.java deleted file mode 100644 index 0f40072c5dd..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Time.java +++ /dev/null @@ -1,42 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Time", - namespace = JsPackage.GLOBAL) -public class Time { - public static native void addBitWidth(Builder builder, double bitWidth); - - public static native void addUnit(Builder builder, int unit); - - public static native double createTime(Builder builder, int unit, double bitWidth); - - public static native double endTime(Builder builder); - - public static native Time getRootAsTime(ByteBuffer bb, Time obj); - - public static native Time getRootAsTime(ByteBuffer bb); - - public static native Time getSizePrefixedRootAsTime(ByteBuffer bb, Time obj); - - public static native Time getSizePrefixedRootAsTime(ByteBuffer bb); - - public static native void startTime(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native Time __init(double i, ByteBuffer bb); - - public native double bitWidth(); - - public native int unit(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/TimeUnit.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/TimeUnit.java deleted file mode 100644 index 780d7cc9e08..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/TimeUnit.java +++ /dev/null @@ -1,18 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.TimeUnit", - namespace = JsPackage.GLOBAL) -public class TimeUnit { - public static int MICROSECOND, - MILLISECOND, - NANOSECOND, - SECOND; -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Timestamp.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Timestamp.java deleted file mode 100644 index fa1ef0923ed..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Timestamp.java +++ /dev/null @@ -1,77 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import elemental2.core.Uint8Array; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Encoding; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Timestamp", - namespace = JsPackage.GLOBAL) -public class Timestamp { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface TimezoneUnionType { - @JsOverlay - static Timestamp.TimezoneUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default String asString() { - return Js.asString(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isString() { - return (Object) this instanceof String; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - public static native void addTimezone(Builder builder, double timezoneOffset); - - public static native void addUnit(Builder builder, int unit); - - public static native double createTimestamp( - Builder builder, int unit, double timezoneOffset); - - public static native double endTimestamp(Builder builder); - - public static native Timestamp getRootAsTimestamp(ByteBuffer bb, Timestamp obj); - - public static native Timestamp getRootAsTimestamp(ByteBuffer bb); - - public static native Timestamp getSizePrefixedRootAsTimestamp(ByteBuffer bb, Timestamp obj); - - public static native Timestamp getSizePrefixedRootAsTimestamp(ByteBuffer bb); - - public static native void startTimestamp(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native Timestamp __init(double i, ByteBuffer bb); - - public native Timestamp.TimezoneUnionType timezone(); - - public native Timestamp.TimezoneUnionType timezone(Encoding optionalEncoding); - - public native int unit(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Type.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Type.java deleted file mode 100644 index 6336d2a547f..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Type.java +++ /dev/null @@ -1,36 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Type", - namespace = JsPackage.GLOBAL) -public class Type { - public static int Binary, - Bool, - Date, - Decimal, - Duration, - FixedSizeBinary, - FixedSizeList, - FloatingPoint, - Int, - Interval, - LargeBinary, - LargeList, - LargeUtf8, - List, - Map, - NONE, - Null, - Struct_, - Time, - Timestamp, - Union, - Utf8; -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Union.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Union.java deleted file mode 100644 index a3ff98e8975..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Union.java +++ /dev/null @@ -1,122 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import elemental2.core.Int32Array; -import elemental2.core.JsArray; -import elemental2.core.Uint8Array; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Union", - namespace = JsPackage.GLOBAL) -public class Union { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateTypeIdsVectorDataUnionType { - @JsOverlay - static Union.CreateTypeIdsVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int32Array asInt32Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt32Array() { - return (Object) this instanceof Int32Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - public static native void addMode(Builder builder, int mode); - - public static native void addTypeIds(Builder builder, double typeIdsOffset); - - @Deprecated - public static native double createTypeIdsVector( - Builder builder, Union.CreateTypeIdsVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createTypeIdsVector(Builder builder, Int32Array data) { - return createTypeIdsVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createTypeIdsVector(Builder builder, JsArray data) { - return createTypeIdsVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createTypeIdsVector(Builder builder, Uint8Array data) { - return createTypeIdsVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createTypeIdsVector(Builder builder, double[] data) { - return createTypeIdsVector(builder, Js.>uncheckedCast(data)); - } - - public static native double createUnion(Builder builder, int mode, double typeIdsOffset); - - public static native double endUnion(Builder builder); - - public static native Union getRootAsUnion(ByteBuffer bb, Union obj); - - public static native Union getRootAsUnion(ByteBuffer bb); - - public static native Union getSizePrefixedRootAsUnion(ByteBuffer bb, Union obj); - - public static native Union getSizePrefixedRootAsUnion(ByteBuffer bb); - - public static native void startTypeIdsVector(Builder builder, double numElems); - - public static native void startUnion(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native Union __init(double i, ByteBuffer bb); - - public native int mode(); - - public native double typeIds(double index); - - public native Int32Array typeIdsArray(); - - public native double typeIdsLength(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/UnionMode.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/UnionMode.java deleted file mode 100644 index 30553887de2..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/UnionMode.java +++ /dev/null @@ -1,16 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.UnionMode", - namespace = JsPackage.GLOBAL) -public class UnionMode { - public static int Dense, - Sparse; -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Utf8.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Utf8.java deleted file mode 100644 index 1149f1f8977..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/arrow/flight/flatbuf/schema_generated/org/apache/arrow/flatbuf/Utf8.java +++ /dev/null @@ -1,34 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.arrow.flight.flatbuf.schema_generated.org.apache.arrow.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.arrow.flight.flatbuf.Schema_generated.org.apache.arrow.flatbuf.Utf8", - namespace = JsPackage.GLOBAL) -public class Utf8 { - public static native double createUtf8(Builder builder); - - public static native double endUtf8(Builder builder); - - public static native Utf8 getRootAsUtf8(ByteBuffer bb, Utf8 obj); - - public static native Utf8 getRootAsUtf8(ByteBuffer bb); - - public static native Utf8 getSizePrefixedRootAsUtf8(ByteBuffer bb, Utf8 obj); - - public static native Utf8 getSizePrefixedRootAsUtf8(ByteBuffer bb); - - public static native void startUtf8(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native Utf8 __init(double i, ByteBuffer bb); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/flatbuffers/Builder.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/flatbuffers/Builder.java deleted file mode 100644 index 0c9118583d8..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/flatbuffers/Builder.java +++ /dev/null @@ -1,143 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.flatbuffers; - -import elemental2.core.Uint8Array; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType(isNative = true, name = "dhinternal.flatbuffers.Builder", namespace = JsPackage.GLOBAL) -public class Builder { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateStringSUnionType { - @JsOverlay - static Builder.CreateStringSUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default String asString() { - return Js.asString(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isString() { - return (Object) this instanceof String; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - public static native ByteBuffer growByteBuffer(ByteBuffer bb); - - public Builder() {} - - public Builder(double initial_size) {} - - public native void addFieldFloat32(double voffset, double value, double defaultValue); - - public native void addFieldFloat64(double voffset, double value, double defaultValue); - - public native void addFieldInt16(double voffset, double value, double defaultValue); - - public native void addFieldInt32(double voffset, double value, double defaultValue); - - public native void addFieldInt64(double voffset, Long value, Long defaultValue); - - public native void addFieldInt8(double voffset, double value, double defaultValue); - - public native void addFieldOffset(double voffset, double value, double defaultValue); - - public native void addFieldStruct(double voffset, double value, double defaultValue); - - public native void addFloat32(double value); - - public native void addFloat64(double value); - - public native void addInt16(double value); - - public native void addInt32(double value); - - public native void addInt64(Long value); - - public native void addInt8(double value); - - public native void addOffset(double offset); - - public native Uint8Array asUint8Array(); - - public native void clear(); - - public native Long createLong(double low, double high); - - public native double createString(Builder.CreateStringSUnionType s); - - @JsOverlay - public final double createString(String s) { - return createString(Js.uncheckedCast(s)); - } - - @JsOverlay - public final double createString(Uint8Array s) { - return createString(Js.uncheckedCast(s)); - } - - public native ByteBuffer dataBuffer(); - - public native double endObject(); - - public native double endVector(); - - public native void finish(double root_table, String file_identifier, boolean size_prefix); - - public native void finish(double root_table, String file_identifier); - - public native void finish(double root_table); - - public native void finishSizePrefixed(double root_table, String file_identifier); - - public native void finishSizePrefixed(double root_table); - - public native void forceDefaults(boolean forceDefaults); - - public native void nested(double obj); - - public native void notNested(); - - public native double offset(); - - public native void pad(double byte_size); - - public native void prep(double size, double additional_bytes); - - public native void requiredField(double table, double field); - - public native void slot(double voffset); - - public native void startObject(double numfields); - - public native void startVector(double elem_size, double num_elems, double alignment); - - public native void writeFloat32(double value); - - public native void writeFloat64(double value); - - public native void writeInt16(double value); - - public native void writeInt32(double value); - - public native void writeInt64(Long value); - - public native void writeInt8(double value); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/flatbuffers/ByteBuffer.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/flatbuffers/ByteBuffer.java deleted file mode 100644 index 85a71935d92..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/flatbuffers/ByteBuffer.java +++ /dev/null @@ -1,115 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.flatbuffers; - -import elemental2.core.Uint8Array; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType(isNative = true, name = "dhinternal.flatbuffers.ByteBuffer", namespace = JsPackage.GLOBAL) -public class ByteBuffer { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface __stringUnionType { - @JsOverlay - static ByteBuffer.__stringUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default String asString() { - return Js.asString(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isString() { - return (Object) this instanceof String; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - public static native ByteBuffer allocate(double byte_size); - - public ByteBuffer(Uint8Array bytes) {} - - public native boolean __has_identifier(String ident); - - public native double __indirect(double offset); - - public native double __offset(double bb_pos, double vtable_offset); - - public native ByteBuffer.__stringUnionType __string(double offset, int optionalEncoding); - - public native ByteBuffer.__stringUnionType __string(double offset); - - public native T __union(T t, double offset); - - public native double __vector(double offset); - - public native double __vector_len(double offset); - - public native Uint8Array bytes(); - - public native double capacity(); - - public native void clear(); - - public native Long createLong(double low, double high); - - public native String getBufferIdentifier(); - - public native double position(); - - public native double readFloat32(double offset); - - public native double readFloat64(double offset); - - public native double readInt16(double offset); - - public native double readInt32(double offset); - - public native Long readInt64(double offset); - - public native double readInt8(double offset); - - public native double readUint16(double offset); - - public native double readUint32(double offset); - - public native Long readUint64(double offset); - - public native double readUint8(double offset); - - public native void setPosition(double position); - - public native void writeFloat32(double offset, double value); - - public native void writeFloat64(double offset, double value); - - public native void writeInt16(double offset, double value); - - public native void writeInt32(double offset, double value); - - public native void writeInt64(double offset, Long value); - - public native void writeInt8(double offset, double value); - - public native void writeUint16(double offset, double value); - - public native void writeUint32(double offset, double value); - - public native void writeUint64(double offset, Long value); - - public native void writeUint8(double offset, double value); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/flatbuffers/Encoding.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/flatbuffers/Encoding.java deleted file mode 100644 index be269d6968a..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/flatbuffers/Encoding.java +++ /dev/null @@ -1,13 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.flatbuffers; - -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType(isNative = true, name = "dhinternal.flatbuffers.Encoding", namespace = JsPackage.GLOBAL) -public class Encoding { - public static int UTF16_STRING, - UTF8_BYTES; -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/flatbuffers/Long.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/flatbuffers/Long.java deleted file mode 100644 index e7551a82d7d..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/flatbuffers/Long.java +++ /dev/null @@ -1,25 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.flatbuffers; - -import jsinterop.annotations.JsMethod; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType(isNative = true, name = "dhinternal.flatbuffers.Long", namespace = JsPackage.GLOBAL) -public class Long { - public static Long ZERO; - - public static native Long create(double low, double high); - - public double high; - public double low; - - public Long(double low, double high) {} - - @JsMethod(name = "equals") - public native boolean equals_(Object other); - - public native double toFloat64(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/flatbuffers/Table.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/flatbuffers/Table.java deleted file mode 100644 index e2dcada04fb..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/flatbuffers/Table.java +++ /dev/null @@ -1,31 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.flatbuffers; - -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsProperty; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; -import jsinterop.base.JsPropertyMap; - -@JsType(isNative = true, name = "dhinternal.flatbuffers.Table", namespace = JsPackage.GLOBAL) -public interface Table { - @JsOverlay - static Table create() { - return Js.uncheckedCast(JsPropertyMap.of()); - } - - @JsProperty - ByteBuffer getBb(); - - @JsProperty - double getBb_pos(); - - @JsProperty - void setBb(ByteBuffer bb); - - @JsProperty - void setBb_pos(double bb_pos); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageMessageType.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageMessageType.java deleted file mode 100644 index 9dc50f672a7..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageMessageType.java +++ /dev/null @@ -1,23 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf; - -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven.barrage.flatbuf.Barrage_generated.io.deephaven.barrage.flatbuf.BarrageMessageType", - namespace = JsPackage.GLOBAL) -public class BarrageMessageType { - public static int BarragePublicationRequest, - BarrageSerializationOptions, - BarrageSnapshotRequest, - BarrageSubscriptionRequest, - BarrageUpdateMetadata, - NewSessionRequest, - None, - RefreshSessionRequest, - SessionInfoResponse; -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageMessageWrapper.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageMessageWrapper.java deleted file mode 100644 index 7de97b929cd..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageMessageWrapper.java +++ /dev/null @@ -1,130 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf; - -import elemental2.core.Int8Array; -import elemental2.core.JsArray; -import elemental2.core.Uint8Array; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven.barrage.flatbuf.Barrage_generated.io.deephaven.barrage.flatbuf.BarrageMessageWrapper", - namespace = JsPackage.GLOBAL) -public class BarrageMessageWrapper { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateMsgPayloadVectorDataUnionType { - @JsOverlay - static BarrageMessageWrapper.CreateMsgPayloadVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - public static native void addMagic(Builder builder, double magic); - - public static native void addMsgPayload(Builder builder, double msgPayloadOffset); - - public static native void addMsgType(Builder builder, int msgType); - - public static native double createBarrageMessageWrapper( - Builder builder, double magic, int msgType, double msgPayloadOffset); - - @Deprecated - public static native double createMsgPayloadVector( - Builder builder, BarrageMessageWrapper.CreateMsgPayloadVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createMsgPayloadVector(Builder builder, Int8Array data) { - return createMsgPayloadVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createMsgPayloadVector(Builder builder, JsArray data) { - return createMsgPayloadVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createMsgPayloadVector(Builder builder, Uint8Array data) { - return createMsgPayloadVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createMsgPayloadVector(Builder builder, double[] data) { - return createMsgPayloadVector(builder, Js.>uncheckedCast(data)); - } - - public static native double endBarrageMessageWrapper(Builder builder); - - public static native BarrageMessageWrapper getRootAsBarrageMessageWrapper( - ByteBuffer bb, BarrageMessageWrapper obj); - - public static native BarrageMessageWrapper getRootAsBarrageMessageWrapper(ByteBuffer bb); - - public static native BarrageMessageWrapper getSizePrefixedRootAsBarrageMessageWrapper( - ByteBuffer bb, BarrageMessageWrapper obj); - - public static native BarrageMessageWrapper getSizePrefixedRootAsBarrageMessageWrapper( - ByteBuffer bb); - - public static native void startBarrageMessageWrapper(Builder builder); - - public static native void startMsgPayloadVector(Builder builder, double numElems); - - public ByteBuffer bb; - public double bb_pos; - - public native BarrageMessageWrapper __init(double i, ByteBuffer bb); - - public native double magic(); - - public native double msgPayload(double index); - - public native Int8Array msgPayloadArray(); - - public native double msgPayloadLength(); - - public native int msgType(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageModColumnMetadata.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageModColumnMetadata.java deleted file mode 100644 index 076793a4be2..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageModColumnMetadata.java +++ /dev/null @@ -1,125 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf; - -import elemental2.core.Int8Array; -import elemental2.core.JsArray; -import elemental2.core.Uint8Array; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven.barrage.flatbuf.Barrage_generated.io.deephaven.barrage.flatbuf.BarrageModColumnMetadata", - namespace = JsPackage.GLOBAL) -public class BarrageModColumnMetadata { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateModifiedRowsVectorDataUnionType { - @JsOverlay - static BarrageModColumnMetadata.CreateModifiedRowsVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - public static native void addModifiedRows(Builder builder, double modifiedRowsOffset); - - public static native double createBarrageModColumnMetadata( - Builder builder, double modifiedRowsOffset); - - @Deprecated - public static native double createModifiedRowsVector( - Builder builder, BarrageModColumnMetadata.CreateModifiedRowsVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createModifiedRowsVector(Builder builder, Int8Array data) { - return createModifiedRowsVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createModifiedRowsVector(Builder builder, JsArray data) { - return createModifiedRowsVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createModifiedRowsVector(Builder builder, Uint8Array data) { - return createModifiedRowsVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createModifiedRowsVector(Builder builder, double[] data) { - return createModifiedRowsVector(builder, Js.>uncheckedCast(data)); - } - - public static native double endBarrageModColumnMetadata(Builder builder); - - public static native BarrageModColumnMetadata getRootAsBarrageModColumnMetadata( - ByteBuffer bb, BarrageModColumnMetadata obj); - - public static native BarrageModColumnMetadata getRootAsBarrageModColumnMetadata(ByteBuffer bb); - - public static native BarrageModColumnMetadata getSizePrefixedRootAsBarrageModColumnMetadata( - ByteBuffer bb, BarrageModColumnMetadata obj); - - public static native BarrageModColumnMetadata getSizePrefixedRootAsBarrageModColumnMetadata( - ByteBuffer bb); - - public static native void startBarrageModColumnMetadata(Builder builder); - - public static native void startModifiedRowsVector(Builder builder, double numElems); - - public ByteBuffer bb; - public double bb_pos; - - public native BarrageModColumnMetadata __init(double i, ByteBuffer bb); - - public native double modifiedRows(double index); - - public native Int8Array modifiedRowsArray(); - - public native double modifiedRowsLength(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarragePublicationOptions.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarragePublicationOptions.java deleted file mode 100644 index 0726110f013..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarragePublicationOptions.java +++ /dev/null @@ -1,42 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven.barrage.flatbuf.Barrage_generated.io.deephaven.barrage.flatbuf.BarragePublicationOptions", - namespace = JsPackage.GLOBAL) -public class BarragePublicationOptions { - public static native void addUseDeephavenNulls(Builder builder, boolean useDeephavenNulls); - - public static native double createBarragePublicationOptions( - Builder builder, boolean useDeephavenNulls); - - public static native double endBarragePublicationOptions(Builder builder); - - public static native BarragePublicationOptions getRootAsBarragePublicationOptions( - ByteBuffer bb, BarragePublicationOptions obj); - - public static native BarragePublicationOptions getRootAsBarragePublicationOptions(ByteBuffer bb); - - public static native BarragePublicationOptions getSizePrefixedRootAsBarragePublicationOptions( - ByteBuffer bb, BarragePublicationOptions obj); - - public static native BarragePublicationOptions getSizePrefixedRootAsBarragePublicationOptions( - ByteBuffer bb); - - public static native void startBarragePublicationOptions(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native BarragePublicationOptions __init(double i, ByteBuffer bb); - - public native boolean useDeephavenNulls(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarragePublicationRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarragePublicationRequest.java deleted file mode 100644 index ee5fbe08fcc..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarragePublicationRequest.java +++ /dev/null @@ -1,125 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf; - -import elemental2.core.Int8Array; -import elemental2.core.JsArray; -import elemental2.core.Uint8Array; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven.barrage.flatbuf.Barrage_generated.io.deephaven.barrage.flatbuf.BarragePublicationRequest", - namespace = JsPackage.GLOBAL) -public class BarragePublicationRequest { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateTicketVectorDataUnionType { - @JsOverlay - static BarragePublicationRequest.CreateTicketVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - public static native void addPublishOptions(Builder builder, double publishOptionsOffset); - - public static native void addTicket(Builder builder, double ticketOffset); - - @Deprecated - public static native double createTicketVector( - Builder builder, BarragePublicationRequest.CreateTicketVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createTicketVector(Builder builder, Int8Array data) { - return createTicketVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createTicketVector(Builder builder, JsArray data) { - return createTicketVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createTicketVector(Builder builder, Uint8Array data) { - return createTicketVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createTicketVector(Builder builder, double[] data) { - return createTicketVector(builder, Js.>uncheckedCast(data)); - } - - public static native double endBarragePublicationRequest(Builder builder); - - public static native BarragePublicationRequest getRootAsBarragePublicationRequest( - ByteBuffer bb, BarragePublicationRequest obj); - - public static native BarragePublicationRequest getRootAsBarragePublicationRequest(ByteBuffer bb); - - public static native BarragePublicationRequest getSizePrefixedRootAsBarragePublicationRequest( - ByteBuffer bb, BarragePublicationRequest obj); - - public static native BarragePublicationRequest getSizePrefixedRootAsBarragePublicationRequest( - ByteBuffer bb); - - public static native void startBarragePublicationRequest(Builder builder); - - public static native void startTicketVector(Builder builder, double numElems); - - public ByteBuffer bb; - public double bb_pos; - - public native BarragePublicationRequest __init(double i, ByteBuffer bb); - - public native BarragePublicationOptions publishOptions(); - - public native BarragePublicationOptions publishOptions(BarragePublicationOptions obj); - - public native double ticket(double index); - - public native Int8Array ticketArray(); - - public native double ticketLength(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageSnapshotOptions.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageSnapshotOptions.java deleted file mode 100644 index efedac5d190..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageSnapshotOptions.java +++ /dev/null @@ -1,59 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven.barrage.flatbuf.Barrage_generated.io.deephaven.barrage.flatbuf.BarrageSnapshotOptions", - namespace = JsPackage.GLOBAL) -public class BarrageSnapshotOptions { - public static native void addBatchSize(Builder builder, double batchSize); - - public static native void addColumnConversionMode( - Builder builder, int columnConversionMode); - - public static native void addMaxMessageSize(Builder builder, double maxMessageSize); - - public static native void addUseDeephavenNulls(Builder builder, boolean useDeephavenNulls); - - public static native double createBarrageSnapshotOptions( - Builder builder, - int columnConversionMode, - boolean useDeephavenNulls, - double batchSize, - double maxMessageSize); - - public static native double endBarrageSnapshotOptions(Builder builder); - - public static native BarrageSnapshotOptions getRootAsBarrageSnapshotOptions( - ByteBuffer bb, BarrageSnapshotOptions obj); - - public static native BarrageSnapshotOptions getRootAsBarrageSnapshotOptions(ByteBuffer bb); - - public static native BarrageSnapshotOptions getSizePrefixedRootAsBarrageSnapshotOptions( - ByteBuffer bb, BarrageSnapshotOptions obj); - - public static native BarrageSnapshotOptions getSizePrefixedRootAsBarrageSnapshotOptions( - ByteBuffer bb); - - public static native void startBarrageSnapshotOptions(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native BarrageSnapshotOptions __init(double i, ByteBuffer bb); - - public native double batchSize(); - - public native int columnConversionMode(); - - public native double maxMessageSize(); - - public native boolean useDeephavenNulls(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageSnapshotRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageSnapshotRequest.java deleted file mode 100644 index 8e8d43e6c58..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageSnapshotRequest.java +++ /dev/null @@ -1,287 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf; - -import elemental2.core.Int8Array; -import elemental2.core.JsArray; -import elemental2.core.Uint8Array; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven.barrage.flatbuf.Barrage_generated.io.deephaven.barrage.flatbuf.BarrageSnapshotRequest", - namespace = JsPackage.GLOBAL) -public class BarrageSnapshotRequest { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateColumnsVectorDataUnionType { - @JsOverlay - static BarrageSnapshotRequest.CreateColumnsVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateTicketVectorDataUnionType { - @JsOverlay - static BarrageSnapshotRequest.CreateTicketVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateViewportVectorDataUnionType { - @JsOverlay - static BarrageSnapshotRequest.CreateViewportVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - public static native void addColumns(Builder builder, double columnsOffset); - - public static native void addReverseViewport(Builder builder, boolean reverseViewport); - - public static native void addSnapshotOptions(Builder builder, double snapshotOptionsOffset); - - public static native void addTicket(Builder builder, double ticketOffset); - - public static native void addViewport(Builder builder, double viewportOffset); - - @Deprecated - public static native double createColumnsVector( - Builder builder, BarrageSnapshotRequest.CreateColumnsVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createColumnsVector(Builder builder, Int8Array data) { - return createColumnsVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createColumnsVector(Builder builder, JsArray data) { - return createColumnsVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createColumnsVector(Builder builder, Uint8Array data) { - return createColumnsVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createColumnsVector(Builder builder, double[] data) { - return createColumnsVector(builder, Js.>uncheckedCast(data)); - } - - @Deprecated - public static native double createTicketVector( - Builder builder, BarrageSnapshotRequest.CreateTicketVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createTicketVector(Builder builder, Int8Array data) { - return createTicketVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createTicketVector(Builder builder, JsArray data) { - return createTicketVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createTicketVector(Builder builder, Uint8Array data) { - return createTicketVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createTicketVector(Builder builder, double[] data) { - return createTicketVector(builder, Js.>uncheckedCast(data)); - } - - @Deprecated - public static native double createViewportVector( - Builder builder, BarrageSnapshotRequest.CreateViewportVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createViewportVector(Builder builder, Int8Array data) { - return createViewportVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createViewportVector(Builder builder, JsArray data) { - return createViewportVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createViewportVector(Builder builder, Uint8Array data) { - return createViewportVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createViewportVector(Builder builder, double[] data) { - return createViewportVector(builder, Js.>uncheckedCast(data)); - } - - public static native double endBarrageSnapshotRequest(Builder builder); - - public static native BarrageSnapshotRequest getRootAsBarrageSnapshotRequest( - ByteBuffer bb, BarrageSnapshotRequest obj); - - public static native BarrageSnapshotRequest getRootAsBarrageSnapshotRequest(ByteBuffer bb); - - public static native BarrageSnapshotRequest getSizePrefixedRootAsBarrageSnapshotRequest( - ByteBuffer bb, BarrageSnapshotRequest obj); - - public static native BarrageSnapshotRequest getSizePrefixedRootAsBarrageSnapshotRequest( - ByteBuffer bb); - - public static native void startBarrageSnapshotRequest(Builder builder); - - public static native void startColumnsVector(Builder builder, double numElems); - - public static native void startTicketVector(Builder builder, double numElems); - - public static native void startViewportVector(Builder builder, double numElems); - - public ByteBuffer bb; - public double bb_pos; - - public native BarrageSnapshotRequest __init(double i, ByteBuffer bb); - - public native double columns(double index); - - public native Int8Array columnsArray(); - - public native double columnsLength(); - - public native boolean reverseViewport(); - - public native BarrageSnapshotOptions snapshotOptions(); - - public native BarrageSnapshotOptions snapshotOptions(BarrageSnapshotOptions obj); - - public native double ticket(double index); - - public native Int8Array ticketArray(); - - public native double ticketLength(); - - public native double viewport(double index); - - public native Int8Array viewportArray(); - - public native double viewportLength(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageSubscriptionOptions.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageSubscriptionOptions.java deleted file mode 100644 index 9b6d39b53ec..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageSubscriptionOptions.java +++ /dev/null @@ -1,65 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf; - -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven.barrage.flatbuf.Barrage_generated.io.deephaven.barrage.flatbuf.BarrageSubscriptionOptions", - namespace = JsPackage.GLOBAL) -public class BarrageSubscriptionOptions { - public static native void addBatchSize(Builder builder, double batchSize); - - public static native void addColumnConversionMode( - Builder builder, int columnConversionMode); - - public static native void addMaxMessageSize(Builder builder, double maxMessageSize); - - public static native void addMinUpdateIntervalMs(Builder builder, double minUpdateIntervalMs); - - public static native void addUseDeephavenNulls(Builder builder, boolean useDeephavenNulls); - - public static native double createBarrageSubscriptionOptions( - Builder builder, - int columnConversionMode, - boolean useDeephavenNulls, - double minUpdateIntervalMs, - double batchSize, - double maxMessageSize); - - public static native double endBarrageSubscriptionOptions(Builder builder); - - public static native BarrageSubscriptionOptions getRootAsBarrageSubscriptionOptions( - ByteBuffer bb, BarrageSubscriptionOptions obj); - - public static native BarrageSubscriptionOptions getRootAsBarrageSubscriptionOptions( - ByteBuffer bb); - - public static native BarrageSubscriptionOptions getSizePrefixedRootAsBarrageSubscriptionOptions( - ByteBuffer bb, BarrageSubscriptionOptions obj); - - public static native BarrageSubscriptionOptions getSizePrefixedRootAsBarrageSubscriptionOptions( - ByteBuffer bb); - - public static native void startBarrageSubscriptionOptions(Builder builder); - - public ByteBuffer bb; - public double bb_pos; - - public native BarrageSubscriptionOptions __init(double i, ByteBuffer bb); - - public native double batchSize(); - - public native int columnConversionMode(); - - public native double maxMessageSize(); - - public native double minUpdateIntervalMs(); - - public native boolean useDeephavenNulls(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageSubscriptionRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageSubscriptionRequest.java deleted file mode 100644 index 807184ee34c..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageSubscriptionRequest.java +++ /dev/null @@ -1,298 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf; - -import elemental2.core.Int8Array; -import elemental2.core.JsArray; -import elemental2.core.Uint8Array; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven.barrage.flatbuf.Barrage_generated.io.deephaven.barrage.flatbuf.BarrageSubscriptionRequest", - namespace = JsPackage.GLOBAL) -public class BarrageSubscriptionRequest { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateColumnsVectorDataUnionType { - @JsOverlay - static BarrageSubscriptionRequest.CreateColumnsVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateTicketVectorDataUnionType { - @JsOverlay - static BarrageSubscriptionRequest.CreateTicketVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateViewportVectorDataUnionType { - @JsOverlay - static BarrageSubscriptionRequest.CreateViewportVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - public static native void addColumns(Builder builder, double columnsOffset); - - public static native void addReverseViewport(Builder builder, boolean reverseViewport); - - public static native void addSubscriptionOptions( - Builder builder, double subscriptionOptionsOffset); - - public static native void addTicket(Builder builder, double ticketOffset); - - public static native void addViewport(Builder builder, double viewportOffset); - - @Deprecated - public static native double createColumnsVector( - Builder builder, BarrageSubscriptionRequest.CreateColumnsVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createColumnsVector(Builder builder, Int8Array data) { - return createColumnsVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createColumnsVector(Builder builder, JsArray data) { - return createColumnsVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createColumnsVector(Builder builder, Uint8Array data) { - return createColumnsVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createColumnsVector(Builder builder, double[] data) { - return createColumnsVector(builder, Js.>uncheckedCast(data)); - } - - @Deprecated - public static native double createTicketVector( - Builder builder, BarrageSubscriptionRequest.CreateTicketVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createTicketVector(Builder builder, Int8Array data) { - return createTicketVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createTicketVector(Builder builder, JsArray data) { - return createTicketVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createTicketVector(Builder builder, Uint8Array data) { - return createTicketVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createTicketVector(Builder builder, double[] data) { - return createTicketVector(builder, Js.>uncheckedCast(data)); - } - - @Deprecated - public static native double createViewportVector( - Builder builder, BarrageSubscriptionRequest.CreateViewportVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createViewportVector(Builder builder, Int8Array data) { - return createViewportVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createViewportVector(Builder builder, JsArray data) { - return createViewportVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createViewportVector(Builder builder, Uint8Array data) { - return createViewportVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createViewportVector(Builder builder, double[] data) { - return createViewportVector(builder, Js.>uncheckedCast(data)); - } - - public static native double endBarrageSubscriptionRequest(Builder builder); - - public static native BarrageSubscriptionRequest getRootAsBarrageSubscriptionRequest( - ByteBuffer bb, BarrageSubscriptionRequest obj); - - public static native BarrageSubscriptionRequest getRootAsBarrageSubscriptionRequest( - ByteBuffer bb); - - public static native BarrageSubscriptionRequest getSizePrefixedRootAsBarrageSubscriptionRequest( - ByteBuffer bb, BarrageSubscriptionRequest obj); - - public static native BarrageSubscriptionRequest getSizePrefixedRootAsBarrageSubscriptionRequest( - ByteBuffer bb); - - public static native void startBarrageSubscriptionRequest(Builder builder); - - public static native void startColumnsVector(Builder builder, double numElems); - - public static native void startTicketVector(Builder builder, double numElems); - - public static native void startViewportVector(Builder builder, double numElems); - - public ByteBuffer bb; - public double bb_pos; - - public native BarrageSubscriptionRequest __init(double i, ByteBuffer bb); - - public native double columns(double index); - - public native Int8Array columnsArray(); - - public native double columnsLength(); - - public native boolean reverseViewport(); - - public native BarrageSubscriptionOptions subscriptionOptions(); - - public native BarrageSubscriptionOptions subscriptionOptions(BarrageSubscriptionOptions obj); - - public native double ticket(double index); - - public native Int8Array ticketArray(); - - public native double ticketLength(); - - public native double viewport(double index); - - public native Int8Array viewportArray(); - - public native double viewportLength(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageUpdateMetadata.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageUpdateMetadata.java deleted file mode 100644 index 3d9f4bd51df..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/BarrageUpdateMetadata.java +++ /dev/null @@ -1,575 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf; - -import elemental2.core.Int8Array; -import elemental2.core.JsArray; -import elemental2.core.Uint8Array; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Long; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven.barrage.flatbuf.Barrage_generated.io.deephaven.barrage.flatbuf.BarrageUpdateMetadata", - namespace = JsPackage.GLOBAL) -public class BarrageUpdateMetadata { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateAddedRowsIncludedVectorDataUnionType { - @JsOverlay - static BarrageUpdateMetadata.CreateAddedRowsIncludedVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateAddedRowsVectorDataUnionType { - @JsOverlay - static BarrageUpdateMetadata.CreateAddedRowsVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateEffectiveColumnSetVectorDataUnionType { - @JsOverlay - static BarrageUpdateMetadata.CreateEffectiveColumnSetVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateEffectiveViewportVectorDataUnionType { - @JsOverlay - static BarrageUpdateMetadata.CreateEffectiveViewportVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateRemovedRowsVectorDataUnionType { - @JsOverlay - static BarrageUpdateMetadata.CreateRemovedRowsVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateShiftDataVectorDataUnionType { - @JsOverlay - static BarrageUpdateMetadata.CreateShiftDataVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - public static native void addAddedRows(Builder builder, double addedRowsOffset); - - public static native void addAddedRowsIncluded(Builder builder, double addedRowsIncludedOffset); - - public static native void addEffectiveColumnSet(Builder builder, double effectiveColumnSetOffset); - - public static native void addEffectiveReverseViewport( - Builder builder, boolean effectiveReverseViewport); - - public static native void addEffectiveViewport(Builder builder, double effectiveViewportOffset); - - public static native void addFirstSeq(Builder builder, Long firstSeq); - - public static native void addIsSnapshot(Builder builder, boolean isSnapshot); - - public static native void addLastSeq(Builder builder, Long lastSeq); - - public static native void addModColumnNodes(Builder builder, double modColumnNodesOffset); - - public static native void addRemovedRows(Builder builder, double removedRowsOffset); - - public static native void addShiftData(Builder builder, double shiftDataOffset); - - @Deprecated - public static native double createAddedRowsIncludedVector( - Builder builder, BarrageUpdateMetadata.CreateAddedRowsIncludedVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createAddedRowsIncludedVector(Builder builder, Int8Array data) { - return createAddedRowsIncludedVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createAddedRowsIncludedVector(Builder builder, JsArray data) { - return createAddedRowsIncludedVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createAddedRowsIncludedVector(Builder builder, Uint8Array data) { - return createAddedRowsIncludedVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createAddedRowsIncludedVector(Builder builder, double[] data) { - return createAddedRowsIncludedVector(builder, Js.>uncheckedCast(data)); - } - - @Deprecated - public static native double createAddedRowsVector( - Builder builder, BarrageUpdateMetadata.CreateAddedRowsVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createAddedRowsVector(Builder builder, Int8Array data) { - return createAddedRowsVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createAddedRowsVector(Builder builder, JsArray data) { - return createAddedRowsVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createAddedRowsVector(Builder builder, Uint8Array data) { - return createAddedRowsVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createAddedRowsVector(Builder builder, double[] data) { - return createAddedRowsVector(builder, Js.>uncheckedCast(data)); - } - - public static native double createBarrageUpdateMetadata( - Builder builder, - Long firstSeq, - Long lastSeq, - boolean isSnapshot, - double effectiveViewportOffset, - boolean effectiveReverseViewport, - double effectiveColumnSetOffset, - double addedRowsOffset, - double removedRowsOffset, - double shiftDataOffset, - double addedRowsIncludedOffset, - double modColumnNodesOffset); - - @Deprecated - public static native double createEffectiveColumnSetVector( - Builder builder, BarrageUpdateMetadata.CreateEffectiveColumnSetVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createEffectiveColumnSetVector(Builder builder, Int8Array data) { - return createEffectiveColumnSetVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createEffectiveColumnSetVector(Builder builder, JsArray data) { - return createEffectiveColumnSetVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createEffectiveColumnSetVector(Builder builder, Uint8Array data) { - return createEffectiveColumnSetVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createEffectiveColumnSetVector(Builder builder, double[] data) { - return createEffectiveColumnSetVector(builder, Js.>uncheckedCast(data)); - } - - @Deprecated - public static native double createEffectiveViewportVector( - Builder builder, BarrageUpdateMetadata.CreateEffectiveViewportVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createEffectiveViewportVector(Builder builder, Int8Array data) { - return createEffectiveViewportVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createEffectiveViewportVector(Builder builder, JsArray data) { - return createEffectiveViewportVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createEffectiveViewportVector(Builder builder, Uint8Array data) { - return createEffectiveViewportVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createEffectiveViewportVector(Builder builder, double[] data) { - return createEffectiveViewportVector(builder, Js.>uncheckedCast(data)); - } - - public static native double createModColumnNodesVector(Builder builder, JsArray data); - - @JsOverlay - public static final double createModColumnNodesVector(Builder builder, double[] data) { - return createModColumnNodesVector(builder, Js.>uncheckedCast(data)); - } - - @Deprecated - public static native double createRemovedRowsVector( - Builder builder, BarrageUpdateMetadata.CreateRemovedRowsVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createRemovedRowsVector(Builder builder, Int8Array data) { - return createRemovedRowsVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createRemovedRowsVector(Builder builder, JsArray data) { - return createRemovedRowsVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createRemovedRowsVector(Builder builder, Uint8Array data) { - return createRemovedRowsVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createRemovedRowsVector(Builder builder, double[] data) { - return createRemovedRowsVector(builder, Js.>uncheckedCast(data)); - } - - @Deprecated - public static native double createShiftDataVector( - Builder builder, BarrageUpdateMetadata.CreateShiftDataVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createShiftDataVector(Builder builder, Int8Array data) { - return createShiftDataVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createShiftDataVector(Builder builder, JsArray data) { - return createShiftDataVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createShiftDataVector(Builder builder, Uint8Array data) { - return createShiftDataVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createShiftDataVector(Builder builder, double[] data) { - return createShiftDataVector(builder, Js.>uncheckedCast(data)); - } - - public static native double endBarrageUpdateMetadata(Builder builder); - - public static native BarrageUpdateMetadata getRootAsBarrageUpdateMetadata( - ByteBuffer bb, BarrageUpdateMetadata obj); - - public static native BarrageUpdateMetadata getRootAsBarrageUpdateMetadata(ByteBuffer bb); - - public static native BarrageUpdateMetadata getSizePrefixedRootAsBarrageUpdateMetadata( - ByteBuffer bb, BarrageUpdateMetadata obj); - - public static native BarrageUpdateMetadata getSizePrefixedRootAsBarrageUpdateMetadata( - ByteBuffer bb); - - public static native void startAddedRowsIncludedVector(Builder builder, double numElems); - - public static native void startAddedRowsVector(Builder builder, double numElems); - - public static native void startBarrageUpdateMetadata(Builder builder); - - public static native void startEffectiveColumnSetVector(Builder builder, double numElems); - - public static native void startEffectiveViewportVector(Builder builder, double numElems); - - public static native void startModColumnNodesVector(Builder builder, double numElems); - - public static native void startRemovedRowsVector(Builder builder, double numElems); - - public static native void startShiftDataVector(Builder builder, double numElems); - - public ByteBuffer bb; - public double bb_pos; - - public native BarrageUpdateMetadata __init(double i, ByteBuffer bb); - - public native double addedRows(double index); - - public native Int8Array addedRowsArray(); - - public native double addedRowsIncluded(double index); - - public native Int8Array addedRowsIncludedArray(); - - public native double addedRowsIncludedLength(); - - public native double addedRowsLength(); - - public native double effectiveColumnSet(double index); - - public native Int8Array effectiveColumnSetArray(); - - public native double effectiveColumnSetLength(); - - public native boolean effectiveReverseViewport(); - - public native double effectiveViewport(double index); - - public native Int8Array effectiveViewportArray(); - - public native double effectiveViewportLength(); - - public native Long firstSeq(); - - public native boolean isSnapshot(); - - public native Long lastSeq(); - - public native BarrageModColumnMetadata modColumnNodes(double index, BarrageModColumnMetadata obj); - - public native BarrageModColumnMetadata modColumnNodes(double index); - - public native double modColumnNodesLength(); - - public native double removedRows(double index); - - public native Int8Array removedRowsArray(); - - public native double removedRowsLength(); - - public native double shiftData(double index); - - public native Int8Array shiftDataArray(); - - public native double shiftDataLength(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/ColumnConversionMode.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/ColumnConversionMode.java deleted file mode 100644 index 30a750f44a6..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/ColumnConversionMode.java +++ /dev/null @@ -1,17 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf; - -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven.barrage.flatbuf.Barrage_generated.io.deephaven.barrage.flatbuf.ColumnConversionMode", - namespace = JsPackage.GLOBAL) -public class ColumnConversionMode { - public static int JavaSerialization, - Stringify, - ThrowError; -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/NewSessionRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/NewSessionRequest.java deleted file mode 100644 index a1f002c434a..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/NewSessionRequest.java +++ /dev/null @@ -1,125 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf; - -import elemental2.core.Int8Array; -import elemental2.core.JsArray; -import elemental2.core.Uint8Array; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven.barrage.flatbuf.Barrage_generated.io.deephaven.barrage.flatbuf.NewSessionRequest", - namespace = JsPackage.GLOBAL) -public class NewSessionRequest { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreatePayloadVectorDataUnionType { - @JsOverlay - static NewSessionRequest.CreatePayloadVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - public static native void addPayload(Builder builder, double payloadOffset); - - public static native void addProtocolVersion(Builder builder, double protocolVersion); - - public static native double createNewSessionRequest( - Builder builder, double protocolVersion, double payloadOffset); - - @Deprecated - public static native double createPayloadVector( - Builder builder, NewSessionRequest.CreatePayloadVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createPayloadVector(Builder builder, Int8Array data) { - return createPayloadVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createPayloadVector(Builder builder, JsArray data) { - return createPayloadVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createPayloadVector(Builder builder, Uint8Array data) { - return createPayloadVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createPayloadVector(Builder builder, double[] data) { - return createPayloadVector(builder, Js.>uncheckedCast(data)); - } - - public static native double endNewSessionRequest(Builder builder); - - public static native NewSessionRequest getRootAsNewSessionRequest( - ByteBuffer bb, NewSessionRequest obj); - - public static native NewSessionRequest getRootAsNewSessionRequest(ByteBuffer bb); - - public static native NewSessionRequest getSizePrefixedRootAsNewSessionRequest( - ByteBuffer bb, NewSessionRequest obj); - - public static native NewSessionRequest getSizePrefixedRootAsNewSessionRequest(ByteBuffer bb); - - public static native void startNewSessionRequest(Builder builder); - - public static native void startPayloadVector(Builder builder, double numElems); - - public ByteBuffer bb; - public double bb_pos; - - public native NewSessionRequest __init(double i, ByteBuffer bb); - - public native double payload(double index); - - public native Int8Array payloadArray(); - - public native double payloadLength(); - - public native double protocolVersion(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/RefreshSessionRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/RefreshSessionRequest.java deleted file mode 100644 index 3fcc2501206..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/RefreshSessionRequest.java +++ /dev/null @@ -1,121 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf; - -import elemental2.core.Int8Array; -import elemental2.core.JsArray; -import elemental2.core.Uint8Array; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven.barrage.flatbuf.Barrage_generated.io.deephaven.barrage.flatbuf.RefreshSessionRequest", - namespace = JsPackage.GLOBAL) -public class RefreshSessionRequest { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateSessionVectorDataUnionType { - @JsOverlay - static RefreshSessionRequest.CreateSessionVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - public static native void addSession(Builder builder, double sessionOffset); - - public static native double createRefreshSessionRequest(Builder builder, double sessionOffset); - - @Deprecated - public static native double createSessionVector( - Builder builder, RefreshSessionRequest.CreateSessionVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createSessionVector(Builder builder, Int8Array data) { - return createSessionVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createSessionVector(Builder builder, JsArray data) { - return createSessionVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createSessionVector(Builder builder, Uint8Array data) { - return createSessionVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createSessionVector(Builder builder, double[] data) { - return createSessionVector(builder, Js.>uncheckedCast(data)); - } - - public static native double endRefreshSessionRequest(Builder builder); - - public static native RefreshSessionRequest getRootAsRefreshSessionRequest( - ByteBuffer bb, RefreshSessionRequest obj); - - public static native RefreshSessionRequest getRootAsRefreshSessionRequest(ByteBuffer bb); - - public static native RefreshSessionRequest getSizePrefixedRootAsRefreshSessionRequest( - ByteBuffer bb, RefreshSessionRequest obj); - - public static native RefreshSessionRequest getSizePrefixedRootAsRefreshSessionRequest( - ByteBuffer bb); - - public static native void startRefreshSessionRequest(Builder builder); - - public static native void startSessionVector(Builder builder, double numElems); - - public ByteBuffer bb; - public double bb_pos; - - public native RefreshSessionRequest __init(double i, ByteBuffer bb); - - public native double session(double index); - - public native Int8Array sessionArray(); - - public native double sessionLength(); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/SessionInfoResponse.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/SessionInfoResponse.java deleted file mode 100644 index 39658a0f27b..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven/barrage/flatbuf/barrage_generated/io/deephaven/barrage/flatbuf/SessionInfoResponse.java +++ /dev/null @@ -1,211 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven.barrage.flatbuf.barrage_generated.io.deephaven.barrage.flatbuf; - -import elemental2.core.Int8Array; -import elemental2.core.JsArray; -import elemental2.core.Uint8Array; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Builder; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer; -import io.deephaven.javascript.proto.dhinternal.flatbuffers.Long; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven.barrage.flatbuf.Barrage_generated.io.deephaven.barrage.flatbuf.SessionInfoResponse", - namespace = JsPackage.GLOBAL) -public class SessionInfoResponse { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateMetadataHeaderVectorDataUnionType { - @JsOverlay - static SessionInfoResponse.CreateMetadataHeaderVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface CreateSessionTokenVectorDataUnionType { - @JsOverlay - static SessionInfoResponse.CreateSessionTokenVectorDataUnionType of(Object o) { - return Js.cast(o); - } - - @JsOverlay - default Int8Array asInt8Array() { - return Js.cast(this); - } - - @JsOverlay - default JsArray asJsArray() { - return Js.cast(this); - } - - @JsOverlay - default Uint8Array asUint8Array() { - return Js.cast(this); - } - - @JsOverlay - default boolean isInt8Array() { - return (Object) this instanceof Int8Array; - } - - @JsOverlay - default boolean isJsArray() { - return (Object) this instanceof JsArray; - } - - @JsOverlay - default boolean isUint8Array() { - return (Object) this instanceof Uint8Array; - } - } - - public static native void addMetadataHeader(Builder builder, double metadataHeaderOffset); - - public static native void addSessionToken(Builder builder, double sessionTokenOffset); - - public static native void addTokenRefreshDeadlineMs(Builder builder, Long tokenRefreshDeadlineMs); - - @Deprecated - public static native double createMetadataHeaderVector( - Builder builder, SessionInfoResponse.CreateMetadataHeaderVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createMetadataHeaderVector(Builder builder, Int8Array data) { - return createMetadataHeaderVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createMetadataHeaderVector(Builder builder, JsArray data) { - return createMetadataHeaderVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createMetadataHeaderVector(Builder builder, Uint8Array data) { - return createMetadataHeaderVector( - builder, - Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createMetadataHeaderVector(Builder builder, double[] data) { - return createMetadataHeaderVector(builder, Js.>uncheckedCast(data)); - } - - public static native double createSessionInfoResponse( - Builder builder, - double metadataHeaderOffset, - double sessionTokenOffset, - Long tokenRefreshDeadlineMs); - - @Deprecated - public static native double createSessionTokenVector( - Builder builder, SessionInfoResponse.CreateSessionTokenVectorDataUnionType data); - - @JsOverlay - @Deprecated - public static final double createSessionTokenVector(Builder builder, Int8Array data) { - return createSessionTokenVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createSessionTokenVector(Builder builder, JsArray data) { - return createSessionTokenVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createSessionTokenVector(Builder builder, Uint8Array data) { - return createSessionTokenVector( - builder, Js.uncheckedCast(data)); - } - - @JsOverlay - @Deprecated - public static final double createSessionTokenVector(Builder builder, double[] data) { - return createSessionTokenVector(builder, Js.>uncheckedCast(data)); - } - - public static native double endSessionInfoResponse(Builder builder); - - public static native SessionInfoResponse getRootAsSessionInfoResponse( - ByteBuffer bb, SessionInfoResponse obj); - - public static native SessionInfoResponse getRootAsSessionInfoResponse(ByteBuffer bb); - - public static native SessionInfoResponse getSizePrefixedRootAsSessionInfoResponse( - ByteBuffer bb, SessionInfoResponse obj); - - public static native SessionInfoResponse getSizePrefixedRootAsSessionInfoResponse(ByteBuffer bb); - - public static native void startMetadataHeaderVector(Builder builder, double numElems); - - public static native void startSessionInfoResponse(Builder builder); - - public static native void startSessionTokenVector(Builder builder, double numElems); - - public ByteBuffer bb; - public double bb_pos; - - public native SessionInfoResponse __init(double i, ByteBuffer bb); - - public native double metadataHeader(double index); - - public native Int8Array metadataHeaderArray(); - - public native double metadataHeaderLength(); - - public native double sessionToken(double index); - - public native Int8Array sessionTokenArray(); - - public native double sessionTokenLength(); - - public native Long tokenRefreshDeadlineMs(); -} diff --git a/web/client-ui/Dockerfile b/web/client-ui/Dockerfile index 4ea68fceda6..9eff159af4b 100644 --- a/web/client-ui/Dockerfile +++ b/web/client-ui/Dockerfile @@ -2,10 +2,10 @@ FROM deephaven/node:local-build WORKDIR /usr/src/app # Most of the time, these versions are the same, except in cases where a patch only affects one of the packages -ARG WEB_VERSION=0.89.0 -ARG GRID_VERSION=0.89.0 -ARG CHART_VERSION=0.89.0 -ARG WIDGET_VERSION=0.89.0 +ARG WEB_VERSION=0.93.0 +ARG GRID_VERSION=0.93.0 +ARG CHART_VERSION=0.93.0 +ARG WIDGET_VERSION=0.93.0 # Pull in the published code-studio package from npmjs and extract is RUN set -eux; \ diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/ColumnStatistics.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/ColumnStatistics.java deleted file mode 100644 index 7a72d3806d5..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/ColumnStatistics.java +++ /dev/null @@ -1,169 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data; - -import java.io.Serializable; -import java.util.Arrays; - -/** - * A DTO containing the result data from a call to GenerateComparableStatsFunction or GenerateNumericalStatsFunction - */ -public class ColumnStatistics implements Serializable { - public enum ColumnType { - NUMERIC, COMPARABLE, DATETIME, NON_COMPARABLE, - } - - private ColumnType type; - private long size; - private long count; - - // Data from a GenerateComparableStatsFunction - private int numUnique; - private String[] uniqueKeys; - private long[] uniqueValues; - - // Data from a GenerateNumericalStatsFunction - private double sum; - private double absSum; - private double min; - private double max; - private double absMin; - private double absMax; - - // Data from a GenerateDateTimeStatsFunction - private long minDateTime; - private long maxDateTime; - - public ColumnType getType() { - return type; - } - - public void setType(ColumnType type) { - this.type = type; - } - - public long getSize() { - return size; - } - - public void setSize(final long size) { - this.size = size; - } - - public long getCount() { - return count; - } - - public void setCount(long count) { - this.count = count; - } - - public int getNumUnique() { - return numUnique; - } - - public void setNumUnique(int numUnique) { - this.numUnique = numUnique; - } - - public String[] getUniqueKeys() { - return uniqueKeys; - } - - public void setUniqueKeys(String[] uniqueKeys) { - this.uniqueKeys = uniqueKeys; - } - - public long[] getUniqueValues() { - return uniqueValues; - } - - public void setUniqueValues(long[] uniqueValues) { - this.uniqueValues = uniqueValues; - } - - public double getSum() { - return sum; - } - - public void setSum(double sum) { - this.sum = sum; - } - - public double getAbsSum() { - return absSum; - } - - public void setAbsSum(double absSum) { - this.absSum = absSum; - } - - public double getMin() { - return min; - } - - public void setMin(double min) { - this.min = min; - } - - public double getMax() { - return max; - } - - public void setMax(double max) { - this.max = max; - } - - public double getAbsMin() { - return absMin; - } - - public void setAbsMin(double absMin) { - this.absMin = absMin; - } - - public double getAbsMax() { - return absMax; - } - - public void setAbsMax(double absMax) { - this.absMax = absMax; - } - - public long getMinDateTime() { - return minDateTime; - } - - public void setMinDateTime(final long minDateTime) { - this.minDateTime = minDateTime; - } - - public long getMaxDateTime() { - return maxDateTime; - } - - public void setMaxDateTime(final long maxDateTime) { - this.maxDateTime = maxDateTime; - } - - @Override - public String toString() { - return "ColumnStatistics{" + - "type=" + type + - ", size=" + size + - ", count=" + count + - ", numUnique=" + numUnique + - ", uniqueKeys=" + Arrays.toString(uniqueKeys) + - ", uniqueValues=" + Arrays.toString(uniqueValues) + - ", sum=" + sum + - ", absSum=" + absSum + - ", min=" + min + - ", max=" + max + - ", absMin=" + absMin + - ", absMax=" + absMax + - ", minDateTime=" + minDateTime + - ", maxDateTime=" + maxDateTime + - '}'; - } -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/DeltaUpdates.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/DeltaUpdates.java deleted file mode 100644 index f6e3e59a753..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/DeltaUpdates.java +++ /dev/null @@ -1,174 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data; - -import io.deephaven.web.shared.data.columns.ColumnData; - -import java.io.Serializable; - -public class DeltaUpdates implements Serializable { - public static class ColumnAdditions implements Serializable { - private int columnIndex; - private ColumnData values; - - public ColumnAdditions() {} - - public ColumnAdditions(int colIndex, ColumnData values) { - setColumnIndex(colIndex); - setValues(values); - } - - public int getColumnIndex() { - return columnIndex; - } - - public void setColumnIndex(final int columnIndex) { - this.columnIndex = columnIndex; - } - - public ColumnData getValues() { - return values; - } - - public void setValues(final ColumnData values) { - this.values = values; - } - } - public static class ColumnModifications implements Serializable { - private int columnIndex; - private RangeSet rowsIncluded; - public ColumnData values; - - public ColumnModifications() {} - - public ColumnModifications(int columnIndex, RangeSet includedModifications, ColumnData columnData) { - setColumnIndex(columnIndex); - setRowsIncluded(includedModifications); - setValues(columnData); - } - - public int getColumnIndex() { - return columnIndex; - } - - public void setColumnIndex(final int columnIndex) { - this.columnIndex = columnIndex; - } - - public RangeSet getRowsIncluded() { - return rowsIncluded; - } - - public void setRowsIncluded(final RangeSet rowsIncluded) { - this.rowsIncluded = rowsIncluded; - } - - public ColumnData getValues() { - return values; - } - - public void setValues(final ColumnData values) { - this.values = values; - } - } - - private long deltaSequence; - private long firstStep; - private long lastStep; - - private RangeSet added; - private RangeSet removed; - - private ShiftedRange[] shiftedRanges; - - private RangeSet includedAdditions; - - private ColumnAdditions[] serializedAdditions; - private ColumnModifications[] serializedModifications; - - public DeltaUpdates() {} - - public DeltaUpdates(RangeSet added, RangeSet removed, ShiftedRange[] shifted, RangeSet includedAdditions, - ColumnAdditions[] addedColumnData, ColumnModifications[] modifiedColumnData) { - setAdded(added); - setRemoved(removed); - setShiftedRanges(shifted); - setIncludedAdditions(includedAdditions); - setSerializedAdditions(addedColumnData); - setSerializedModifications(modifiedColumnData); - } - - - public long getDeltaSequence() { - return deltaSequence; - } - - public void setDeltaSequence(final long deltaSequence) { - this.deltaSequence = deltaSequence; - } - - public long getFirstStep() { - return firstStep; - } - - public void setFirstStep(final long firstStep) { - this.firstStep = firstStep; - } - - public long getLastStep() { - return lastStep; - } - - public void setLastStep(final long lastStep) { - this.lastStep = lastStep; - } - - public RangeSet getAdded() { - return added; - } - - public void setAdded(final RangeSet added) { - this.added = added; - } - - public RangeSet getRemoved() { - return removed; - } - - public void setRemoved(final RangeSet removed) { - this.removed = removed; - } - - public ShiftedRange[] getShiftedRanges() { - return shiftedRanges; - } - - public void setShiftedRanges(final ShiftedRange[] shiftedRanges) { - this.shiftedRanges = shiftedRanges; - } - - public RangeSet getIncludedAdditions() { - return includedAdditions; - } - - public void setIncludedAdditions(final RangeSet includedAdditions) { - this.includedAdditions = includedAdditions; - } - - public ColumnAdditions[] getSerializedAdditions() { - return serializedAdditions; - } - - public void setSerializedAdditions(final ColumnAdditions[] serializedAdditions) { - this.serializedAdditions = serializedAdditions; - } - - public ColumnModifications[] getSerializedModifications() { - return serializedModifications; - } - - public void setSerializedModifications(final ColumnModifications[] serializedModifications) { - this.serializedModifications = serializedModifications; - } -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/LocalDate.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/LocalDate.java deleted file mode 100644 index 69a0cc8cc71..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/LocalDate.java +++ /dev/null @@ -1,47 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data; - -import java.io.Serializable; - -/** - * A simple container for serializing LocalDate values. This should be better for serialization than java.time.LocalDate - * since we use bytes for month and day, and is compatible with GWT (java.time is not available in GWT). - */ -public class LocalDate implements Serializable { - private int year; - private byte monthValue, dayOfMonth; - - public LocalDate(int year, byte monthValue, byte dayOfMonth) { - this.year = year; - this.monthValue = monthValue; - this.dayOfMonth = dayOfMonth; - } - - public LocalDate() {} - - public int getYear() { - return year; - } - - public void setYear(int year) { - this.year = year; - } - - public byte getMonthValue() { - return monthValue; - } - - public void setMonthValue(byte monthValue) { - this.monthValue = monthValue; - } - - public byte getDayOfMonth() { - return dayOfMonth; - } - - public void setDayOfMonth(byte dayOfMonth) { - this.dayOfMonth = dayOfMonth; - } -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/LocalTime.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/LocalTime.java deleted file mode 100644 index 74ea3b1cfb3..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/LocalTime.java +++ /dev/null @@ -1,57 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data; - -import java.io.Serializable; - -/** - * A simple container for serializing local time values. This should be better for serialization than - * java.time.LocalTime since we use bytes for hour, minute and second, and is compatible with GWT (java.time is not - * available in GWT). - */ -public class LocalTime implements Serializable { - private byte hour, minute, second; - private int nano; - - public LocalTime(byte hour, byte minute, byte second, int nano) { - this.hour = hour; - this.minute = minute; - this.second = second; - this.nano = nano; - } - - public LocalTime() {} - - public byte getHour() { - return hour; - } - - public void setHour(byte hour) { - this.hour = hour; - } - - public byte getMinute() { - return minute; - } - - public void setMinute(byte minute) { - this.minute = minute; - } - - public byte getSecond() { - return second; - } - - public void setSecond(byte second) { - this.second = second; - } - - public int getNano() { - return nano; - } - - public void setNano(int nano) { - this.nano = nano; - } -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/Range.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/Range.java index 16ef88a4e60..87144a989f1 100644 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/Range.java +++ b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/Range.java @@ -4,26 +4,17 @@ package io.deephaven.web.shared.data; import javax.annotation.Nonnull; -import java.io.Serializable; /** * Describes a contiguous range of at least one item. Equals/hashcode compare both start and end, but comparing Range * instances will compare only by start - the overlap(Range) method should be used to see if two ranges share at least * one item. */ -public class Range implements Serializable, Comparable { - private long first; - private long last; - - // serialization - Range() { - this(0, 0); - } +public class Range implements Comparable { + private final long first; + private final long last; public Range(long first, long last) { - if (first > last) { - throw new IllegalStateException(first + " > " + last); - } this.first = first; this.last = last; } @@ -36,14 +27,6 @@ public long getLast() { return last; } - void setFirst(long first) { - this.first = first; - } - - void setLast(long last) { - this.last = last; - } - @Override public int compareTo(@Nonnull Range o) { return Long.compare(first, o.first); @@ -78,13 +61,10 @@ public Range[] minus(Range range) { // otherwise either the subtracted section's start is within our range _or_ its end is within our range, // and we can use that to only produce the one range we need to return if (range.first <= first) { - assert range.last >= first : "removed range expected to not end before existing range"; return new Range[] { new Range(range.last + 1, last) }; } else { - assert range.last >= last : "removed range expected to end by the end of the existing range"; - assert range.first <= last : "removed range expected to start before existing range"; return new Range[] { new Range(first, range.first - 1) }; @@ -123,4 +103,8 @@ public String toString() { ", last=" + last + '}'; } + + public Range shift(long delta) { + return new Range(first + delta, last + delta); + } } diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/RangeSet.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/RangeSet.java index c782582a425..3e77c9ccec8 100644 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/RangeSet.java +++ b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/RangeSet.java @@ -3,9 +3,11 @@ // package io.deephaven.web.shared.data; -import java.io.Serializable; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Iterator; +import java.util.List; import java.util.PrimitiveIterator; import java.util.stream.LongStream; @@ -14,7 +16,7 @@ * Iteration protocol, but for now has one method which returns an iterator, and also supports querying the size. * Additionally, we may add support for creating RangeSet objects to better serve some use cases. */ -public class RangeSet implements Serializable { +public class RangeSet { public static RangeSet empty() { return new RangeSet(); @@ -37,210 +39,356 @@ public static RangeSet ofItems(long... items) { return rangeSet; } - public static RangeSet fromSortedRanges(Range[] sortedRanges) { - assert orderedAndNonOverlapping(sortedRanges) : Arrays.toString(sortedRanges); + public static RangeSet fromSortedRanges(List sortedRanges) { + assertOrderedAndNonOverlapping(sortedRanges.toArray(new Range[0])); RangeSet rangeSet = new RangeSet(); rangeSet.sortedRanges = sortedRanges; return rangeSet; } - private static boolean orderedAndNonOverlapping(Range[] sortedRanges) { + public static RangeSet fromSortedRanges(Range[] sortedRanges) { + assertOrderedAndNonOverlapping(sortedRanges); + RangeSet rangeSet = new RangeSet(); + rangeSet.sortedRanges.addAll(Arrays.asList(sortedRanges)); + return rangeSet; + } + + private static void assertOrderedAndNonOverlapping(Range[] sortedRanges) { long lastSeen = -1; for (int i = 0; i < sortedRanges.length; i++) { - if (lastSeen >= sortedRanges[i].getFirst()) { - return false; - } + assert lastSeen < sortedRanges[i].getFirst() || lastSeen == -1 + : sortedRanges[i - 1] + " came before " + sortedRanges[i] + " (index=" + i + ")"; lastSeen = sortedRanges[i].getLast(); } - return true; } - private Range[] sortedRanges = new Range[0]; + private List sortedRanges = new ArrayList<>(); + + private int firstWrongCacheEntry = 0; + private long[] cardinality = new long[0]; + + public void addRangeSet(RangeSet rangeSet) { + if (isEmpty() && !rangeSet.isEmpty()) { + sortedRanges = new ArrayList<>(rangeSet.sortedRanges); + poisonCache(0); + } else if (!rangeSet.isEmpty()) { + RangeAccumulator newRanges = new RangeAccumulator(); + Iterator rangeIterator = sortedRanges.iterator(); + Iterator addIterator = rangeSet.sortedRanges.iterator(); + + Range toCheck = rangeIterator.next(); + Range toAdd = addIterator.next(); + while (true) { + if (toCheck.getLast() < toAdd.getFirst()) { + newRanges.appendRange(toCheck); + if (!rangeIterator.hasNext()) { + toCheck = null; + break; + } + toCheck = rangeIterator.next(); + } else if (toCheck.getFirst() > toAdd.getLast()) { + newRanges.appendRange(toAdd); + + if (!addIterator.hasNext()) { + toAdd = null; + break; + } + toAdd = addIterator.next(); + } else { + Range overlap = toCheck.overlap(toAdd); + assert overlap != null; + newRanges.appendRange(overlap); + + if (!rangeIterator.hasNext()) { + toCheck = null; + break; + } + toCheck = rangeIterator.next(); + + if (!addIterator.hasNext()) { + toAdd = null; + break; + } + toAdd = addIterator.next(); + } + } - public void addRange(Range range) { - // if empty, add as the only entry - if (sortedRanges.length == 0) { - sortedRanges = new Range[] {range}; - return; - } - // if one other entry, test if before, after, or overlapping - if (sortedRanges.length == 1) { - Range existing = sortedRanges[0]; - Range overlap = range.overlap(existing); - if (overlap != null) { - sortedRanges = new Range[] {overlap}; - } else if (existing.compareTo(range) < 0) { - sortedRanges = new Range[] {existing, range}; + // Grab remaining ranges + if (toCheck != null) { + assert toAdd == null; + newRanges.appendRange(toCheck); + while (rangeIterator.hasNext()) { + newRanges.appendRange(rangeIterator.next()); + } } else { - assert existing.compareTo(range) > 0; - sortedRanges = new Range[] {range, existing}; + assert toAdd != null; + newRanges.appendRange(toAdd); + while (addIterator.hasNext()) { + newRanges.appendRange(addIterator.next()); + } } - return; + + this.sortedRanges = newRanges.build(); + poisonCache(0); } + } - // if more than one other entry, binarySearch to find before and after entry, and test both for overlapping - int index = Arrays.binarySearch(sortedRanges, range); - if (index >= 0) { + public void addRange(Range range) { + addRangeSet(RangeSet.fromSortedRanges(Collections.singletonList(range))); + } - // starting with that item, check to see if each following item is part of the existing range - // we know that no range before it will need to be considered, since the set should previously - // have been broken into non-contiguous ranges - Range merged = range; - int end = sortedRanges.length - 1; - for (int i = index; i < sortedRanges.length; i++) { - Range existing = sortedRanges[i]; - // there is an item with the same start, either new item falls within it, or should replace it - Range overlap = existing.overlap(merged); - - if (overlap == null) { - // index before this one is the last item to be replaced - end = i - 1; + public void removeRangeSet(RangeSet rangeSet) { + if (isEmpty() || rangeSet.isEmpty()) { + return; + } + + RangeAccumulator newRanges = new RangeAccumulator(); + RangeIterator rangeIterator = new RangeIterator(sortedRanges); + Iterator removeIterator = rangeSet.sortedRanges.iterator(); + + Range toCheck = rangeIterator.next(); + Range toRemove = removeIterator.next(); + while (true) { + if (toCheck.getLast() < toRemove.getFirst()) { + newRanges.appendRange(toCheck); + if (!rangeIterator.hasNext()) { + toCheck = null; break; } - if (overlap.equals(existing)) { - // the entire range to be added existed within an existing range, we're done - return; + toCheck = rangeIterator.next(); + } else if (toCheck.getFirst() > toRemove.getLast()) { + if (!removeIterator.hasNext()) { + break; + } + toRemove = removeIterator.next(); + } else { + Range[] remaining = toCheck.minus(toRemove); + if (remaining.length == 0) { + // entire range removed, advance to the next range to check + if (!rangeIterator.hasNext()) { + toCheck = null; + break; + } + toCheck = rangeIterator.next(); + } else if (remaining.length == 1) { + Range remainingRange = remaining[0]; + if (remainingRange.compareTo(toCheck) > 0) { + // unremoved range still needs to be checked + rangeIterator.advanceInCurrentRangeToKey(remainingRange.getFirst()); + toCheck = rangeIterator.next(); + + if (!removeIterator.hasNext()) { + break; + } + toRemove = removeIterator.next(); + } else { + // keep the leading, remaining section + newRanges.appendRange(remainingRange); + + // look at the next range + if (!rangeIterator.hasNext()) { + toCheck = null; + break; + } + toCheck = rangeIterator.next(); + } + } else { + assert remaining.length == 2; + newRanges.appendRange(remaining[0]); + + rangeIterator.advanceInCurrentRangeToKey(remaining[1].getFirst()); + toCheck = rangeIterator.next(); + + if (!removeIterator.hasNext()) { + break; + } + toRemove = removeIterator.next(); } - - // grow the region used for replacing - merged = overlap; - } - // splice out [index, end] items, replacing with the newly grown overlap object (may be the same - // size, and only replacing one item) - int newLength = sortedRanges.length - (end - index); - Range[] newArray = new Range[newLength]; - if (index > 0) { - System.arraycopy(sortedRanges, 0, newArray, 0, index); } - newArray[index] = merged; - if (end < sortedRanges.length - 1) { - System.arraycopy(sortedRanges, end + 1, newArray, index + 1, sortedRanges.length - 1 - end); + } + + // Grab remaining ranges + if (toCheck != null) { + newRanges.appendRange(toCheck); + while (rangeIterator.hasNext()) { + newRanges.appendRange(rangeIterator.next()); } - sortedRanges = newArray; - } else { - int proposedIndex = -(index) - 1; - Range merged = range; - // test the item before the proposed location (if any), try to merge it - if (proposedIndex > 0) { - Range before = sortedRanges[proposedIndex - 1]; - Range overlap = before.overlap(merged); + } + + this.sortedRanges = newRanges.build(); + poisonCache(0); + } + + public void removeRange(Range range) { + removeRangeSet(RangeSet.fromSortedRanges(Collections.singletonList(range))); + } + + public void clear() { + sortedRanges.clear(); + poisonCache(0); + } + + private static class RangeAccumulator { + private final List replacement = new ArrayList<>(); + + public void appendRange(Range range) { + if (!replacement.isEmpty()) { + Range lastSeen = replacement.get(replacement.size() - 1); + Range overlap = lastSeen.overlap(range); if (overlap != null) { - // replace the range that we are merging, and start the slice here instead - merged = overlap; - proposedIndex--; - // TODO this will make the loop start here, considering this item twice. not ideal, but not a big - // deal either - } - } - // "end" represents the last item that needs to be merged in to the newly added item. if no items are to be - // merged in, then end will be proposedIndex-1, meaning nothing gets merged in, and the array will grow - // instead of shrinking. - // if we never find an item we cannot merge with, the end of the replaced range is the last item of the old - // array, which could result in the new array having as little as only 1 item - int end = sortedRanges.length - 1; - // until we quit finding matches, test subsequent items - for (int i = proposedIndex; i < sortedRanges.length; i++) { - Range existing = sortedRanges[i]; - Range overlap = existing.overlap(merged); - if (overlap == null) { - // stop at the item before this one - end = i - 1; - break; + replacement.set(replacement.size() - 1, overlap); + } else { + replacement.add(range); } - merged = overlap; - } - int newLength = sortedRanges.length - (end - proposedIndex); - assert newLength > 0 && newLength <= sortedRanges.length + 1; - Range[] newArray = new Range[newLength]; - if (proposedIndex > 0) { - System.arraycopy(sortedRanges, 0, newArray, 0, proposedIndex); - } - newArray[proposedIndex] = merged; - if (end < sortedRanges.length - 1) { - System.arraycopy(sortedRanges, end + 1, newArray, proposedIndex + 1, sortedRanges.length - (end + 1)); + } else { + replacement.add(range); } - sortedRanges = newArray; + } + + public void appendRanges(List ranges) { + appendRange(ranges.get(0)); + replacement.addAll(ranges.subList(0, ranges.size() - 1)); + } + + public void appendRanges(List ranges, long firstItemSubindex) { + Range first = ranges.get(0); + appendRange(new Range(first.getFirst() + firstItemSubindex, first.getLast())); + replacement.addAll(ranges.subList(0, ranges.size() - 1)); + } + + public List build() { + return replacement; } } - public void removeRange(Range range) { - // if empty, nothing to do - if (sortedRanges.length == 0) { - return; + private static class RangeIterator { + private int index = -1; + private final List ranges; + private long key = 0; + + private RangeIterator(List ranges) { + this.ranges = ranges; } - // search the sorted list of ranges and find where the current range starts. two case here when using - // binarySearch, either the removed range starts in the same place as an existing range starts, or - // it starts before an item (and so we check the item before and the item after) - int index = Arrays.binarySearch(sortedRanges, range); - if (index < 0) { - // adjusted index notes where the item would be if it were added, minus _one more_ to see if - // it overlaps the item before it. To compute "the position where the new item belongs", we - // would do (-index - 1), so to examine one item prior to that we'll subtract one more. Then, - // to confirm that we are inserting in a valid position, take the max of that value and zero. - index = Math.max(0, -index - 2); - } - - int beforeCount = -1; - int toRemove = 0; - for (; index < sortedRanges.length; index++) { - Range toCheck = sortedRanges[index]; - if (toCheck.getFirst() > range.getLast()) { - break;// done, this is entirely after the range we're removing - } - if (toCheck.getLast() < range.getFirst()) { - continue;// skip, we don't overlap at all yet - } - Range[] remaining = toCheck.minus(range); - assert remaining != null : "Only early ranges are allowed to not match at all"; - - if (remaining.length == 2) { - // Removed region is entirely within the range we are checking: - // Splice in the one extra item and we're done - this entry - // both started before and ended after the removed section, - // so we don't even "break", we just return - assert toCheck.getFirst() < range.getFirst() : "Expected " + range + " to start after " + toCheck; - assert toCheck.getLast() > range.getLast() : "Expected " + range + " to end after " + toCheck; - assert toRemove == 0 && beforeCount == -1 - : "Expected that no previous items in the RangeSet had been removed toRemove=" + toRemove - + ", beforeCount=" + beforeCount; - - Range[] replacement = new Range[sortedRanges.length + 1]; - if (index > 0) { - System.arraycopy(sortedRanges, 0, replacement, 0, index); - } - replacement[index] = remaining[0]; - replacement[index + 1] = remaining[1]; - System.arraycopy(sortedRanges, index + 1, replacement, index + 2, sortedRanges.length - (index + 1)); + public void advanceInCurrentRangeToKey(long key) { + assert key != 0; + this.key = key; + } + + public boolean hasNext() { + return key == -1 || index < ranges.size() - 1; + } - sortedRanges = replacement; + public Range next() { + if (key != 0) { + Range r = ranges.get(index); + assert key > r.getFirst() && key <= r.getLast(); + r = new Range(key, r.getLast()); + key = 0; - return; + return r; } - if (remaining.length == 1) { - // swap shortened item and move on - sortedRanges[index] = remaining[0]; - } else { - assert remaining.length == 0 : "Array contains a surprising number of items: " + remaining.length; + return ranges.get(++index); + } + + } - // splice out this item as nothing exists here any more and move on - if (toRemove == 0) { - beforeCount = index; + public void applyShifts(ShiftedRange[] shiftedRanges) { + if (shiftedRanges.length == 0 || isEmpty()) { + return; + } + RangeAccumulator newRanges = new RangeAccumulator(); + RangeIterator rangeIterator = new RangeIterator(sortedRanges); + Iterator shiftIterator = Arrays.asList(shiftedRanges).iterator(); + Range toCheck = rangeIterator.next(); + ShiftedRange shiftedRange = shiftIterator.next(); + do { + if (toCheck.getLast() < shiftedRange.getRange().getFirst()) { + // leave this range alone, the range to shift is after it + newRanges.appendRange(toCheck); + if (!rangeIterator.hasNext()) { + toCheck = null; + break; + } + toCheck = rangeIterator.next(); + } else if (toCheck.getFirst() > shiftedRange.getRange().getLast()) { + // skip the rest of this shift, the next range is after it + if (!shiftIterator.hasNext()) { + break; + } + shiftedRange = shiftIterator.next(); + } else { + Range[] remaining = toCheck.minus(shiftedRange.getRange()); + if (remaining.length == 0) { + // entire range shifted + newRanges.appendRange(toCheck.shift(shiftedRange.getDelta())); + if (!rangeIterator.hasNext()) { + toCheck = null; + break; + } + toCheck = rangeIterator.next(); + } else if (remaining.length == 1) { + Range remainingRange = remaining[0]; + + Range[] complimentArr = toCheck.minus(remainingRange); + assert complimentArr.length == 1; + Range compliment = complimentArr[0]; + if (remainingRange.compareTo(toCheck) > 0) { + // shift the compliment + newRanges.appendRange(compliment.shift(shiftedRange.getDelta())); + + // rest of the range still needs to be checked + rangeIterator.advanceInCurrentRangeToKey(remainingRange.getFirst()); + toCheck = rangeIterator.next(); + + // shift is consumed, move to the next one + if (!shiftIterator.hasNext()) { + break; + } + shiftedRange = shiftIterator.next(); + } else { + // keep the remaining section + newRanges.appendRange(remainingRange); + // leftovers after, shift the compliment + newRanges.appendRange(compliment.shift(shiftedRange.getDelta())); + + // look at the next range + if (!rangeIterator.hasNext()) { + toCheck = null; + break; + } + toCheck = rangeIterator.next(); + } + } else { + assert remaining.length == 2; + // We matched the entire shift range, plus a prefix and suffix + // First append the before section + newRanges.appendRange(remaining[0]); + // Then the entire shift range + newRanges.appendRange(shiftedRange.getResultRange()); + + // Visit the rest of the range next + rangeIterator.advanceInCurrentRangeToKey(remaining[1].getFirst()); + toCheck = rangeIterator.next(); + + if (!shiftIterator.hasNext()) { + break; + } + shiftedRange = shiftIterator.next(); } - toRemove++; } + } while (true); + // Grab remaining ranges + if (toCheck != null) { + newRanges.appendRange(toCheck); + while (rangeIterator.hasNext()) { + newRanges.appendRange(rangeIterator.next()); + } } - if (toRemove > 0) { - Range[] replacement = new Range[sortedRanges.length - toRemove]; - System.arraycopy(sortedRanges, 0, replacement, 0, beforeCount); - System.arraycopy(sortedRanges, beforeCount + toRemove, replacement, beforeCount, - sortedRanges.length - beforeCount - toRemove); - sortedRanges = replacement; - } else { - assert beforeCount == -1 : "No items to remove, but beforeCount set?"; - } + sortedRanges = newRanges.build(); + poisonCache(0); } /** @@ -249,17 +397,42 @@ public void removeRange(Range range) { * @return Iterator of {@link Range} */ public Iterator rangeIterator() { - return Arrays.asList(sortedRanges).iterator(); + return sortedRanges.iterator(); } public PrimitiveIterator.OfLong indexIterator() { - return Arrays.stream(sortedRanges) - .flatMapToLong(range -> LongStream.rangeClosed(range.getFirst(), range.getLast())) - .iterator(); + if (isEmpty()) { + return LongStream.empty().iterator(); + } + return new PrimitiveIterator.OfLong() { + private int rangeIndex = 0; + private Range current = sortedRanges.get(0); + private long offsetInRange = 0; + + @Override + public long nextLong() { + long value = current.getFirst() + offsetInRange; + if (++offsetInRange >= current.size()) { + rangeIndex++; + offsetInRange = 0; + if (rangeIndex < rangeCount()) { + current = sortedRanges.get(rangeIndex); + } else { + current = null; + } + } + return value; + } + + @Override + public boolean hasNext() { + return rangeIndex < rangeCount(); + } + }; } public int rangeCount() { - return sortedRanges.length; + return sortedRanges.size(); } /** @@ -270,11 +443,15 @@ public int rangeCount() { * @return long */ public long size() { - return Arrays.stream(sortedRanges).mapToLong(Range::size).sum(); + if (rangeCount() == 0) { + return 0; + } + ensureCardinalityCache(); + return cardinality[sortedRanges.size() - 1]; } public boolean isEmpty() { - return sortedRanges.length == 0; + return rangeCount() == 0; } public boolean contains(long value) { @@ -319,19 +496,58 @@ public boolean includesAllOf(RangeSet other) { return true; } + public boolean includesAnyOf(Range range) { + if (isEmpty()) { + return false; + } + // search the sorted list of ranges and find where the current range starts. two case here when using + // binarySearch, either the removed range starts in the same place as an existing range starts, or + // it starts before an item (and so we check the item before and the item after) + int index = Collections.binarySearch(sortedRanges, range); + if (index >= 0) { + // matching start position + return true; + } + // adjusted index notes where the item would be if it were added, minus _one more_ to see if + // it overlaps the item before it. To compute "the position where the new item belongs", we + // would do (-index - 1), so to examine one item prior to that we'll subtract one more. Then, + // to confirm that we are inserting in a valid position, take the max of that value and zero. + index = Math.max(0, -index - 2); + + // Check if there is any overlap with the prev item + Range target = sortedRanges.get(index); + if (range.getFirst() <= target.getLast() && range.getLast() >= target.getFirst()) { + return true; + } + + // Check if there is a later item, and if there is an overlap with it + index++; + if (index >= rangeCount()) { + return false; + } + target = sortedRanges.get(index); + return range.getFirst() <= target.getLast() && range.getLast() >= target.getFirst(); + } + @Override public String toString() { return "RangeSet{" + - "sortedRanges=" + Arrays.toString(sortedRanges) + + "sortedRanges=" + sortedRanges + '}'; } public long getFirstRow() { - return sortedRanges[0].getFirst(); + return sortedRanges.get(0).getFirst(); } public long getLastRow() { - return sortedRanges[sortedRanges.length - 1].getLast(); + return sortedRanges.get(rangeCount() - 1).getLast(); + } + + public RangeSet copy() { + RangeSet copy = new RangeSet(); + copy.sortedRanges = new ArrayList<>(sortedRanges); + return copy; } @Override @@ -342,20 +558,142 @@ public boolean equals(Object o) { return false; final RangeSet rangeSet = (RangeSet) o; - return Arrays.equals(sortedRanges, rangeSet.sortedRanges); + return sortedRanges.equals(rangeSet.sortedRanges); } @Override public int hashCode() { - return Arrays.hashCode(sortedRanges); + return sortedRanges.hashCode(); + } + + /** + * Indicates that this item has been changed, and should be recomputed. Stores the earliest offset that should be + * recomputed. + */ + private void poisonCache(int rangeIndex) { + firstWrongCacheEntry = Math.min(rangeIndex, firstWrongCacheEntry); + } + + /** + * Ensures that the cardinality cache is correct, by correcting any values after the first wrong entry. + */ + private void ensureCardinalityCache() { + if (firstWrongCacheEntry == rangeCount()) { + return; + } + if (cardinality.length < rangeCount()) { + long[] replacement = new long[rangeCount()]; + System.arraycopy(cardinality, 0, replacement, 0, cardinality.length); + cardinality = replacement; + } + assert firstWrongCacheEntry >= 0 : this; + long cumulative = firstWrongCacheEntry == 0 ? 0 : cardinality[firstWrongCacheEntry - 1]; + for (int i = firstWrongCacheEntry; i < rangeCount(); i++) { + cumulative += sortedRanges.get(i).size(); + this.cardinality[i] = cumulative; + } + firstWrongCacheEntry = rangeCount(); + assert cardinality.length >= rangeCount() : this; + } + + public RangeSet subsetForPositions(RangeSet positions, boolean reversed) { + if (reversed) { + throw new UnsupportedOperationException("reversed=true"); + } + if (positions.isEmpty() || isEmpty()) { + return empty(); + } + ensureCardinalityCache(); + + List ranges = new ArrayList<>(); + + Iterator positionsIter = positions.rangeIterator(); + + int from = 0; + while (positionsIter.hasNext()) { + Range nextPosRange = positionsIter.next(); + if (nextPosRange.getFirst() > size()) { + // Entire range is past the end - since ranges are sorted, we're done + break; + } else if (nextPosRange.getFirst() == size()) { + ranges.add(new Range(getLastRow(), getLastRow())); + break; + } + long rangeToTake = nextPosRange.size(); + + int pos = Arrays.binarySearch(cardinality, from, rangeCount(), nextPosRange.getFirst() + 1); + + long first; + Range target; + long offset; + if (pos >= 0) { + // Position matches the last item in the current range + target = sortedRanges.get(pos); + offset = 1; + } else { + // Position matches an earlier item in + pos = -pos - 1; + target = sortedRanges.get(pos); + long c = cardinality[pos]; + offset = c - nextPosRange.getFirst();// positive value to offset backwards from the end of target + } + assert target != null : this + ".subsetForPositions(" + positions + ")"; + assert offset >= 0 && offset <= target.size() : offset; + first = target.getLast() - offset + 1; + + while (rangeToTake > 0) { + long count = Math.min(offset, rangeToTake); + Range res = new Range(first, first + count - 1); + assert count == res.size(); + ranges.add(res); + + rangeToTake -= count; + pos++; + if (pos >= rangeCount()) { + break; + } + target = sortedRanges.get(pos); + first = target.getFirst(); + offset = target.size(); + } + + from = pos - 1; + } + RangeSet result = RangeSet.fromSortedRanges(ranges.toArray(new Range[0])); + assert result.size() <= positions.size(); + return result; } + public long get(long key) { + if (key == 0) { + return getFirstRow(); + } + ensureCardinalityCache(); + + int pos = Arrays.binarySearch(cardinality, key); - Range[] getSortedRanges() { - return sortedRanges; + if (pos >= 0) { + return sortedRanges.get(pos + 1).getFirst(); + } + Range target = sortedRanges.get(-pos - 1); + long c = cardinality[-pos - 1]; + long offset = c - key;// positive value to offset backwards from the end of target + assert offset >= 0; + return target.getLast() - offset + 1; } - void setSortedRanges(Range[] sortedRanges) { - this.sortedRanges = sortedRanges; + /** + * Removes all keys in the provided rangeset that are present in this. + * + * @param other the rows to remove + * @return any removed keys + */ + public RangeSet extract(RangeSet other) { + RangeSet populatedCopy = copy(); + populatedCopy.removeRangeSet(other); + RangeSet removed = copy(); + removed.removeRangeSet(populatedCopy); + removeRangeSet(removed); + return removed; } } diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/ShiftedRange.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/ShiftedRange.java index 4b1b587be97..f1d9e4848d6 100644 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/ShiftedRange.java +++ b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/ShiftedRange.java @@ -31,4 +31,16 @@ public long getDelta() { public void setDelta(final long delta) { this.delta = delta; } + + public Range getResultRange() { + return range.shift(delta); + } + + @Override + public String toString() { + return "ShiftedRange{" + + "range=" + range + + ", delta=" + delta + + '}'; + } } diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/TableSnapshot.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/TableSnapshot.java deleted file mode 100644 index ae5f8870184..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/TableSnapshot.java +++ /dev/null @@ -1,60 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data; - -import io.deephaven.web.shared.data.columns.ColumnData; - -import java.io.Serializable; - -public class TableSnapshot implements Serializable { - public enum SnapshotType { - INITIAL_SNAPSHOT, UPDATE_COLUMN_SNAPSHOT, UPDATE_ROW_SNAPSHOT, FORCED_SNAPSHOT, - } - - private SnapshotType snapshotType; - - private ColumnData[] dataColumns; - private RangeSet includedRows; - - private long tableSize; - - public TableSnapshot() {} - - public TableSnapshot(RangeSet includedAdditions, ColumnData[] dataColumns, long tableSize) { - this.snapshotType = SnapshotType.INITIAL_SNAPSHOT; - - this.dataColumns = dataColumns; - this.includedRows = includedAdditions; - - this.tableSize = tableSize; - } - - public SnapshotType getSnapshotType() { - return snapshotType; - } - - public ColumnData[] getDataColumns() { - return dataColumns; - } - - public void setDataColumns(ColumnData[] dataColumns) { - this.dataColumns = dataColumns; - } - - public RangeSet getIncludedRows() { - return includedRows; - } - - public void setIncludedRows(RangeSet includedRows) { - this.includedRows = includedRows; - } - - public void setTableSize(long tableSize) { - this.tableSize = tableSize; - } - - public long getTableSize() { - return tableSize; - } -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/TableSubscriptionRequest.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/TableSubscriptionRequest.java deleted file mode 100644 index 3dc940db123..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/TableSubscriptionRequest.java +++ /dev/null @@ -1,76 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data; - -import javax.annotation.Nullable; -import java.io.Serializable; -import java.util.BitSet; - -/** - * A class to encapsulate updates to a given table's subscription. - * - * Because the server is now managing the creation of a "tail table" to only subscribe to updates after all filters have - * run, we are giving the server a bit more rights w.r.t. subscription state management. - * - * The client will now send an array of ViewportSubscription, each containing a subscriptionId (JsTable id) and a - * {@link Viewport} object. - * - * The server will be responsible for merging ranges and columns to create flattened tables with the desired viewports. - * - * For now, we'll likely preserve "bunch them all together" semantics, but we should do performance testing to identify - * if we'll get better performance from having multiple tables of smaller viewport scope (more messages on the wire, but - * less work to do before sending messages). - * - * The {@link #columns} must be non-null (and almost always non-empty), but {@link #rows} may be null to indicate a - * non-viewport subscription. - */ -public class TableSubscriptionRequest implements Serializable { - - private int subscriptionId; - private RangeSet rows; - private BitSet columns; - - public TableSubscriptionRequest() { - - } - - public TableSubscriptionRequest(int subscriptionId, @Nullable RangeSet rows, BitSet columns) { - this.subscriptionId = subscriptionId; - this.rows = rows; - this.columns = columns; - } - - public int getSubscriptionId() { - return subscriptionId; - } - - public RangeSet getRows() { - return rows; - } - - public BitSet getColumns() { - return columns; - } - - void setSubscriptionId(int subscriptionId) { - this.subscriptionId = subscriptionId; - } - - void setRows(RangeSet rows) { - this.rows = rows; - } - - void setColumns(BitSet columns) { - this.columns = columns; - } - - @Override - public String toString() { - return "TableSubscriptionRequest{" + - "subscriptionId=" + subscriptionId + - ", rows=" + rows + - ", columns=" + columns + - '}'; - } -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/Viewport.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/Viewport.java deleted file mode 100644 index bdc19ce9e39..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/Viewport.java +++ /dev/null @@ -1,75 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data; - -import java.io.Serializable; -import java.util.BitSet; - -public class Viewport implements Serializable { - - private final RangeSet rows; - private final BitSet columns; - - public Viewport() { - this(new RangeSet(), new BitSet()); - } - - public Viewport(RangeSet rows, BitSet columns) { - this.rows = rows; - this.columns = columns; - } - - public Viewport merge(Viewport other) { - RangeSet mergedRows = new RangeSet(); - rows.rangeIterator().forEachRemaining(mergedRows::addRange); - other.rows.rangeIterator().forEachRemaining(mergedRows::addRange); - - BitSet mergedColumns = new BitSet(); - mergedColumns.or(columns); - mergedColumns.or(other.columns); - - return new Viewport(mergedRows, mergedColumns); - } - - public RangeSet getRows() { - return rows; - } - - public BitSet getColumns() { - return columns; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - final Viewport viewport = (Viewport) o; - - if (!rows.equals(viewport.rows)) - return false; - return columns.equals(viewport.columns); - } - - @Override - public int hashCode() { - int result = rows.hashCode(); - result = 31 * result + columns.hashCode(); - return result; - } - - @Override - public String toString() { - return "Viewport{" + - "rows=" + rows + - ", columns=" + columns + - '}'; - } - - public boolean isEmpty() { - return rows.size() == 0 && columns.isEmpty(); - } -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/BigDecimalArrayColumnData.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/BigDecimalArrayColumnData.java deleted file mode 100644 index 9a6bed67823..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/BigDecimalArrayColumnData.java +++ /dev/null @@ -1,44 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data.columns; - -import java.math.BigDecimal; -import java.util.Arrays; - -/** - * Holder for data associated with a column of type java.math.BigDecimal. - */ -public class BigDecimalArrayColumnData extends ColumnData { - private BigDecimal[] data; - - public BigDecimalArrayColumnData() {} - - public BigDecimalArrayColumnData(BigDecimal[] data) { - this.data = data; - } - - public BigDecimal[] getData() { - return data; - } - - public void setData(BigDecimal[] data) { - this.data = data; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - final BigDecimalArrayColumnData that = (BigDecimalArrayColumnData) o; - return Arrays.deepEquals(data, that.data); - } - - @Override - public int hashCode() { - return Arrays.deepHashCode(data); - } -} - diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/BigIntegerArrayColumnData.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/BigIntegerArrayColumnData.java deleted file mode 100644 index b9eb9db11fc..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/BigIntegerArrayColumnData.java +++ /dev/null @@ -1,44 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data.columns; - -import java.math.BigInteger; -import java.util.Arrays; - -/** - * Holder for data associated with a column of type java.math.BigInteger. - */ -public class BigIntegerArrayColumnData extends ColumnData { - private BigInteger[] data; - - public BigIntegerArrayColumnData() {} - - public BigIntegerArrayColumnData(BigInteger[] data) { - this.data = data; - } - - public BigInteger[] getData() { - return data; - } - - public void setData(BigInteger[] data) { - this.data = data; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - final BigIntegerArrayColumnData that = (BigIntegerArrayColumnData) o; - return Arrays.deepEquals(data, that.data); - } - - @Override - public int hashCode() { - return Arrays.deepHashCode(data); - } -} - diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/BooleanArrayColumnData.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/BooleanArrayColumnData.java deleted file mode 100644 index 487c6592ca2..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/BooleanArrayColumnData.java +++ /dev/null @@ -1,41 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data.columns; - -import java.util.Arrays; - -public class BooleanArrayColumnData extends ColumnData { - private Boolean[] data; - - public BooleanArrayColumnData() {} - - public BooleanArrayColumnData(Boolean[] data) { - this.data = data; - } - - public Boolean[] getData() { - return data; - } - - public void setData(Boolean[] data) { - this.data = data; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - BooleanArrayColumnData that = (BooleanArrayColumnData) o; - - return Arrays.equals(data, that.data); - } - - @Override - public int hashCode() { - return Arrays.hashCode(data); - } -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/ByteArrayColumnData.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/ByteArrayColumnData.java deleted file mode 100644 index 7bbe9a1b710..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/ByteArrayColumnData.java +++ /dev/null @@ -1,41 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data.columns; - -import java.util.Arrays; - -public class ByteArrayColumnData extends ColumnData { - private byte[] data; - - public ByteArrayColumnData() {} - - public ByteArrayColumnData(byte[] data) { - this.data = data; - } - - public byte[] getData() { - return data; - } - - public void setData(byte[] data) { - this.data = data; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - ByteArrayColumnData that = (ByteArrayColumnData) o; - - return Arrays.equals(data, that.data); - } - - @Override - public int hashCode() { - return Arrays.hashCode(data); - } -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/CharArrayColumnData.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/CharArrayColumnData.java deleted file mode 100644 index 22aae999562..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/CharArrayColumnData.java +++ /dev/null @@ -1,41 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data.columns; - -import java.util.Arrays; - -public class CharArrayColumnData extends ColumnData { - private char[] data; - - public CharArrayColumnData() {} - - public CharArrayColumnData(char[] data) { - this.data = data; - } - - public char[] getData() { - return data; - } - - public void setData(char[] data) { - this.data = data; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - CharArrayColumnData that = (CharArrayColumnData) o; - - return Arrays.equals(data, that.data); - } - - @Override - public int hashCode() { - return Arrays.hashCode(data); - } -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/ColumnData.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/ColumnData.java deleted file mode 100644 index 20817d4672a..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/ColumnData.java +++ /dev/null @@ -1,11 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data.columns; - -import java.io.Serializable; - -public abstract class ColumnData implements Serializable { - - public abstract Object getData(); -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/DoubleArrayColumnData.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/DoubleArrayColumnData.java deleted file mode 100644 index 6864217cfb4..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/DoubleArrayColumnData.java +++ /dev/null @@ -1,41 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data.columns; - -import java.util.Arrays; - -public class DoubleArrayColumnData extends ColumnData { - private double[] data; - - public DoubleArrayColumnData() {} - - public DoubleArrayColumnData(double[] data) { - this.data = data; - } - - public double[] getData() { - return data; - } - - public void setData(double[] data) { - this.data = data; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - DoubleArrayColumnData that = (DoubleArrayColumnData) o; - - return Arrays.equals(data, that.data); - } - - @Override - public int hashCode() { - return Arrays.hashCode(data); - } -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/FloatArrayColumnData.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/FloatArrayColumnData.java deleted file mode 100644 index 8fd13a5962f..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/FloatArrayColumnData.java +++ /dev/null @@ -1,41 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data.columns; - -import java.util.Arrays; - -public class FloatArrayColumnData extends ColumnData { - private float[] data; - - public FloatArrayColumnData() {} - - public FloatArrayColumnData(float[] data) { - this.data = data; - } - - public float[] getData() { - return data; - } - - public void setData(float[] data) { - this.data = data; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - FloatArrayColumnData that = (FloatArrayColumnData) o; - - return Arrays.equals(data, that.data); - } - - @Override - public int hashCode() { - return Arrays.hashCode(data); - } -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/IntArrayColumnData.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/IntArrayColumnData.java deleted file mode 100644 index 49b1b2e4d86..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/IntArrayColumnData.java +++ /dev/null @@ -1,41 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data.columns; - -import java.util.Arrays; - -public class IntArrayColumnData extends ColumnData { - private int[] data; - - public IntArrayColumnData() {} - - public IntArrayColumnData(int[] data) { - this.data = data; - } - - public int[] getData() { - return data; - } - - public void setData(int[] data) { - this.data = data; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - IntArrayColumnData that = (IntArrayColumnData) o; - - return Arrays.equals(data, that.data); - } - - @Override - public int hashCode() { - return Arrays.hashCode(data); - } -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/LocalDateArrayColumnData.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/LocalDateArrayColumnData.java deleted file mode 100644 index 4fd4226279e..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/LocalDateArrayColumnData.java +++ /dev/null @@ -1,46 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data.columns; - -import io.deephaven.web.shared.data.LocalDate; - -import java.util.Arrays; - -/** - * Holder for data associated with a column of type java.time.LocalDate. This type is serialized with a custom LocalDate - * type for efficiency and GWT compatibility. - */ -public class LocalDateArrayColumnData extends ColumnData { - private LocalDate[] data; - - public LocalDateArrayColumnData() {} - - public LocalDateArrayColumnData(LocalDate[] data) { - this.data = data; - } - - public LocalDate[] getData() { - return data; - } - - public void setData(LocalDate[] data) { - this.data = data; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - final LocalDateArrayColumnData that = (LocalDateArrayColumnData) o; - return Arrays.deepEquals(data, that.data); - } - - @Override - public int hashCode() { - return Arrays.deepHashCode(data); - } -} - diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/LocalTimeArrayColumnData.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/LocalTimeArrayColumnData.java deleted file mode 100644 index 2e8ddc291c3..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/LocalTimeArrayColumnData.java +++ /dev/null @@ -1,45 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data.columns; - -import io.deephaven.web.shared.data.LocalTime; - -import java.util.Arrays; - -/** - * Holder for data associated with a column of type java.time.LocalTime. This type is serialized with a custom LocalTime - * type for efficiency and GWT compatibility. - */ -public class LocalTimeArrayColumnData extends ColumnData { - private LocalTime[] data; - - public LocalTimeArrayColumnData() {} - - public LocalTimeArrayColumnData(LocalTime[] data) { - this.data = data; - } - - public LocalTime[] getData() { - return data; - } - - public void setData(LocalTime[] data) { - this.data = data; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - final LocalTimeArrayColumnData that = (LocalTimeArrayColumnData) o; - return Arrays.deepEquals(data, that.data); - } - - @Override - public int hashCode() { - return Arrays.deepHashCode(data); - } -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/LongArrayColumnData.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/LongArrayColumnData.java deleted file mode 100644 index a9e252568b1..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/LongArrayColumnData.java +++ /dev/null @@ -1,41 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data.columns; - -import java.util.Arrays; - -public class LongArrayColumnData extends ColumnData { - private long[] data; - - public LongArrayColumnData() {} - - public LongArrayColumnData(long[] data) { - this.data = data; - } - - public long[] getData() { - return data; - } - - public void setData(long[] data) { - this.data = data; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - LongArrayColumnData that = (LongArrayColumnData) o; - - return Arrays.equals(data, that.data); - } - - @Override - public int hashCode() { - return Arrays.hashCode(data); - } -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/ShortArrayColumnData.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/ShortArrayColumnData.java deleted file mode 100644 index d8c0540ce96..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/ShortArrayColumnData.java +++ /dev/null @@ -1,41 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data.columns; - -import java.util.Arrays; - -public class ShortArrayColumnData extends ColumnData { - private short[] data; - - public ShortArrayColumnData() {} - - public ShortArrayColumnData(short[] data) { - this.data = data; - } - - public short[] getData() { - return data; - } - - public void setData(short[] data) { - this.data = data; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - ShortArrayColumnData that = (ShortArrayColumnData) o; - - return Arrays.equals(data, that.data); - } - - @Override - public int hashCode() { - return Arrays.hashCode(data); - } -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/StringArrayArrayColumnData.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/StringArrayArrayColumnData.java deleted file mode 100644 index 721e0aabd72..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/StringArrayArrayColumnData.java +++ /dev/null @@ -1,41 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data.columns; - -import java.util.Arrays; - -public class StringArrayArrayColumnData extends ColumnData { - private String[][] data; - - public StringArrayArrayColumnData() {} - - public StringArrayArrayColumnData(String[][] data) { - this.data = data; - } - - public String[][] getData() { - return data; - } - - public void setData(String[][] data) { - this.data = data; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - StringArrayArrayColumnData that = (StringArrayArrayColumnData) o; - - return Arrays.deepEquals(data, that.data); - } - - @Override - public int hashCode() { - return Arrays.deepHashCode(data); - } -} diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/StringArrayColumnData.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/StringArrayColumnData.java deleted file mode 100644 index 338402855bf..00000000000 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/columns/StringArrayColumnData.java +++ /dev/null @@ -1,41 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.web.shared.data.columns; - -import java.util.Arrays; - -public class StringArrayColumnData extends ColumnData { - private String[] data; - - public StringArrayColumnData() {} - - public StringArrayColumnData(String[] data) { - this.data = data; - } - - public String[] getData() { - return data; - } - - public void setData(String[] data) { - this.data = data; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - StringArrayColumnData that = (StringArrayColumnData) o; - - return Arrays.equals(data, that.data); - } - - @Override - public int hashCode() { - return Arrays.hashCode(data); - } -} diff --git a/web/shared-beans/src/test/java/io/deephaven/web/shared/data/RangeSetTest.java b/web/shared-beans/src/test/java/io/deephaven/web/shared/data/RangeSetTest.java index d28a702bbbb..a2c2522b2d9 100644 --- a/web/shared-beans/src/test/java/io/deephaven/web/shared/data/RangeSetTest.java +++ b/web/shared-beans/src/test/java/io/deephaven/web/shared/data/RangeSetTest.java @@ -128,21 +128,24 @@ public void testAddExistingRange() { @Test public void testOverlappingRangesInDifferentOrder() { - // add three items in each possible order to a rangeset, ensure results are always the same + // add five ranges, where some overlap others, in each possible order to a rangeset, ensure results are always + // the same Range rangeA = new Range(100, 108); Range rangeB = new Range(105, 112); Range rangeC = new Range(110, 115); - Collections2.permutations(Arrays.asList(rangeA, rangeB, rangeC)).forEach(list -> { + Range rangeD = new Range(100, 113); + Range rangeE = new Range(101, 115); + Collections2.permutations(Arrays.asList(rangeA, rangeB, rangeC, rangeD, rangeE)).forEach(list -> { RangeSet rangeSet = new RangeSet(); list.forEach(rangeSet::addRange); - assertEquals(16, rangeSet.size()); + assertEquals(list.toString(), 16, rangeSet.size()); assertEquals(list.toString(), Collections.singletonList(new Range(100, 115)), asList(rangeSet)); }); - // same three items, but with another before that will not overlap with them + // same five items, but with another before that will not overlap with them Range before = new Range(0, 4); - Collections2.permutations(Arrays.asList(before, rangeA, rangeB, rangeC)).forEach(list -> { + Collections2.permutations(Arrays.asList(before, rangeA, rangeB, rangeC, rangeD, rangeE)).forEach(list -> { RangeSet rangeSet = new RangeSet(); list.forEach(rangeSet::addRange); @@ -150,9 +153,9 @@ public void testOverlappingRangesInDifferentOrder() { assertEquals(list.toString(), Arrays.asList(new Range(0, 4), new Range(100, 115)), asList(rangeSet)); }); - // same three items, but with another following that will not overlap with them + // same five items, but with another following that will not overlap with them Range after = new Range(200, 204); - Collections2.permutations(Arrays.asList(after, rangeA, rangeB, rangeC)).forEach(list -> { + Collections2.permutations(Arrays.asList(after, rangeA, rangeB, rangeC, rangeD, rangeE)).forEach(list -> { RangeSet rangeSet = new RangeSet(); list.forEach(rangeSet::addRange); @@ -212,6 +215,41 @@ public void testIncludesAllOf() { assertFalse(rangeSet.includesAllOf(RangeSet.ofRange(54, 60))); } + @Test + public void testIncludesAnyOf() { + RangeSet rangeSet = new RangeSet(); + rangeSet.addRange(new Range(0, 19)); + rangeSet.addRange(new Range(50, 54)); + + assertTrue(rangeSet.includesAnyOf(new Range(0, 19))); + assertTrue(rangeSet.includesAnyOf(new Range(50, 54))); + + rangeSet.indexIterator().forEachRemaining((LongConsumer) l -> { + assertTrue(rangeSet.includesAnyOf(new Range(l, l))); + }); + + assertTrue(rangeSet.includesAnyOf(new Range(0, 20))); + assertTrue(rangeSet.includesAnyOf(new Range(10, 20))); + assertTrue(rangeSet.includesAnyOf(new Range(19, 20))); + + assertTrue(rangeSet.includesAnyOf(new Range(19, 30))); + assertFalse(rangeSet.includesAnyOf(new Range(20, 30))); + assertFalse(rangeSet.includesAnyOf(new Range(21, 30))); + + assertFalse(rangeSet.includesAnyOf(new Range(30, 40))); + + assertFalse(rangeSet.includesAnyOf(new Range(40, 49))); + assertTrue(rangeSet.includesAnyOf(new Range(40, 50))); + assertFalse(rangeSet.includesAnyOf(new Range(40, 41))); + assertTrue(rangeSet.includesAnyOf(new Range(40, 54))); + + assertTrue(rangeSet.includesAnyOf(new Range(49, 54))); + assertTrue(rangeSet.includesAnyOf(new Range(50, 55))); + assertTrue(rangeSet.includesAnyOf(new Range(50, 60))); + + assertTrue(rangeSet.includesAnyOf(new Range(54, 60))); + assertFalse(rangeSet.includesAnyOf(new Range(55, 60))); + } @Test public void testRemove() { @@ -475,4 +513,123 @@ public void testLarge() { assertEquals(RangeSet.ofItems(largeA, largeB), rangeSet); } + @Test + public void testSubsetForPostions() { + RangeSet initialRange = RangeSet.ofItems(2, 4, 6, 8); + assertEquals(RangeSet.ofItems(4, 8), initialRange.subsetForPositions(RangeSet.ofItems(1, 3), false)); + assertEquals(RangeSet.ofItems(4, 8), initialRange.subsetForPositions(RangeSet.ofItems(1, 3, 4), false)); + assertEquals(RangeSet.ofItems(4, 8), initialRange.subsetForPositions(RangeSet.ofItems(1, 3, 5), false)); + assertEquals(initialRange, initialRange.subsetForPositions(RangeSet.ofItems(0, 1, 2, 3, 4, 5, 100), false)); + assertEquals(initialRange, initialRange.subsetForPositions(RangeSet.ofItems(0, 1, 2, 3, 100), false)); + + assertEquals(RangeSet.ofItems(4, 6, 8), initialRange.subsetForPositions(RangeSet.ofRange(1, 3), false)); + assertEquals(RangeSet.ofItems(2, 4, 6), initialRange.subsetForPositions(RangeSet.ofRange(0, 2), false)); + assertEquals(initialRange, initialRange.subsetForPositions(RangeSet.ofRange(0, 3), false)); + assertEquals(initialRange, initialRange.subsetForPositions(RangeSet.ofRange(0, 9), false)); + + initialRange = RangeSet.ofRange(10, 109); + assertEquals(RangeSet.ofItems(12, 14), initialRange.subsetForPositions(RangeSet.ofItems(2, 4), false)); + assertEquals(RangeSet.ofItems(12, 14), initialRange.subsetForPositions(RangeSet.ofItems(2, 4, 101), false)); + + assertEquals(RangeSet.empty(), RangeSet.empty().subsetForPositions(RangeSet.ofItems(0), false)); + assertEquals(RangeSet.ofItems(99), + RangeSet.ofRange(0, 99).subsetForPositions(RangeSet.ofRange(100, 104), false)); + + initialRange = RangeSet.empty(); + assertEquals(0, initialRange.size()); + initialRange.addRange(new Range(0, 1)); + assertEquals(2, initialRange.size()); + initialRange.addRange(new Range(2, 3)); + assertEquals(4, initialRange.size()); + initialRange.removeRange(new Range(0, 3)); + assertEquals(0, initialRange.size()); + initialRange.addRange(new Range(0, 1)); + assertEquals(2, initialRange.size()); + + initialRange = RangeSet.ofItems(1, 4, 5, 6); + assertEquals(RangeSet.ofItems(1, 4, 5, 6), initialRange.subsetForPositions(RangeSet.ofRange(0, 3), false)); + assertEquals(RangeSet.ofItems(1, 5, 6), initialRange.subsetForPositions(RangeSet.ofItems(0, 2, 3), false)); + assertEquals(RangeSet.ofItems(1, 4, 6), initialRange.subsetForPositions(RangeSet.ofItems(0, 1, 3), false)); + assertEquals(RangeSet.ofItems(1, 4, 5), initialRange.subsetForPositions(RangeSet.ofItems(0, 1, 2), false)); + assertEquals(RangeSet.ofItems(1, 5), initialRange.subsetForPositions(RangeSet.ofItems(0, 2), false)); + assertEquals(RangeSet.ofItems(4, 5), initialRange.subsetForPositions(RangeSet.ofRange(1, 2), false)); + assertEquals(RangeSet.ofItems(4, 5, 6), initialRange.subsetForPositions(RangeSet.ofRange(1, 3), false)); + assertEquals(RangeSet.ofItems(5, 6), initialRange.subsetForPositions(RangeSet.ofRange(2, 3), false)); + } + + @Test + public void testGet() { + long[] rows = {0, 1, 4, 5, 7, 9}; + RangeSet initialRange = RangeSet.ofItems(rows); + + for (int i = 0; i < rows.length; i++) { + assertEquals("i=" + i, rows[i], initialRange.get(i)); + } + + initialRange.removeRange(new Range(0, 1)); + } + + @Test + public void testShift() { + RangeSet r = RangeSet.ofRange(0, 2); + r.applyShifts(new ShiftedRange[] { + new ShiftedRange(new Range(0, 2), 2) + }); + assertEquals(RangeSet.ofRange(2, 4), r); + r.applyShifts(new ShiftedRange[] { + new ShiftedRange(new Range(2, 6), -2) + }); + assertEquals(RangeSet.ofRange(0, 2), r); + r.applyShifts(new ShiftedRange[] { + new ShiftedRange(new Range(1, 2), 3) + }); + assertEquals(RangeSet.ofItems(0, 4, 5), r); + r.applyShifts(new ShiftedRange[] { + new ShiftedRange(new Range(4, 4), -1) + }); + assertEquals(RangeSet.ofItems(0, 3, 5), r); + + r = RangeSet.ofItems(0, 3, 4, 5, 6, 10); + r.applyShifts(new ShiftedRange[] { + new ShiftedRange(new Range(4, 4), 2), + new ShiftedRange(new Range(5, 6), 3), + }); + assertEquals(RangeSet.ofItems(0, 3, 6, 8, 9, 10), r); + + + r = RangeSet.fromSortedRanges(new Range[] { + new Range(0, 1), + new Range(3, 5), + new Range(7, 13), + new Range(15, 19), + }); + r.applyShifts(new ShiftedRange[] { + new ShiftedRange(new Range(3, 4), -1), + new ShiftedRange(new Range(7, 13), -1), + new ShiftedRange(new Range(15, 17), -2), + }); + assertEquals(RangeSet.fromSortedRanges(new Range[] { + new Range(0, 3), + new Range(5, 15), + new Range(18, 19), + }), r); + + + r = RangeSet.fromSortedRanges(new Range[] { + new Range(28972, 28987), + new Range(28989, 29003), + new Range(29005, 29011), + new Range(29013, 29013), + new Range(29015, 29018), + new Range(29020, 29020), + new Range(29022, 29024), + new Range(29026, 29039), + }); + r.applyShifts(new ShiftedRange[] { + new ShiftedRange(new Range(28989, 29011), 2), + new ShiftedRange(new Range(29013, 29013), 1), + new ShiftedRange(new Range(29022, 29024), -1), + new ShiftedRange(new Range(29026, 29026), -2), + }); + } }