From ed3a523df8c51e03d34837e48e1be01780e9fcd3 Mon Sep 17 00:00:00 2001 From: Devin Smith Date: Wed, 28 Aug 2024 14:01:14 -0700 Subject: [PATCH 01/15] feat: Create deephaven.time time-type aliases (#5269) This adds `TimeZoneLike`, `LocalDateLike`, `InstantLike`, `ZonedDateTimeLike`, `DurationLike`, and `PeriodLike` as aliased union types for objects that can be coerced into said type. This makes it easier for public APIs to reference the proper time-types. `S3Instructions`, `json.instant_val`, `TableReplayer`, `time_table`, and the `time.to_j_` APIs have been updated to reference the new types. --- py/server/deephaven/experimental/s3.py | 21 +++---- py/server/deephaven/json/__init__.py | 11 ++-- py/server/deephaven/replay.py | 15 ++--- py/server/deephaven/table_factory.py | 24 ++++---- py/server/deephaven/time.py | 85 ++++++++++++++++++-------- py/server/tests/test_table_factory.py | 5 +- 6 files changed, 91 insertions(+), 70 deletions(-) diff --git a/py/server/deephaven/experimental/s3.py b/py/server/deephaven/experimental/s3.py index db6168aca16..1fe105e7a40 100644 --- a/py/server/deephaven/experimental/s3.py +++ b/py/server/deephaven/experimental/s3.py @@ -1,16 +1,13 @@ # # Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending # -import datetime from typing import Optional, Union import jpy -import numpy as np -import pandas as pd -from deephaven import time, DHError +from deephaven import DHError from deephaven._wrapper import JObjectWrapper -from deephaven.dtypes import Duration +from deephaven.time import DurationLike, to_j_duration # If we move S3 to a permanent module, we should remove this try/except block and just import the types directly. try: @@ -38,10 +35,8 @@ def __init__(self, max_concurrent_requests: Optional[int] = None, read_ahead_count: Optional[int] = None, fragment_size: Optional[int] = None, - connection_timeout: Union[ - Duration, int, str, datetime.timedelta, np.timedelta64, pd.Timedelta, None] = None, - read_timeout: Union[ - Duration, int, str, datetime.timedelta, np.timedelta64, pd.Timedelta, None] = None, + connection_timeout: Optional[DurationLike] = None, + read_timeout: Optional[DurationLike] = None, access_key_id: Optional[str] = None, secret_access_key: Optional[str] = None, anonymous_access: bool = False, @@ -62,11 +57,11 @@ def __init__(self, fragment. Defaults to 32, which means fetch the next 32 fragments in advance when reading the current fragment. fragment_size (int): the maximum size of each fragment to read, defaults to 64 KiB. If there are fewer bytes remaining in the file, the fetched fragment can be smaller. - connection_timeout (Union[Duration, int, str, datetime.timedelta, np.timedelta64, pd.Timedelta]): + connection_timeout (DurationLike): the amount of time to wait when initially establishing a connection before giving up and timing out, can be expressed as an integer in nanoseconds, a time interval string, e.g. "PT00:00:00.001" or "PT1s", or other time duration types. Default to 2 seconds. - read_timeout (Union[Duration, int, str, datetime.timedelta, np.timedelta64, pd.Timedelta]): + read_timeout (DurationLike): the amount of time to wait when reading a fragment before giving up and timing out, can be expressed as an integer in nanoseconds, a time interval string, e.g. "PT00:00:00.001" or "PT1s", or other time duration types. Default to 2 seconds. @@ -111,10 +106,10 @@ def __init__(self, builder.fragmentSize(fragment_size) if connection_timeout is not None: - builder.connectionTimeout(time.to_j_duration(connection_timeout)) + builder.connectionTimeout(to_j_duration(connection_timeout)) if read_timeout is not None: - builder.readTimeout(time.to_j_duration(read_timeout)) + builder.readTimeout(to_j_duration(read_timeout)) if ((access_key_id is not None and secret_access_key is None) or (access_key_id is None and secret_access_key is not None)): diff --git a/py/server/deephaven/json/__init__.py b/py/server/deephaven/json/__init__.py index 3112b2d68cc..3cc9b986e33 100644 --- a/py/server/deephaven/json/__init__.py +++ b/py/server/deephaven/json/__init__.py @@ -72,7 +72,7 @@ from deephaven import dtypes from deephaven._wrapper import JObjectWrapper -from deephaven.time import to_j_instant +from deephaven.time import InstantLike, to_j_instant from deephaven._jpy import strict_cast @@ -1056,14 +1056,13 @@ def string_val( return JsonValue(builder.build()) -# TODO(deephaven-core#5269): Create deephaven.time time-type aliases def instant_val( allow_missing: bool = True, allow_null: bool = True, number_format: Literal[None, "s", "ms", "us", "ns"] = None, allow_decimal: bool = False, - on_missing: Optional[Any] = None, - on_null: Optional[Any] = None, + on_missing: Optional[InstantLike] = None, + on_null: Optional[InstantLike] = None, ) -> JsonValue: """Creates an Instant value. For example, the JSON string @@ -1110,8 +1109,8 @@ def instant_val( the epoch. When not set, a JSON string in the ISO-8601 format is expected. allow_decimal (bool): if the Instant value is allowed to be a JSON decimal type, default is False. Only valid when number_format is specified. - on_missing (Optional[Any]): the value to use when the JSON value is missing and allow_missing is True, default is None. - on_null (Optional[Any]): the value to use when the JSON value is null and allow_null is True, default is None. + on_missing (Optional[InstantLike]): the value to use when the JSON value is missing and allow_missing is True, default is None. + on_null (Optional[InstantLike]): the value to use when the JSON value is null and allow_null is True, default is None. Returns: the Instant value diff --git a/py/server/deephaven/replay.py b/py/server/deephaven/replay.py index 78c8e018af1..b578c5e3d66 100644 --- a/py/server/deephaven/replay.py +++ b/py/server/deephaven/replay.py @@ -4,16 +4,12 @@ """ This module provides support for replaying historical data. """ -from typing import Union import jpy -import datetime -import numpy as np -import pandas as pd - -from deephaven import dtypes, DHError, time +from deephaven import DHError, time from deephaven._wrapper import JObjectWrapper from deephaven.table import Table +from deephaven.time import InstantLike _JReplayer = jpy.get_type("io.deephaven.engine.table.impl.replay.Replayer") @@ -27,14 +23,13 @@ class TableReplayer(JObjectWrapper): j_object_type = _JReplayer - def __init__(self, start_time: Union[dtypes.Instant, int, str, datetime.datetime, np.datetime64, pd.Timestamp], - end_time: Union[dtypes.Instant, int, str, datetime.datetime, np.datetime64, pd.Timestamp]): + def __init__(self, start_time: InstantLike, end_time: InstantLike): """Initializes the replayer. Args: - start_time (Union[dtypes.Instant, int, str, datetime.datetime, np.datetime64, pd.Timestamp]): + start_time (InstantLike): replay start time. Integer values are nanoseconds since the Epoch. - end_time (Union[dtypes.Instant, int, str, datetime.datetime, np.datetime64, pd.Timestamp]): + end_time (InstantLike): replay end time. Integer values are nanoseconds since the Epoch. Raises: diff --git a/py/server/deephaven/table_factory.py b/py/server/deephaven/table_factory.py index 02316928573..3b425583ee6 100644 --- a/py/server/deephaven/table_factory.py +++ b/py/server/deephaven/table_factory.py @@ -4,20 +4,19 @@ """ This module provides various ways to make a Deephaven table. """ -import datetime from typing import Callable, List, Dict, Any, Union, Sequence, Tuple, Mapping, Optional import jpy -import numpy as np import pandas as pd -from deephaven import execution_context, DHError, time +from deephaven import execution_context, DHError from deephaven._wrapper import JObjectWrapper from deephaven.column import InputColumn -from deephaven.dtypes import DType, Duration, Instant +from deephaven.dtypes import DType from deephaven.execution_context import ExecutionContext from deephaven.jcompat import j_lambda, j_list_to_list, to_sequence from deephaven.table import Table, TableDefinition, TableDefinitionLike +from deephaven.time import DurationLike, InstantLike, to_j_duration, to_j_instant from deephaven.update_graph import auto_locking_ctx _JTableFactory = jpy.get_type("io.deephaven.engine.table.TableFactory") @@ -53,16 +52,16 @@ def empty_table(size: int) -> Table: raise DHError(e, "failed to create an empty table.") from e -def time_table(period: Union[Duration, int, str, datetime.timedelta, np.timedelta64, pd.Timedelta], - start_time: Union[None, Instant, int, str, datetime.datetime, np.datetime64, pd.Timestamp] = None, +def time_table(period: DurationLike, + start_time: Optional[InstantLike] = None, blink_table: bool = False) -> Table: """Creates a table that adds a new row on a regular interval. Args: - period (Union[dtypes.Duration, int, str, datetime.timedelta, np.timedelta64, pd.Timedelta]): + period (DurationLike): time interval between new row additions, can be expressed as an integer in nanoseconds, a time interval string, e.g. "PT00:00:00.001" or "PT1s", or other time duration types. - start_time (Union[None, Instant, int, str, datetime.datetime, np.datetime64, pd.Timestamp], optional): + start_time (Optional[InstantLike]): start time for adding new rows, defaults to None which means use the current time as the start time. blink_table (bool, optional): if the time table should be a blink table, defaults to False @@ -76,14 +75,13 @@ def time_table(period: Union[Duration, int, str, datetime.timedelta, np.timedelt try: builder = _JTableTools.timeTableBuilder() - if not isinstance(period, str) and not isinstance(period, int): - period = time.to_j_duration(period) + if period is None: + raise ValueError("period must be specified") - builder.period(period) + builder.period(to_j_duration(period)) if start_time: - start_time = time.to_j_instant(start_time) - builder.startTime(start_time) + builder.startTime(to_j_instant(start_time)) if blink_table: builder.blinkTable(blink_table) diff --git a/py/server/deephaven/time.py b/py/server/deephaven/time.py index 208dc25c383..5868392c0c8 100644 --- a/py/server/deephaven/time.py +++ b/py/server/deephaven/time.py @@ -29,6 +29,50 @@ _NANOS_PER_SECOND = 1000000000 _NANOS_PER_MICRO = 1000 +if sys.version_info >= (3, 10): + from typing import TypeAlias # novermin + + TimeZoneLike : TypeAlias = Union[TimeZone, str, datetime.tzinfo, datetime.datetime, pandas.Timestamp] + """A Union representing objects that can be coerced into a TimeZone.""" + + LocalDateLike : TypeAlias = Union[LocalDate, str, datetime.date, datetime.datetime, numpy.datetime64, pandas.Timestamp] + """A Union representing objects that can be coerced into a LocalDate.""" + + LocalTimeLike : TypeAlias = Union[LocalTime, int, str, datetime.time, datetime.datetime, numpy.datetime64, pandas.Timestamp] + """A Union representing objects that can be coerced into a LocalTime.""" + + InstantLike : TypeAlias = Union[Instant, int, str, datetime.datetime, numpy.datetime64, pandas.Timestamp] + """A Union representing objects that can be coerced into an Instant.""" + + ZonedDateTimeLike : TypeAlias = Union[ZonedDateTime, str, datetime.datetime, numpy.datetime64, pandas.Timestamp] + """A Union representing objects that can be coerced into a ZonedDateTime.""" + + DurationLike : TypeAlias = Union[Duration, int, str, datetime.timedelta, numpy.timedelta64, pandas.Timedelta] + """A Union representing objects that can be coerced into a Duration.""" + + PeriodLike : TypeAlias = Union[Period, str, datetime.timedelta, numpy.timedelta64, pandas.Timedelta] + """A Union representing objects that can be coerced into a Period.""" +else: + TimeZoneLike = Union[TimeZone, str, datetime.tzinfo, datetime.datetime, pandas.Timestamp] + """A Union representing objects that can be coerced into a TimeZone.""" + + LocalDateLike = Union[LocalDate, str, datetime.date, datetime.datetime, numpy.datetime64, pandas.Timestamp] + """A Union representing objects that can be coerced into a LocalDate.""" + + LocalTimeLike = Union[LocalTime, int, str, datetime.time, datetime.datetime, numpy.datetime64, pandas.Timestamp] + """A Union representing objects that can be coerced into a LocalTime.""" + + InstantLike = Union[Instant, int, str, datetime.datetime, numpy.datetime64, pandas.Timestamp] + """A Union representing objects that can be coerced into an Instant.""" + + ZonedDateTimeLike = Union[ZonedDateTime, str, datetime.datetime, numpy.datetime64, pandas.Timestamp] + """A Union representing objects that can be coerced into a ZonedDateTime.""" + + DurationLike = Union[Duration, int, str, datetime.timedelta, numpy.timedelta64, pandas.Timedelta] + """A Union representing objects that can be coerced into a Duration.""" + + PeriodLike = Union[Period, str, datetime.timedelta, numpy.timedelta64, pandas.Timedelta] + """A Union representing objects that can be coerced into a Period.""" # region Clock @@ -223,15 +267,14 @@ def _tzinfo_to_j_time_zone(tzi: datetime.tzinfo) -> TimeZone: raise TypeError(f"Unsupported conversion: {str(type(tzi))} -> TimeZone\n\tDetails:\n\t{details}") -def to_j_time_zone(tz: Union[None, TimeZone, str, datetime.tzinfo, datetime.datetime, pandas.Timestamp]) -> \ - Optional[TimeZone]: +def to_j_time_zone(tz: Optional[TimeZoneLike]) -> Optional[TimeZone]: """ Converts a time zone value to a Java TimeZone. Time zone values can be None, a Java TimeZone, a string, a datetime.tzinfo, a datetime.datetime, or a pandas.Timestamp. Args: - tz (Union[None, TimeZone, str, datetime.tzinfo, datetime.datetime, pandas.Timestamp]): A time zone value. + tz (Optional[TimeZoneLike]): A time zone value. If None is provided, None is returned. If a string is provided, it is parsed as a time zone name. @@ -266,8 +309,7 @@ def to_j_time_zone(tz: Union[None, TimeZone, str, datetime.tzinfo, datetime.date raise DHError(e) from e -def to_j_local_date(dt: Union[None, LocalDate, str, datetime.date, datetime.datetime, - numpy.datetime64, pandas.Timestamp]) -> Optional[LocalDate]: +def to_j_local_date(dt: Optional[LocalDateLike]) -> Optional[LocalDate]: """ Converts a date time value to a Java LocalDate. Date time values can be None, a Java LocalDate, a string, a datetime.date, a datetime.datetime, @@ -276,8 +318,7 @@ def to_j_local_date(dt: Union[None, LocalDate, str, datetime.date, datetime.date Date strings can be formatted according to the ISO 8601 date time format as 'YYYY-MM-DD'. Args: - dt (Union[None, LocalDate, str, datetime.date, datetime.datetime, numpy.datetime64, pandas.Timestamp]): - A date time value. If None is provided, None is returned. + dt (Optional[LocalDateLike]): A date time value. If None is provided, None is returned. Returns: LocalDate @@ -305,8 +346,7 @@ def to_j_local_date(dt: Union[None, LocalDate, str, datetime.date, datetime.date raise DHError(e) from e -def to_j_local_time(dt: Union[None, LocalTime, int, str, datetime.time, datetime.datetime, - numpy.datetime64, pandas.Timestamp]) -> Optional[LocalTime]: +def to_j_local_time(dt: Optional[LocalTimeLike]) -> Optional[LocalTime]: """ Converts a date time value to a Java LocalTime. Date time values can be None, a Java LocalTime, an int, a string, a datetime.time, a datetime.datetime, @@ -317,8 +357,7 @@ def to_j_local_time(dt: Union[None, LocalTime, int, str, datetime.time, datetime Time strings can be formatted as 'hh:mm:ss[.nnnnnnnnn]'. Args: - dt (Union[None, LocalTime, int, str, datetime.time, datetime.datetime, numpy.datetime64, pandas.Timestamp]): - A date time value. If None is provided, None is returned. + dt (Optional[LocalTimeLike]): A date time value. If None is provided, None is returned. Returns: LocalTime @@ -351,8 +390,7 @@ def to_j_local_time(dt: Union[None, LocalTime, int, str, datetime.time, datetime raise DHError(e) from e -def to_j_instant(dt: Union[None, Instant, int, str, datetime.datetime, numpy.datetime64, pandas.Timestamp]) -> \ - Optional[Instant]: +def to_j_instant(dt: Optional[InstantLike]) -> Optional[Instant]: """ Converts a date time value to a Java Instant. Date time values can be None, a Java Instant, an int, a string, a datetime.datetime, @@ -366,8 +404,7 @@ def to_j_instant(dt: Union[None, Instant, int, str, datetime.datetime, numpy.dat from the Epoch. Expected date ranges are used to infer the units. Args: - dt (Union[None, Instant, int, str, datetime.datetime, numpy.datetime64, pandas.Timestamp]): A date time value. - If None is provided, None is returned. + dt (Optional[InstantLike]): A date time value. If None is provided, None is returned. Returns: Instant, TypeError @@ -403,8 +440,7 @@ def to_j_instant(dt: Union[None, Instant, int, str, datetime.datetime, numpy.dat raise DHError(e) from e -def to_j_zdt(dt: Union[None, ZonedDateTime, str, datetime.datetime, numpy.datetime64, pandas.Timestamp]) -> \ - Optional[ZonedDateTime]: +def to_j_zdt(dt: Optional[ZonedDateTimeLike]) -> Optional[ZonedDateTime]: """ Converts a date time value to a Java ZonedDateTime. Date time values can be None, a Java ZonedDateTime, a string, a datetime.datetime, @@ -419,8 +455,7 @@ def to_j_zdt(dt: Union[None, ZonedDateTime, str, datetime.datetime, numpy.dateti Converting a numpy.datetime64 to a ZonedDateTime will use the Deephaven default time zone. Args: - dt (Union[None, ZonedDateTime, str, datetime.datetime, numpy.datetime64, pandas.Timestamp]): - A date time value. If None is provided, None is returned. + dt (Optional[ZonedDateTimeLike]): A date time value. If None is provided, None is returned. Returns: ZonedDateTime @@ -455,8 +490,7 @@ def to_j_zdt(dt: Union[None, ZonedDateTime, str, datetime.datetime, numpy.dateti raise DHError(e) from e -def to_j_duration(dt: Union[None, Duration, int, str, datetime.timedelta, numpy.timedelta64, pandas.Timedelta]) -> \ - Optional[Duration]: +def to_j_duration(dt: Optional[DurationLike]) -> Optional[Duration]: """ Converts a time duration value to a Java Duration, which is a unit of time in terms of clock time (24-hour days, hours, minutes, seconds, and nanoseconds). @@ -480,8 +514,7 @@ def to_j_duration(dt: Union[None, Duration, int, str, datetime.timedelta, numpy. | "-PT-6H+3M" -- parses as "+6 hours and -3 minutes" Args: - dt (Union[None, Duration, int, str, datetime.timedelta, numpy.timedelta64, pandas.Timedelta]): - A time duration value. If None is provided, None is returned. + dt (Optional[DurationLike]): A time duration value. If None is provided, None is returned. Returns: Duration @@ -515,8 +548,7 @@ def to_j_duration(dt: Union[None, Duration, int, str, datetime.timedelta, numpy. raise DHError(e) from e -def to_j_period(dt: Union[None, Period, str, datetime.timedelta, numpy.timedelta64, pandas.Timedelta]) -> \ - Optional[Period]: +def to_j_period(dt: Optional[PeriodLike]) -> Optional[Period]: """ Converts a time duration value to a Java Period, which is a unit of time in terms of calendar time (days, weeks, months, years, etc.). @@ -537,8 +569,7 @@ def to_j_period(dt: Union[None, Period, str, datetime.timedelta, numpy.timedelta | "-P1Y2M" -- -1 Year, -2 Months Args: - dt (Union[None, Period, str, datetime.timedelta, numpy.timedelta64, pandas.Timedelta]): - A Python period or period string. If None is provided, None is returned. + dt (Optional[PeriodLike]): A Python period or period string. If None is provided, None is returned. Returns: Period diff --git a/py/server/tests/test_table_factory.py b/py/server/tests/test_table_factory.py index fe66ba992ef..c4fb1cbea36 100644 --- a/py/server/tests/test_table_factory.py +++ b/py/server/tests/test_table_factory.py @@ -93,10 +93,13 @@ def test_time_table_blink(self): def test_time_table_error(self): with self.assertRaises(DHError) as cm: - t = time_table("PT00:0a:01") + time_table("PT00:0a:01") self.assertIn("DateTimeParseException", cm.exception.root_cause) + with self.assertRaises(DHError): + time_table(None) + def test_merge(self): t1 = self.test_table.update(formulas=["Timestamp=epochNanosToInstant(0L)"]) t2 = self.test_table.update(formulas=["Timestamp=nowSystem()"]) From e739b3cf4954509a0e4a9b86d68dd4fb8f06ee9c Mon Sep 17 00:00:00 2001 From: Shivam Malhotra Date: Wed, 28 Aug 2024 16:49:51 -0500 Subject: [PATCH 02/15] fix: For broken pydocs for S3 (#6000) --- py/server/deephaven/experimental/s3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/py/server/deephaven/experimental/s3.py b/py/server/deephaven/experimental/s3.py index 1fe105e7a40..ade4e5f8629 100644 --- a/py/server/deephaven/experimental/s3.py +++ b/py/server/deephaven/experimental/s3.py @@ -49,9 +49,9 @@ def __init__(self, Args: region_name (str): the region name for reading parquet files. If not provided, the default region will be - picked by the AWS SDK from 'aws.region' system property, "AWS_REGION" environment variable, the - {user.home}/.aws/credentials or {user.home}/.aws/config files, or from EC2 metadata service, if running in - EC2. + picked by the AWS SDK from 'aws.region' system property, "AWS_REGION" environment variable, the + {user.home}/.aws/credentials or {user.home}/.aws/config files, or from EC2 metadata service, if running + in EC2. max_concurrent_requests (int): the maximum number of concurrent requests for reading files, default is 256. read_ahead_count (int): the number of fragments to send asynchronous read requests for while reading the current fragment. Defaults to 32, which means fetch the next 32 fragments in advance when reading the current fragment. From d7034dd28bbefcd1d0515305448ca65e8c72ef5e Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Fri, 30 Aug 2024 09:10:55 -0400 Subject: [PATCH 03/15] feat(csharp/client): Wrappers for some Enterprise Core+ features (#6003) Adding C# client support for a few of the needed Enterprise Core+ features. --- csharp/client/DeephavenClient/Client.cs | 11 ++- .../DeephavenClient/DeephavenClient.csproj | 8 +- .../DeephavenClient/TableHandleManager.cs | 11 ++- .../dhe_client/session/DndClient.cs | 57 +++++++++++++++ .../session/DndTableHandleManager.cs | 32 ++++++++ .../dhe_client/session/SessionManager.cs | 73 +++++++++++++++++++ .../{Interop => interop}/InteropSupport.cs | 12 ++- .../TestApi/BasicInteropInteractions.cs | 0 .../{Utility => utility}/ColumnFactory.cs | 0 .../DeephavenConstants.cs | 0 .../{Utility => utility}/Schema.cs | 0 .../{Utility => utility}/TableMaker.cs | 0 12 files changed, 192 insertions(+), 12 deletions(-) create mode 100644 csharp/client/DeephavenClient/dhe_client/session/DndClient.cs create mode 100644 csharp/client/DeephavenClient/dhe_client/session/DndTableHandleManager.cs create mode 100644 csharp/client/DeephavenClient/dhe_client/session/SessionManager.cs rename csharp/client/DeephavenClient/{Interop => interop}/InteropSupport.cs (90%) rename csharp/client/DeephavenClient/{Interop => interop}/TestApi/BasicInteropInteractions.cs (100%) rename csharp/client/DeephavenClient/{Utility => utility}/ColumnFactory.cs (100%) rename csharp/client/DeephavenClient/{Utility => utility}/DeephavenConstants.cs (100%) rename csharp/client/DeephavenClient/{Utility => utility}/Schema.cs (100%) rename csharp/client/DeephavenClient/{Utility => utility}/TableMaker.cs (100%) diff --git a/csharp/client/DeephavenClient/Client.cs b/csharp/client/DeephavenClient/Client.cs index f4439a1caae..887f51448e1 100644 --- a/csharp/client/DeephavenClient/Client.cs +++ b/csharp/client/DeephavenClient/Client.cs @@ -17,13 +17,13 @@ public static Client Connect(string target, ClientOptions options) { return new Client(clientResult, manager); } - private Client(NativePtr self, TableHandleManager manager) { + private protected Client(NativePtr self, TableHandleManager manager) { Self = self; Manager = manager; } ~Client() { - ReleaseUnmanagedResources(); + ReleaseUnmanagedResources(true); } public void Close() { @@ -31,14 +31,17 @@ public void Close() { } public void Dispose() { - ReleaseUnmanagedResources(); + ReleaseUnmanagedResources(true); GC.SuppressFinalize(this); } - private void ReleaseUnmanagedResources() { + protected virtual void ReleaseUnmanagedResources(bool destructSelf) { if (!Self.TryRelease(out var old)) { return; } + if (!destructSelf) { + return; + } NativeClient.deephaven_client_Client_dtor(old); } } diff --git a/csharp/client/DeephavenClient/DeephavenClient.csproj b/csharp/client/DeephavenClient/DeephavenClient.csproj index 707e4dad819..ac6d099d291 100644 --- a/csharp/client/DeephavenClient/DeephavenClient.csproj +++ b/csharp/client/DeephavenClient/DeephavenClient.csproj @@ -1,10 +1,16 @@  - net7.0 + net8.0 enable enable True + + + + + + diff --git a/csharp/client/DeephavenClient/TableHandleManager.cs b/csharp/client/DeephavenClient/TableHandleManager.cs index 09af0ba87fc..01dc64660b0 100644 --- a/csharp/client/DeephavenClient/TableHandleManager.cs +++ b/csharp/client/DeephavenClient/TableHandleManager.cs @@ -3,7 +3,7 @@ namespace Deephaven.DeephavenClient; -public sealed class TableHandleManager : IDisposable { +public class TableHandleManager : IDisposable { internal NativePtr Self; private readonly Dictionary _subscriptions; @@ -13,18 +13,21 @@ internal TableHandleManager(NativePtr self) { } ~TableHandleManager() { - ReleaseUnmanagedResources(); + ReleaseUnmanagedResources(true); } public void Dispose() { - ReleaseUnmanagedResources(); + ReleaseUnmanagedResources(true); GC.SuppressFinalize(this); } - private void ReleaseUnmanagedResources() { + protected virtual void ReleaseUnmanagedResources(bool destructSelf) { if (!Self.TryRelease(out var old)) { return; } + if (!destructSelf) { + return; + } NativeTableHandleManager.deephaven_client_TableHandleManager_dtor(old); } diff --git a/csharp/client/DeephavenClient/dhe_client/session/DndClient.cs b/csharp/client/DeephavenClient/dhe_client/session/DndClient.cs new file mode 100644 index 00000000000..e6db330881e --- /dev/null +++ b/csharp/client/DeephavenClient/dhe_client/session/DndClient.cs @@ -0,0 +1,57 @@ +using System.Runtime.InteropServices; +using Deephaven.DeephavenClient; +using Deephaven.DeephavenClient.Interop; + +namespace Deephaven.DheClient.Session; + +public sealed class DndClient : Client { + internal new NativePtr Self; + public Int64 PqSerial; + + internal static DndClient OfNativePtr(NativePtr dndClient) { + NativeDndClient.deephaven_enterprise_session_DndClient_GetManager(dndClient, + out var dndManagerResult, out var status1); + status1.OkOrThrow(); + NativeDndClient.deephaven_enterprise_session_DndClient_PqSerial(dndClient, + out var pqSerial, out var status2); + status2.OkOrThrow(); + var dndManager = new DndTableHandleManager(dndManagerResult); + + return new DndClient(dndClient, dndManager, pqSerial); + } + + private DndClient(NativePtr self, DndTableHandleManager manager, + Int64 pqSerial) + : base(self.UnsafeCast(), manager) { + Self = self; + PqSerial = pqSerial; + } + + protected override void ReleaseUnmanagedResources(bool destructSelf) { + base.ReleaseUnmanagedResources(false); + if (!Self.TryRelease(out var old)) { + return; + } + if (!destructSelf) { + return; + } + NativeDndClient.deephaven_enterprise_session_DndClient_dtor(old); + } +} + + +internal partial class NativeDndClient { + [LibraryImport(LibraryPaths.DhEnterprise, StringMarshalling = StringMarshalling.Utf8)] + public static partial void deephaven_enterprise_session_DndClient_dtor( + NativePtr self); + + [LibraryImport(LibraryPaths.DhEnterprise, StringMarshalling = StringMarshalling.Utf8)] + public static partial void deephaven_enterprise_session_DndClient_GetManager( + NativePtr self, + out NativePtr result, out ErrorStatus status); + + [LibraryImport(LibraryPaths.DhEnterprise, StringMarshalling = StringMarshalling.Utf8)] + public static partial void deephaven_enterprise_session_DndClient_PqSerial( + NativePtr self, + out Int64 result, out ErrorStatus status); +} diff --git a/csharp/client/DeephavenClient/dhe_client/session/DndTableHandleManager.cs b/csharp/client/DeephavenClient/dhe_client/session/DndTableHandleManager.cs new file mode 100644 index 00000000000..59c88df657e --- /dev/null +++ b/csharp/client/DeephavenClient/dhe_client/session/DndTableHandleManager.cs @@ -0,0 +1,32 @@ +using System.Runtime.InteropServices; +using Deephaven.DeephavenClient; +using Deephaven.DeephavenClient.Interop; + +namespace Deephaven.DheClient.Session; + +public sealed class DndTableHandleManager : TableHandleManager { + internal new NativePtr Self; + + internal DndTableHandleManager(NativePtr self) : + base(self.UnsafeCast()) { + Self = self; + } + + protected override void ReleaseUnmanagedResources(bool destructSelf) { + base.ReleaseUnmanagedResources(false); + if (!Self.TryRelease(out var old)) { + return; + } + if (!destructSelf) { + return; + } + NativeDndTableHandleManager.deephaven_enterprise_session_DndTableHandleManager_dtor(old); + } + +} + +internal partial class NativeDndTableHandleManager { + [LibraryImport(LibraryPaths.DhEnterprise, StringMarshalling = StringMarshalling.Utf8)] + public static partial void deephaven_enterprise_session_DndTableHandleManager_dtor( + NativePtr self); +} diff --git a/csharp/client/DeephavenClient/dhe_client/session/SessionManager.cs b/csharp/client/DeephavenClient/dhe_client/session/SessionManager.cs new file mode 100644 index 00000000000..c0d79cfd7b5 --- /dev/null +++ b/csharp/client/DeephavenClient/dhe_client/session/SessionManager.cs @@ -0,0 +1,73 @@ +using System.Runtime.InteropServices; +using Deephaven.DeephavenClient.Interop; + +namespace Deephaven.DheClient.Session; + +public class SessionManager : IDisposable { + internal NativePtr Self; + + public static SessionManager FromUrl(string descriptiveName, string jsonUrl) { + NativeSessionManager.deephaven_enterprise_session_SessionManager_FromUrl(descriptiveName, + jsonUrl, out var sessionResult, out var status); + status.OkOrThrow(); + return new SessionManager(sessionResult); + } + + private SessionManager(NativePtr self) { + Self = self; + } + + ~SessionManager() { + ReleaseUnmanagedResources(); + } + + public void Close() { + Dispose(); + } + + public void Dispose() { + ReleaseUnmanagedResources(); + GC.SuppressFinalize(this); + } + + public bool PasswordAuthentication(string user, string password, string operateAs) { + NativeSessionManager.deephaven_enterprise_session_SessionManager_PasswordAuthentication( + Self, user, password, operateAs, out var result, out var status); + status.OkOrThrow(); + return (bool)result; + } + + public DndClient ConnectToPqByName(string pqName, bool removeOnClose) { + NativeSessionManager.deephaven_enterprise_session_SessionManager_ConnectToPqByName( + Self, pqName, (InteropBool)removeOnClose, out var result, out var status); + status.OkOrThrow(); + return DndClient.OfNativePtr(result); + } + + private void ReleaseUnmanagedResources() { + if (!Self.TryRelease(out var old)) { + return; + } + NativeSessionManager.deephaven_enterprise_session_SessionManager_dtor(old); + } +} + +internal partial class NativeSessionManager { + [LibraryImport(LibraryPaths.DhEnterprise, StringMarshalling = StringMarshalling.Utf8)] + public static partial void deephaven_enterprise_session_SessionManager_dtor( + NativePtr self); + + [LibraryImport(LibraryPaths.DhEnterprise, StringMarshalling = StringMarshalling.Utf8)] + public static partial void deephaven_enterprise_session_SessionManager_FromUrl(string descriptiveName, + string jsonUrl, out NativePtr result, out ErrorStatus status); + + [LibraryImport(LibraryPaths.DhEnterprise, StringMarshalling = StringMarshalling.Utf8)] + public static partial void deephaven_enterprise_session_SessionManager_PasswordAuthentication( + NativePtr self, string user, string password, string operateAs, + out InteropBool result, out ErrorStatus status); + + [LibraryImport(LibraryPaths.DhEnterprise, StringMarshalling = StringMarshalling.Utf8)] + public static partial void deephaven_enterprise_session_SessionManager_ConnectToPqByName( + NativePtr self, string pqName, InteropBool removeOnClose, + out NativePtr result, out ErrorStatus status); +} diff --git a/csharp/client/DeephavenClient/Interop/InteropSupport.cs b/csharp/client/DeephavenClient/interop/InteropSupport.cs similarity index 90% rename from csharp/client/DeephavenClient/Interop/InteropSupport.cs rename to csharp/client/DeephavenClient/interop/InteropSupport.cs index 83684b28c90..f6e67a3a311 100644 --- a/csharp/client/DeephavenClient/Interop/InteropSupport.cs +++ b/csharp/client/DeephavenClient/interop/InteropSupport.cs @@ -5,9 +5,11 @@ namespace Deephaven.DeephavenClient.Interop; -internal class LibraryPaths { - internal const string Dhcore = "dhcore"; - internal const string Dhclient = "dhclient"; +public class LibraryPaths { + public const string Dhcore = "dhcore"; + public const string Dhclient = "dhclient"; + // public const string DhEnterprise = @"dhe_client"; // does not work + public const string DhEnterprise = @"dhe_client.dll"; // works } /// @@ -32,6 +34,10 @@ public bool TryRelease(out NativePtr oldPtr) { return true; } + public NativePtr UnsafeCast() { + return new NativePtr(ptr); + } + public readonly bool IsNull => ptr == IntPtr.Zero; } diff --git a/csharp/client/DeephavenClient/Interop/TestApi/BasicInteropInteractions.cs b/csharp/client/DeephavenClient/interop/TestApi/BasicInteropInteractions.cs similarity index 100% rename from csharp/client/DeephavenClient/Interop/TestApi/BasicInteropInteractions.cs rename to csharp/client/DeephavenClient/interop/TestApi/BasicInteropInteractions.cs diff --git a/csharp/client/DeephavenClient/Utility/ColumnFactory.cs b/csharp/client/DeephavenClient/utility/ColumnFactory.cs similarity index 100% rename from csharp/client/DeephavenClient/Utility/ColumnFactory.cs rename to csharp/client/DeephavenClient/utility/ColumnFactory.cs diff --git a/csharp/client/DeephavenClient/Utility/DeephavenConstants.cs b/csharp/client/DeephavenClient/utility/DeephavenConstants.cs similarity index 100% rename from csharp/client/DeephavenClient/Utility/DeephavenConstants.cs rename to csharp/client/DeephavenClient/utility/DeephavenConstants.cs diff --git a/csharp/client/DeephavenClient/Utility/Schema.cs b/csharp/client/DeephavenClient/utility/Schema.cs similarity index 100% rename from csharp/client/DeephavenClient/Utility/Schema.cs rename to csharp/client/DeephavenClient/utility/Schema.cs diff --git a/csharp/client/DeephavenClient/Utility/TableMaker.cs b/csharp/client/DeephavenClient/utility/TableMaker.cs similarity index 100% rename from csharp/client/DeephavenClient/Utility/TableMaker.cs rename to csharp/client/DeephavenClient/utility/TableMaker.cs From 9383c3e36603c281a8800466ff236c42658e9c4c Mon Sep 17 00:00:00 2001 From: Shivam Malhotra Date: Fri, 30 Aug 2024 13:43:27 -0500 Subject: [PATCH 04/15] fix: Use parquet converted type only if logical type not set (#5997) If a column in a parquet file has both logical type and converted type set, we should prioritize logical type over converted type. --- extensions/iceberg/s3/build.gradle | 24 +++++------ .../iceberg/util/IcebergLocalStackTest.java | 11 ++--- .../iceberg/util/IcebergMinIOTest.java | 11 ++--- .../iceberg/util/IcebergToolsTest.java | 40 +++++++++++-------- .../parquet/base/ParquetFileReader.java | 28 +++++-------- 5 files changed, 52 insertions(+), 62 deletions(-) diff --git a/extensions/iceberg/s3/build.gradle b/extensions/iceberg/s3/build.gradle index bde1e84bc7f..c580989fcb2 100644 --- a/extensions/iceberg/s3/build.gradle +++ b/extensions/iceberg/s3/build.gradle @@ -26,6 +26,10 @@ dependencies { runtimeOnly libs.awssdk.sts runtimeOnly libs.awssdk.glue + testImplementation libs.junit4 + + testImplementation project(':engine-test-utils') + testImplementation libs.testcontainers testImplementation libs.testcontainers.junit.jupiter testImplementation libs.testcontainers.localstack @@ -39,20 +43,10 @@ dependencies { testRuntimeOnly libs.slf4j.simple } -test { - useJUnitPlatform { - excludeTags("testcontainers") - } -} +TestTools.addEngineOutOfBandTest(project) -tasks.register('testOutOfBand', Test) { - useJUnitPlatform { - includeTags("testcontainers") - } +testOutOfBand.dependsOn Docker.registryTask(project, 'localstack') +testOutOfBand.systemProperty 'testcontainers.localstack.image', Docker.localImageName('localstack') - dependsOn Docker.registryTask(project, 'localstack') - systemProperty 'testcontainers.localstack.image', Docker.localImageName('localstack') - - dependsOn Docker.registryTask(project, 'minio') - systemProperty 'testcontainers.minio.image', Docker.localImageName('minio') -} +testOutOfBand.dependsOn Docker.registryTask(project, 'minio') +testOutOfBand.systemProperty 'testcontainers.minio.image', Docker.localImageName('minio') \ No newline at end of file diff --git a/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergLocalStackTest.java b/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergLocalStackTest.java index 578e358985e..de15eceaa04 100644 --- a/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergLocalStackTest.java +++ b/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergLocalStackTest.java @@ -3,24 +3,21 @@ // package io.deephaven.iceberg.util; - import io.deephaven.extensions.s3.S3Instructions.Builder; import io.deephaven.extensions.s3.testlib.SingletonContainers; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; +import org.junit.BeforeClass; import software.amazon.awssdk.services.s3.S3AsyncClient; -@Tag("testcontainers") public class IcebergLocalStackTest extends IcebergToolsTest { - @BeforeAll - static void initContainer() { + @BeforeClass + public static void initContainer() { // ensure container is started so container startup time isn't associated with a specific test SingletonContainers.LocalStack.init(); } @Override - public Builder s3Instructions(Builder builder) { + public Builder s3Instructions(final Builder builder) { return SingletonContainers.LocalStack.s3Instructions(builder); } diff --git a/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergMinIOTest.java b/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergMinIOTest.java index 804d2d01746..0e789a64df2 100644 --- a/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergMinIOTest.java +++ b/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergMinIOTest.java @@ -3,20 +3,17 @@ // package io.deephaven.iceberg.util; - import io.deephaven.extensions.s3.S3Instructions.Builder; import io.deephaven.extensions.s3.testlib.SingletonContainers; import io.deephaven.stats.util.OSUtil; import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; +import org.junit.BeforeClass; import software.amazon.awssdk.services.s3.S3AsyncClient; -@Tag("testcontainers") public class IcebergMinIOTest extends IcebergToolsTest { - @BeforeAll - static void initContainer() { + @BeforeClass + public static void initContainer() { // TODO(deephaven-core#5116): MinIO testcontainers does not work on OS X Assumptions.assumeFalse(OSUtil.runningMacOS(), "OSUtil.runningMacOS()"); // ensure container is started so container startup time isn't associated with a specific test @@ -24,7 +21,7 @@ static void initContainer() { } @Override - public Builder s3Instructions(Builder builder) { + public Builder s3Instructions(final Builder builder) { return SingletonContainers.MinIO.s3Instructions(builder); } diff --git a/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergToolsTest.java b/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergToolsTest.java index 44a17942cdf..2218e9e3556 100644 --- a/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergToolsTest.java +++ b/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergToolsTest.java @@ -9,17 +9,21 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.locations.TableDataException; +import io.deephaven.engine.testutil.junit4.EngineCleanup; import io.deephaven.extensions.s3.S3Instructions; import io.deephaven.iceberg.TestCatalog.IcebergTestCatalog; import io.deephaven.iceberg.TestCatalog.IcebergTestFileIO; +import io.deephaven.test.types.OutOfBandTest; import org.apache.iceberg.Snapshot; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.io.FileIO; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.CreateBucketRequest; @@ -47,6 +51,7 @@ import static io.deephaven.iceberg.util.IcebergCatalogAdapter.SNAPSHOT_DEFINITION; import static io.deephaven.iceberg.util.IcebergCatalogAdapter.TABLES_DEFINITION; +@Category(OutOfBandTest.class) public abstract class IcebergToolsTest { private static final TableDefinition SALES_SINGLE_DEFINITION = TableDefinition.of( @@ -110,8 +115,11 @@ public abstract class IcebergToolsTest { private Catalog resourceCatalog; private FileIO resourceFileIO; - @BeforeEach - void setUp() throws ExecutionException, InterruptedException { + @Rule + public final EngineCleanup framework = new EngineCleanup(); + + @Before + public void setUp() throws ExecutionException, InterruptedException { bucket = "warehouse"; asyncClient = s3AsyncClient(); asyncClient.createBucket(CreateBucketRequest.builder().bucket(bucket).build()).get(); @@ -129,6 +137,16 @@ void setUp() throws ExecutionException, InterruptedException { .build(); } + @After + public void tearDown() throws ExecutionException, InterruptedException { + for (String key : keys) { + asyncClient.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(key).build()).get(); + } + keys.clear(); + asyncClient.deleteBucket(DeleteBucketRequest.builder().bucket(bucket).build()).get(); + asyncClient.close(); + } + private void uploadParquetFiles(final File root, final String prefixToRemove) throws ExecutionException, InterruptedException, TimeoutException { for (final File file : root.listFiles()) { @@ -175,16 +193,6 @@ private void uploadSalesRenamed() throws ExecutionException, InterruptedExceptio warehousePath); } - @AfterEach - public void tearDown() throws ExecutionException, InterruptedException { - for (String key : keys) { - asyncClient.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(key).build()).get(); - } - keys.clear(); - asyncClient.deleteBucket(DeleteBucketRequest.builder().bucket(bucket).build()).get(); - asyncClient.close(); - } - @Test public void testListNamespaces() { final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog, resourceFileIO); @@ -805,7 +813,7 @@ public void testOpenAllTypesTable() throws ExecutionException, InterruptedExcept final TableIdentifier tableId = TableIdentifier.of(ns, "all_types"); // Verify we retrieved all the rows. - final io.deephaven.engine.table.Table table = adapter.readTable(tableId, instructions); + final io.deephaven.engine.table.Table table = adapter.readTable(tableId, instructions).select(); Assert.eq(table.size(), "table.size()", 10, "10 rows in the table"); Assert.equals(table.getDefinition(), "table.getDefinition()", ALL_TYPES_DEF); } diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java index 22df95783a0..da085646a41 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java @@ -297,20 +297,13 @@ private static void buildChildren(Types.GroupBuilder builder, IteratorReference for conversions + */ + private static LogicalTypeAnnotation getLogicalTypeFromConvertedType( final ConvertedType convertedType, final SchemaElement schemaElement) throws ParquetFileReaderException { switch (convertedType) { @@ -429,17 +428,12 @@ private static LogicalTypeAnnotation getLogicalTypeAnnotation( case DATE: return LogicalTypeAnnotation.dateType(); case TIME_MILLIS: - // isAdjustedToUTC parameter is ignored while reading Parquet TIME type, so disregard it here return LogicalTypeAnnotation.timeType(true, LogicalTypeAnnotation.TimeUnit.MILLIS); case TIME_MICROS: return LogicalTypeAnnotation.timeType(true, LogicalTypeAnnotation.TimeUnit.MICROS); case TIMESTAMP_MILLIS: - // TIMESTAMP_MILLIS is always adjusted to UTC - // ref: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md return LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS); case TIMESTAMP_MICROS: - // TIMESTAMP_MICROS is always adjusted to UTC - // ref: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md return LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MICROS); case INTERVAL: return LogicalTypeAnnotation.IntervalLogicalTypeAnnotation.getInstance(); From 499ba56d2f43a420f572cd189b60b3b0a759ce87 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Fri, 30 Aug 2024 15:19:26 -0400 Subject: [PATCH 05/15] fix(cpp-client): Bump arrow version (fixes Windows build) (#6002) One of the arrow source files is missing an `#include `. I'm guessing Linux builds don't care because they pick it up, transitively, somewhere else. Unfortunately, Windows builds fail. The folks at vcpkg seem to have figured this out and have provided a patch in port version 1 of arrow 16.1.0 (hence 16.1.0#1) --- cpp-client/deephaven/vcpkg.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp-client/deephaven/vcpkg.json b/cpp-client/deephaven/vcpkg.json index c96d5038d11..e52660ae681 100644 --- a/cpp-client/deephaven/vcpkg.json +++ b/cpp-client/deephaven/vcpkg.json @@ -11,7 +11,7 @@ ], "overrides": [ - { "name": "arrow", "version": "16.0.0" }, + { "name": "arrow", "version": "16.1.0#1" }, { "name": "protobuf", "version": "4.25.1" } ] } From 0932ef2553908eec707594953adb370052884b7b Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Fri, 30 Aug 2024 15:20:46 -0400 Subject: [PATCH 06/15] feat(csharp/ExcelAddIn): First version of the Excel AddIn (#6004) --- csharp/ExcelAddIn/DeephavenExcelFunctions.cs | 67 ++++ csharp/ExcelAddIn/ExcelAddIn.csproj | 24 ++ csharp/ExcelAddIn/ExcelAddIn.csproj.user | 12 + csharp/ExcelAddIn/ExcelAddIn.sln | 31 ++ .../ExcelAddIn/Properties/launchSettings.json | 9 + csharp/ExcelAddIn/README.md | 200 +++++++++++ csharp/ExcelAddIn/StateManager.cs | 78 +++++ csharp/ExcelAddIn/exceldna/ExcelDnaHelpers.cs | 42 +++ .../ConnectionManagerDialogFactory.cs | 102 ++++++ .../factories/CredentialsDialogFactory.cs | 53 +++ .../factories/SessionBaseFactory.cs | 25 ++ csharp/ExcelAddIn/models/Credentials.cs | 37 ++ csharp/ExcelAddIn/models/Session.cs | 90 +++++ csharp/ExcelAddIn/models/SimpleModels.cs | 13 + csharp/ExcelAddIn/models/TableTriple.cs | 37 ++ .../operations/SnapshotOperation.cs | 81 +++++ .../operations/SubscribeOperation.cs | 121 +++++++ .../providers/CorePlusClientProvider.cs | 73 ++++ .../providers/DefaultSessionProvider.cs | 88 +++++ .../ExcelAddIn/providers/SessionProvider.cs | 115 ++++++ .../ExcelAddIn/providers/SessionProviders.cs | 108 ++++++ .../providers/TableHandleProvider.cs | 142 ++++++++ csharp/ExcelAddIn/util/ActionAsDisposable.cs | 21 ++ csharp/ExcelAddIn/util/ObserverContainer.cs | 52 +++ csharp/ExcelAddIn/util/Renderer.cs | 33 ++ csharp/ExcelAddIn/util/StatusOr.cs | 57 +++ csharp/ExcelAddIn/util/TableDescriptor.cs | 2 + csharp/ExcelAddIn/util/Utility.cs | 17 + csharp/ExcelAddIn/util/WorkerThread.cs | 64 ++++ .../viewmodels/ConnectionManagerDialogRow.cs | 126 +++++++ .../viewmodels/CredentialsDialogViewModel.cs | 207 +++++++++++ .../views/ConnectionManagerDialog.Designer.cs | 87 +++++ .../views/ConnectionManagerDialog.cs | 75 ++++ .../views/ConnectionManagerDialog.resx | 123 +++++++ .../views/CredentialsDialog.Designer.cs | 330 ++++++++++++++++++ csharp/ExcelAddIn/views/CredentialsDialog.cs | 59 ++++ .../ExcelAddIn/views/CredentialsDialog.resx | 120 +++++++ 37 files changed, 2921 insertions(+) create mode 100644 csharp/ExcelAddIn/DeephavenExcelFunctions.cs create mode 100644 csharp/ExcelAddIn/ExcelAddIn.csproj create mode 100644 csharp/ExcelAddIn/ExcelAddIn.csproj.user create mode 100644 csharp/ExcelAddIn/ExcelAddIn.sln create mode 100644 csharp/ExcelAddIn/Properties/launchSettings.json create mode 100644 csharp/ExcelAddIn/README.md create mode 100644 csharp/ExcelAddIn/StateManager.cs create mode 100644 csharp/ExcelAddIn/exceldna/ExcelDnaHelpers.cs create mode 100644 csharp/ExcelAddIn/factories/ConnectionManagerDialogFactory.cs create mode 100644 csharp/ExcelAddIn/factories/CredentialsDialogFactory.cs create mode 100644 csharp/ExcelAddIn/factories/SessionBaseFactory.cs create mode 100644 csharp/ExcelAddIn/models/Credentials.cs create mode 100644 csharp/ExcelAddIn/models/Session.cs create mode 100644 csharp/ExcelAddIn/models/SimpleModels.cs create mode 100644 csharp/ExcelAddIn/models/TableTriple.cs create mode 100644 csharp/ExcelAddIn/operations/SnapshotOperation.cs create mode 100644 csharp/ExcelAddIn/operations/SubscribeOperation.cs create mode 100644 csharp/ExcelAddIn/providers/CorePlusClientProvider.cs create mode 100644 csharp/ExcelAddIn/providers/DefaultSessionProvider.cs create mode 100644 csharp/ExcelAddIn/providers/SessionProvider.cs create mode 100644 csharp/ExcelAddIn/providers/SessionProviders.cs create mode 100644 csharp/ExcelAddIn/providers/TableHandleProvider.cs create mode 100644 csharp/ExcelAddIn/util/ActionAsDisposable.cs create mode 100644 csharp/ExcelAddIn/util/ObserverContainer.cs create mode 100644 csharp/ExcelAddIn/util/Renderer.cs create mode 100644 csharp/ExcelAddIn/util/StatusOr.cs create mode 100644 csharp/ExcelAddIn/util/TableDescriptor.cs create mode 100644 csharp/ExcelAddIn/util/Utility.cs create mode 100644 csharp/ExcelAddIn/util/WorkerThread.cs create mode 100644 csharp/ExcelAddIn/viewmodels/ConnectionManagerDialogRow.cs create mode 100644 csharp/ExcelAddIn/viewmodels/CredentialsDialogViewModel.cs create mode 100644 csharp/ExcelAddIn/views/ConnectionManagerDialog.Designer.cs create mode 100644 csharp/ExcelAddIn/views/ConnectionManagerDialog.cs create mode 100644 csharp/ExcelAddIn/views/ConnectionManagerDialog.resx create mode 100644 csharp/ExcelAddIn/views/CredentialsDialog.Designer.cs create mode 100644 csharp/ExcelAddIn/views/CredentialsDialog.cs create mode 100644 csharp/ExcelAddIn/views/CredentialsDialog.resx diff --git a/csharp/ExcelAddIn/DeephavenExcelFunctions.cs b/csharp/ExcelAddIn/DeephavenExcelFunctions.cs new file mode 100644 index 00000000000..d80d6627c85 --- /dev/null +++ b/csharp/ExcelAddIn/DeephavenExcelFunctions.cs @@ -0,0 +1,67 @@ +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using Deephaven.ExcelAddIn.ExcelDna; +using Deephaven.ExcelAddIn.Factories; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Operations; +using Deephaven.ExcelAddIn.Providers; +using Deephaven.ExcelAddIn.Viewmodels; +using Deephaven.ExcelAddIn.Views; +using ExcelDna.Integration; + +namespace Deephaven.ExcelAddIn; + +public static class DeephavenExcelFunctions { + private static readonly StateManager StateManager = new(); + + [ExcelCommand(MenuName = "Deephaven", MenuText = "Connections")] + public static void ShowConnectionsDialog() { + ConnectionManagerDialogFactory.CreateAndShow(StateManager); + } + + [ExcelFunction(Description = "Snapshots a table", IsThreadSafe = true)] + public static object DEEPHAVEN_SNAPSHOT(string tableDescriptor, object filter, object wantHeaders) { + if (!TryInterpretCommonArgs(tableDescriptor, filter, wantHeaders, out var td, out var filt, out var wh, out var errorText)) { + return errorText; + } + + // These two are used by ExcelDNA to share results for identical invocations. The functionName is arbitary but unique. + const string functionName = "Deephaven.ExcelAddIn.DeephavenExcelFunctions.DEEPHAVEN_SNAPSHOT"; + var parms = new[] { tableDescriptor, filter, wantHeaders }; + ExcelObservableSource eos = () => new SnapshotOperation(td!, filt, wh, StateManager); + return ExcelAsyncUtil.Observe(functionName, parms, eos); + } + + [ExcelFunction(Description = "Subscribes to a table", IsThreadSafe = true)] + public static object DEEPHAVEN_SUBSCRIBE(string tableDescriptor, object filter, object wantHeaders) { + if (!TryInterpretCommonArgs(tableDescriptor, filter, wantHeaders, out var td, out var filt, out var wh, out string errorText)) { + return errorText; + } + // These two are used by ExcelDNA to share results for identical invocations. The functionName is arbitary but unique. + const string functionName = "Deephaven.ExcelAddIn.DeephavenExcelFunctions.DEEPHAVEN_SUBSCRIBE"; + var parms = new[] { tableDescriptor, filter, wantHeaders }; + ExcelObservableSource eos = () => new SubscribeOperation(td, filt, wh, StateManager); + return ExcelAsyncUtil.Observe(functionName, parms, eos); + } + + private static bool TryInterpretCommonArgs(string tableDescriptor, object filter, object wantHeaders, + [NotNullWhen(true)]out TableTriple? tableDescriptorResult, out string filterResult, out bool wantHeadersResult, out string errorText) { + filterResult = ""; + wantHeadersResult = false; + if (!TableTriple.TryParse(tableDescriptor, out tableDescriptorResult, out errorText)) { + return false; + } + + if (!ExcelDnaHelpers.TryInterpretAs(filter, "", out filterResult)) { + errorText = "Can't interpret FILTER argument"; + return false; + } + + + if (!ExcelDnaHelpers.TryInterpretAs(wantHeaders, false, out wantHeadersResult)) { + errorText = "Can't interpret WANT_HEADERS argument"; + return false; + } + return true; + } +} diff --git a/csharp/ExcelAddIn/ExcelAddIn.csproj b/csharp/ExcelAddIn/ExcelAddIn.csproj new file mode 100644 index 00000000000..4858c64169e --- /dev/null +++ b/csharp/ExcelAddIn/ExcelAddIn.csproj @@ -0,0 +1,24 @@ + + + + net8.0-windows + enable + enable + true + True + + + + True + + + + True + + + + + + + + diff --git a/csharp/ExcelAddIn/ExcelAddIn.csproj.user b/csharp/ExcelAddIn/ExcelAddIn.csproj.user new file mode 100644 index 00000000000..f39927bb679 --- /dev/null +++ b/csharp/ExcelAddIn/ExcelAddIn.csproj.user @@ -0,0 +1,12 @@ + + + + + + Form + + + Form + + + \ No newline at end of file diff --git a/csharp/ExcelAddIn/ExcelAddIn.sln b/csharp/ExcelAddIn/ExcelAddIn.sln new file mode 100644 index 00000000000..1b006efe92c --- /dev/null +++ b/csharp/ExcelAddIn/ExcelAddIn.sln @@ -0,0 +1,31 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.7.34221.43 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ExcelAddIn", "ExcelAddIn.csproj", "{08852A0D-DB96-404E-B3CE-BF30F2AD3F74}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DeephavenClient", "..\client\DeephavenClient\DeephavenClient.csproj", "{6848407D-1CEF-4433-92F4-6047AE3D2C52}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {08852A0D-DB96-404E-B3CE-BF30F2AD3F74}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {08852A0D-DB96-404E-B3CE-BF30F2AD3F74}.Debug|Any CPU.Build.0 = Debug|Any CPU + {08852A0D-DB96-404E-B3CE-BF30F2AD3F74}.Release|Any CPU.ActiveCfg = Release|Any CPU + {08852A0D-DB96-404E-B3CE-BF30F2AD3F74}.Release|Any CPU.Build.0 = Release|Any CPU + {6848407D-1CEF-4433-92F4-6047AE3D2C52}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6848407D-1CEF-4433-92F4-6047AE3D2C52}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6848407D-1CEF-4433-92F4-6047AE3D2C52}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6848407D-1CEF-4433-92F4-6047AE3D2C52}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {A22A4DB3-DD84-46EB-96A6-7935E9E59356} + EndGlobalSection +EndGlobal diff --git a/csharp/ExcelAddIn/Properties/launchSettings.json b/csharp/ExcelAddIn/Properties/launchSettings.json new file mode 100644 index 00000000000..5a0aac58161 --- /dev/null +++ b/csharp/ExcelAddIn/Properties/launchSettings.json @@ -0,0 +1,9 @@ +{ + "profiles": { + "Excel": { + "commandName": "Executable", + "executablePath": "C:\\Program Files\\Microsoft Office\\root\\Office16\\EXCEL.EXE", + "commandLineArgs": "/x \"ExcelAddIn-AddIn64.xll\"" + } + } +} \ No newline at end of file diff --git a/csharp/ExcelAddIn/README.md b/csharp/ExcelAddIn/README.md new file mode 100644 index 00000000000..9b957b00615 --- /dev/null +++ b/csharp/ExcelAddIn/README.md @@ -0,0 +1,200 @@ +# Building the Excel Add-In on Windows 10 / 11. + +These instructions show how to install and run the Deephaven Excel Add-In +on Windows. These instructions also happen to build the Deephaven C# Client as a +side-effect. However if your goal is to build the Deephaven C# Client, +please see [repository root]/csharp/client/README.md (does not exist yet). + +We have tested these instructions on Windows 10 and 11 with Visual Studio +Community Edition. + +# Before using the Excel Add-In + +To actually use the Deephaven Excel Add-In, you will eventually need to have +at least one Community Core or Enterprise Core+ server running. You don't need +the server yet, and you can successfully follow these build instructions +without a server. However, you will eventually need a server when you want to +run it. + +If you don't have a Deephaven Community Core server installation, +you can use these instructions to build one. +https://deephaven.io/core/docs/how-to-guides/launch-build/ + +Note that it is only possible to build a server on Linux. Building a server +on Windows is not currently supported. + +For Deephaven Enterprise Core+, contact your IT administrator. + +# Building the Excel Add-In on Windows 10 / Windows 11 + +## Prerequisites + +## Build machine specifications + +In our experience following this instructions on a fresh Windows 11 VM +required a total of 125G of disk space to install and build everything. +We recommend a machine with at least 200G of free disk space in order to +leave a comfortable margin. + +Also, building the dependencies with vcpkg is very resource-intensive. +A machine that has more cores will be able to finish faster. +We recommend at least 16 cores for the build process. Note that *running* +the Excel Add-In does not require that many cores. + +## Tooling + +### Excel + +You will need a recent version of Excel installed. We recommend Office 21 +or Office 365. Note that the Add-In only works with locally-installed +versions of Excel (i.e. software installed on your computer). It does not +work with the browser-based web version of Excel. + +### .NET + +Install the .NET Core SDK, version 8.0. +Look for the "Windows | x64" link +[here](https://dotnet.microsoft.com/en-us/download/dotnet/8.0) + +### Visual Studio + +Install Visual Studio 2022 Community Edition (or Professional, or Enterprise) +from [here](https://visualstudio.microsoft.com/downloads/) + +When the installer runs, select both workloads +"Desktop development with C++" and ".NET desktop development". + +If Visual Studio is already installed, use Tools -> Get Tools and Features +to add those workloads if necessary. + +### git + +Use your preferred version of git, or install Git from +[here](https://git-scm.com/download/win) + +## C++ client + +The Deephaven Excel Add-In relies on the Deephaven C# Client, which in turn +requires the Deephaven C++ Client (Community Core version). Furthermore, if +you want to use Enterprise Core+ features, you also need the Deephaven C++ +Client for Enterprise Core+. + +The instructions below describe how to build these libraries. + +### Build the Deephaven C++ Client (Community Core version) + +Follow the instructions at [repository root]/cpp-client/README.md under the +section, under "Building the C++ client on Windows 10 / Windows 11". + +When that process is done, you will have C++ client binaries in the +directory you specified in your DHINSTALL environment variable. + +### (Optional) build the Deephaven C++ Client (Enterprise Core+ version) + +To access Enterprise features, build the Enterprise Core+ version as well. +It will also store its binaries in the same DHINSTALL directory. + +(instructions TODO) + +## Build the Excel Add-In and C# Add-In + +You can build the Add-In from inside Visual Studio or from the Visual Studio +Command Prompt. + +### From within Visual Studio + +1. Open the Visual Studio solution file +[repository root]\csharp\ExcelAddIn\ExcelAddIn.sln + +2. Click on BUILD -> Build solution + +### From the Visual Studio Command Prompt + +``` +cd [repository root]\csharp\ExcelAddIn +devenv ExcelAddIn.sln /build Release +``` + +## Run the Add-In + +### From within Visual Studio + +1. In order to actually function, the Add-In requires the C++ Client binaries + built in the above steps. The easiest thing to do is simply copy all the + binaries into your Visual Studio build directory: + +Assuming a Debug build: + +copy /Y %DHINSTALL%\bin [repository root]\csharp\ExcelAddIn\bin\Debug\net8.0-windows + +If you are doing a Release build, change "Debug" to "Release" in the above path. + +2. Inside Visual Studio Select Debug -> Start Debugging + +Visual Studio will launch Excel automatically. Excel will launch with a +Security Notice because the add-in is not signed. Click "Enable this add-in +for this session only." + +### From standalone Excel + +To install the add-in into Excel, we need put the relevant files into a +directory and then point to that directory. For simplicity, we will use +the already-established %DHINSTALL%\bin directory, which already has all the +relevant files except for the add-in's XLL file. + +``` +copy [repository root]\csharp\ExcelAddIn\bin\Debug\net8.0-windows\publish\ExcelAddIn-Addin64-packed.xll %DHINSTALL%\bin +``` + +Note the above file comes from the "publish" subdirectory. + +Then, run Excel and follow the following steps. + +1. Click on "Options" at the lower left of the window. +2. Click on "Add-ins" on the left, second from the bottom. +3. At the bottom of the screen click, near "Manage", select "Excel Add-ins" + from the pulldown, and then click "Go..." +4. In the next screen click "Browse..." +5. Navigate to your %DHINSTALL%\bin directory and click on the ExcelAddIn-Addin64-packed.xll file that you recently copied there +6. Click OK +7. Click OK + + +## Test the add-in + +### Without connecting to a Deephaven server + +1. In Excel, click on Add-ins -> Deephaven -> Connections. This should bring + up a Connections form. If so, the C# code is functioning correctly. + +2. In the following steps we deliberately use nonsense connection settings + in order to quickly trigger an error. Even thought he connection settings + are nonsense, getting an error quickly confirms that the functionality + of the C++ code is working. + +3. Inside Connections, click the "New Connection" button. A "Credentials + Editor" window will pop up. Inside this window, enter "con1" for the + Connection ID, select the "Community Core" button, and enter a nonsense + endpoint address like "abc.def" + +4. Press Test Credentials. You should immediately see an error like + "Can't get configuration constants. Error 14: DNS resolution failed for + abc.def" + +5. Enterprise users can do a similar test by selecting the Enterprise Core+ + button. Putting a nonsense protocol like abc://def in the JSON URl field + will quickly lead to an error. + +### By connecting to a Deephaven server + +If the above tests pass, the add-in is probably installed correctly. + +To test the add-in with a Deephaven server, you will need the following +information. + +1. For Community Core, you will need a Connection string in the form of + address:port. For example, 10.0.1.50:10000 + +2. For Enterprise Core+, you will need a JSON URL that references your + Core+ installation. For example + https://10.0.1.50:8123/iris/connection.json diff --git a/csharp/ExcelAddIn/StateManager.cs b/csharp/ExcelAddIn/StateManager.cs new file mode 100644 index 00000000000..c1b849b7b34 --- /dev/null +++ b/csharp/ExcelAddIn/StateManager.cs @@ -0,0 +1,78 @@ +using Deephaven.DeephavenClient.ExcelAddIn.Util; +using Deephaven.DeephavenClient; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Providers; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn; + +public class StateManager { + public readonly WorkerThread WorkerThread = WorkerThread.Create(); + private readonly SessionProviders _sessionProviders; + + public StateManager() { + _sessionProviders = new SessionProviders(WorkerThread); + } + + public IDisposable SubscribeToSessions(IObserver> observer) { + return _sessionProviders.Subscribe(observer); + } + + public IDisposable SubscribeToSession(EndpointId endpointId, IObserver> observer) { + return _sessionProviders.SubscribeToSession(endpointId, observer); + } + + public IDisposable SubscribeToCredentials(EndpointId endpointId, IObserver> observer) { + return _sessionProviders.SubscribeToCredentials(endpointId, observer); + } + + public IDisposable SubscribeToDefaultSession(IObserver> observer) { + return _sessionProviders.SubscribeToDefaultSession(observer); + } + + public IDisposable SubscribeToDefaultCredentials(IObserver> observer) { + return _sessionProviders.SubscribeToDefaultCredentials(observer); + } + + public IDisposable SubscribeToTableTriple(TableTriple descriptor, string filter, + IObserver> observer) { + // There is a chain with three elements. + // The final observer (i.e. the argument to this method) will be a subscriber to a TableHandleProvider that we create here. + // That TableHandleProvider will in turn be a subscriber to a session. + + // So: + // 1. Make a TableHandleProvider + // 2. Subscribe it to either the session provider named by the endpoint id + // or to the default session provider + // 3. Subscribe our observer to it + // 4. Return a dispose action that disposes both Subscribes + + var thp = new TableHandleProvider(WorkerThread, descriptor, filter); + var disposer1 = descriptor.EndpointId == null ? + SubscribeToDefaultSession(thp) : + SubscribeToSession(descriptor.EndpointId, thp); + var disposer2 = thp.Subscribe(observer); + + // The disposer for this needs to dispose both "inner" disposers. + return ActionAsDisposable.Create(() => { + WorkerThread.Invoke(() => { + var temp1 = Utility.Exchange(ref disposer1, null); + var temp2 = Utility.Exchange(ref disposer2, null); + temp2?.Dispose(); + temp1?.Dispose(); + }); + }); + } + + public void SetCredentials(CredentialsBase credentials) { + _sessionProviders.SetCredentials(credentials); + } + + public void SetDefaultCredentials(CredentialsBase credentials) { + _sessionProviders.SetDefaultCredentials(credentials); + } + + public void Reconnect(EndpointId id) { + _sessionProviders.Reconnect(id); + } +} diff --git a/csharp/ExcelAddIn/exceldna/ExcelDnaHelpers.cs b/csharp/ExcelAddIn/exceldna/ExcelDnaHelpers.cs new file mode 100644 index 00000000000..4ceb44805a3 --- /dev/null +++ b/csharp/ExcelAddIn/exceldna/ExcelDnaHelpers.cs @@ -0,0 +1,42 @@ +using Deephaven.ExcelAddIn.Util; +using ExcelDna.Integration; + +namespace Deephaven.ExcelAddIn.ExcelDna; + +internal class ExcelDnaHelpers { + public static bool TryInterpretAs(object value, T defaultValue, out T result) { + result = defaultValue; + if (value is ExcelMissing) { + return true; + } + + if (value is T tValue) { + result = tValue; + return true; + } + + return false; + } + + public static IObserver> WrapExcelObserver(IExcelObserver inner) { + return new ExcelObserverWrapper(inner); + } + + private class ExcelObserverWrapper(IExcelObserver inner) : IObserver> { + public void OnNext(StatusOr sov) { + if (!sov.GetValueOrStatus(out var value, out var status)) { + // Reformat the status text as an object[,] 2D array so Excel renders it as 1x1 "table". + value = new object[,] { { status } }; + } + inner.OnNext(value); + } + + public void OnCompleted() { + inner.OnCompleted(); + } + + public void OnError(Exception error) { + inner.OnError(error); + } + } +} diff --git a/csharp/ExcelAddIn/factories/ConnectionManagerDialogFactory.cs b/csharp/ExcelAddIn/factories/ConnectionManagerDialogFactory.cs new file mode 100644 index 00000000000..cdf6053f327 --- /dev/null +++ b/csharp/ExcelAddIn/factories/ConnectionManagerDialogFactory.cs @@ -0,0 +1,102 @@ +using Deephaven.ExcelAddIn.Viewmodels; +using Deephaven.ExcelAddIn.ViewModels; +using Deephaven.ExcelAddIn.Views; +using System.Diagnostics; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Factories; + +internal static class ConnectionManagerDialogFactory { + public static void CreateAndShow(StateManager sm) { + // The "new" button creates a "New/Edit Credentials" dialog + void OnNewButtonClicked() { + var cvm = CredentialsDialogViewModel.OfEmpty(); + var dialog = CredentialsDialogFactory.Create(sm, cvm); + dialog.Show(); + } + + var cmDialog = new ConnectionManagerDialog(OnNewButtonClicked); + cmDialog.Show(); + var cmso = new ConnectionManagerSessionObserver(sm, cmDialog); + var disposer = sm.SubscribeToSessions(cmso); + + cmDialog.Closed += (_, _) => { + disposer.Dispose(); + cmso.Dispose(); + }; + } +} + +internal class ConnectionManagerSessionObserver( + StateManager stateManager, + ConnectionManagerDialog cmDialog) : IObserver>, IDisposable { + private readonly List _disposables = new(); + + public void OnNext(AddOrRemove aor) { + if (!aor.IsAdd) { + // TODO(kosak) + Debug.WriteLine("Remove is not handled"); + return; + } + + var endpointId = aor.Value; + + var statusRow = new ConnectionManagerDialogRow(endpointId.Id, stateManager); + // We watch for session and credential state changes in our ID + var sessDisposable = stateManager.SubscribeToSession(endpointId, statusRow); + var credDisposable = stateManager.SubscribeToCredentials(endpointId, statusRow); + + // And we also watch for credentials changes in the default session (just to keep + // track of whether we are still the default) + var dct = new DefaultCredentialsTracker(statusRow); + var defaultCredDisposable = stateManager.SubscribeToDefaultCredentials(dct); + + // We'll do our AddRow on the GUI thread, and, while we're on the GUI thread, we'll add + // our disposables to our saved disposables. + cmDialog.Invoke(() => { + _disposables.Add(sessDisposable); + _disposables.Add(credDisposable); + _disposables.Add(defaultCredDisposable); + cmDialog.AddRow(statusRow); + }); + } + + public void Dispose() { + // Since the GUI thread is where we added these disposables, the GUI thread is where we will + // access and dispose them. + cmDialog.Invoke(() => { + var temp = _disposables.ToArray(); + _disposables.Clear(); + foreach (var disposable in temp) { + disposable.Dispose(); + } + }); + } + + public void OnCompleted() { + // TODO(kosak) + throw new NotImplementedException(); + } + + public void OnError(Exception error) { + // TODO(kosak) + throw new NotImplementedException(); + } +} + +internal class DefaultCredentialsTracker(ConnectionManagerDialogRow statusRow) : IObserver> { + public void OnNext(StatusOr value) { + statusRow.SetDefaultCredentials(value); + } + + public void OnCompleted() { + // TODO(kosak) + throw new NotImplementedException(); + } + + public void OnError(Exception error) { + // TODO(kosak) + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/csharp/ExcelAddIn/factories/CredentialsDialogFactory.cs b/csharp/ExcelAddIn/factories/CredentialsDialogFactory.cs new file mode 100644 index 00000000000..4212c250c31 --- /dev/null +++ b/csharp/ExcelAddIn/factories/CredentialsDialogFactory.cs @@ -0,0 +1,53 @@ +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Providers; +using Deephaven.ExcelAddIn.ViewModels; +using ExcelAddIn.views; + +namespace Deephaven.ExcelAddIn.Factories; + +internal static class CredentialsDialogFactory { + public static CredentialsDialog Create(StateManager sm, CredentialsDialogViewModel cvm) { + CredentialsDialog? credentialsDialog = null; + + void OnSetCredentialsButtonClicked() { + if (!cvm.TryMakeCredentials(out var newCreds, out var error)) { + ShowMessageBox(error); + return; + } + + sm.SetCredentials(newCreds); + if (cvm.IsDefault) { + sm.SetDefaultCredentials(newCreds); + } + } + + void OnTestCredentialsButtonClicked() { + if (!cvm.TryMakeCredentials(out var newCreds, out var error)) { + ShowMessageBox(error); + return; + } + + credentialsDialog!.SetTestResultsBox("Checking credentials"); + + sm.WorkerThread.Invoke(() => { + var state = "OK"; + try { + var temp = SessionBaseFactory.Create(newCreds, sm.WorkerThread); + temp.Dispose(); + } catch (Exception ex) { + state = ex.Message; + } + + credentialsDialog!.SetTestResultsBox(state); + }); + } + + // Save in captured variable so that the lambdas can access it. + credentialsDialog = new CredentialsDialog(cvm, OnSetCredentialsButtonClicked, OnTestCredentialsButtonClicked); + return credentialsDialog; + } + + private static void ShowMessageBox(string error) { + MessageBox.Show(error, "Please provide missing fields", MessageBoxButtons.OK); + } +} diff --git a/csharp/ExcelAddIn/factories/SessionBaseFactory.cs b/csharp/ExcelAddIn/factories/SessionBaseFactory.cs new file mode 100644 index 00000000000..0719e58fc2f --- /dev/null +++ b/csharp/ExcelAddIn/factories/SessionBaseFactory.cs @@ -0,0 +1,25 @@ +using Deephaven.DeephavenClient; +using Deephaven.DheClient.Session; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Factories; + +internal static class SessionBaseFactory { + public static SessionBase Create(CredentialsBase credentials, WorkerThread workerThread) { + return credentials.AcceptVisitor( + core => { + var client = Client.Connect(core.ConnectionString, new ClientOptions()); + return new CoreSession(client); + }, + + corePlus => { + var session = SessionManager.FromUrl("Deephaven Excel", corePlus.JsonUrl); + if (!session.PasswordAuthentication(corePlus.User, corePlus.Password, corePlus.OperateAs)) { + throw new Exception("Authentication failed"); + } + + return new CorePlusSession(session, workerThread); + }); + } +} diff --git a/csharp/ExcelAddIn/models/Credentials.cs b/csharp/ExcelAddIn/models/Credentials.cs new file mode 100644 index 00000000000..3368a59de04 --- /dev/null +++ b/csharp/ExcelAddIn/models/Credentials.cs @@ -0,0 +1,37 @@ +namespace Deephaven.ExcelAddIn.Models; + +public abstract class CredentialsBase(EndpointId id) { + public readonly EndpointId Id = id; + + public static CredentialsBase OfCore(EndpointId id, string connectionString) { + return new CoreCredentials(id, connectionString); + } + + public static CredentialsBase OfCorePlus(EndpointId id, string jsonUrl, string userId, + string password, string operateAs) { + return new CorePlusCredentials(id, jsonUrl, userId, password, operateAs); + } + + public abstract T AcceptVisitor(Func ofCore, + Func ofCorePlus); +} + +public sealed class CoreCredentials(EndpointId id, string connectionString) : CredentialsBase(id) { + public readonly string ConnectionString = connectionString; + + public override T AcceptVisitor(Func ofCore, Func ofCorePlus) { + return ofCore(this); + } +} + +public sealed class CorePlusCredentials(EndpointId id, string jsonUrl, string user, string password, + string operateAs) : CredentialsBase(id) { + public readonly string JsonUrl = jsonUrl; + public readonly string User = user; + public readonly string Password = password; + public readonly string OperateAs = operateAs; + + public override T AcceptVisitor(Func ofCore, Func ofCorePlus) { + return ofCorePlus(this); + } +} diff --git a/csharp/ExcelAddIn/models/Session.cs b/csharp/ExcelAddIn/models/Session.cs new file mode 100644 index 00000000000..a3cbfa1998f --- /dev/null +++ b/csharp/ExcelAddIn/models/Session.cs @@ -0,0 +1,90 @@ +using Deephaven.DeephavenClient.ExcelAddIn.Util; +using Deephaven.DeephavenClient; +using Deephaven.DheClient.Session; +using Deephaven.ExcelAddIn.Providers; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Models; + +/// +/// A "Session" is an abstraction meant to represent a Core or Core+ "session". +/// For Core, this means having a valid Client. +/// For Core+, this means having a SessionManager, through which you can subscribe to PQs and get Clients. +/// +public abstract class SessionBase : IDisposable { + /// + /// This is meant to act like a Visitor pattern with lambdas. + /// + public abstract T Visit(Func onCore, Func onCorePlus); + + public abstract void Dispose(); +} + +public sealed class CoreSession(Client client) : SessionBase { + public Client? Client = client; + + public override T Visit(Func onCore, Func onCorePlus) { + return onCore(this); + } + + public override void Dispose() { + Utility.Exchange(ref Client, null)?.Dispose(); + } +} + +public sealed class CorePlusSession(SessionManager sessionManager, WorkerThread workerThread) : SessionBase { + private SessionManager? _sessionManager = sessionManager; + private readonly Dictionary _clientProviders = new(); + + public override T Visit(Func onCore, Func onCorePlus) { + return onCorePlus(this); + } + + public IDisposable SubscribeToPq(PersistentQueryId persistentQueryId, + IObserver> observer) { + if (_sessionManager == null) { + throw new Exception("Object has been disposed"); + } + + CorePlusClientProvider? cp = null; + IDisposable? disposer = null; + + workerThread.Invoke(() => { + if (!_clientProviders.TryGetValue(persistentQueryId, out cp)) { + cp = CorePlusClientProvider.Create(workerThread, _sessionManager, persistentQueryId); + _clientProviders.Add(persistentQueryId, cp); + } + + disposer = cp.Subscribe(observer); + }); + + return ActionAsDisposable.Create(() => { + workerThread.Invoke(() => { + var old = Utility.Exchange(ref disposer, null); + // Do nothing if caller Disposes me multiple times. + if (old == null) { + return; + } + old.Dispose(); + + // Slightly weird. If "old.Dispose()" has removed the last subscriber, + // then dispose it and remove it from our dictionary. + cp!.DisposeIfEmpty(() => _clientProviders.Remove(persistentQueryId)); + }); + }); + } + + public override void Dispose() { + if (workerThread.InvokeIfRequired(Dispose)) { + return; + } + + var localCps = _clientProviders.Values.ToArray(); + _clientProviders.Clear(); + Utility.Exchange(ref _sessionManager, null)?.Dispose(); + + foreach (var cp in localCps) { + cp.Dispose(); + } + } +} diff --git a/csharp/ExcelAddIn/models/SimpleModels.cs b/csharp/ExcelAddIn/models/SimpleModels.cs new file mode 100644 index 00000000000..02a89b370b7 --- /dev/null +++ b/csharp/ExcelAddIn/models/SimpleModels.cs @@ -0,0 +1,13 @@ +namespace Deephaven.ExcelAddIn.Models; + +public record AddOrRemove(bool IsAdd, T Value) { + public static AddOrRemove OfAdd(T value) { + return new AddOrRemove(true, value); + } +} + +public record EndpointId(string Id) { + public override string ToString() => Id; +} + +public record PersistentQueryId(string Id); diff --git a/csharp/ExcelAddIn/models/TableTriple.cs b/csharp/ExcelAddIn/models/TableTriple.cs new file mode 100644 index 00000000000..95d7e7847b5 --- /dev/null +++ b/csharp/ExcelAddIn/models/TableTriple.cs @@ -0,0 +1,37 @@ +namespace Deephaven.ExcelAddIn.Models; + +public record TableTriple( + EndpointId? EndpointId, + PersistentQueryId? PersistentQueryId, + string TableName) { + + public static bool TryParse(string text, out TableTriple result, out string errorText) { + // Accepts strings of the following form + // 1. "table" (becomes null, null, "table") + // 2. "endpoint:table" (becomes endpoint, null, table) + // 3. "pq/table" (becomes null, pq, table) + // 4. "endpoint:pq/table" (becomes endpoint, pq, table) + EndpointId? epId = null; + PersistentQueryId? pqid = null; + var tableName = ""; + var colonIndex = text.IndexOf(':'); + if (colonIndex > 0) { + // cases 2 and 4: pull out the endpointId, and then reduce to cases 1 and 3 + epId = new EndpointId(text[..colonIndex]); + text = text[(colonIndex + 1)..]; + } + + var slashIndex = text.IndexOf('/'); + if (slashIndex > 0) { + // case 3: pull out the slash, and reduce to case 1 + pqid = new PersistentQueryId(text[..slashIndex]); + text = text[(slashIndex + 1)..]; + } + + tableName = text; + result = new TableTriple(epId, pqid, tableName); + errorText = ""; + // This version never fails to parse, but we leave open the option in our API to do so. + return true; + } +} diff --git a/csharp/ExcelAddIn/operations/SnapshotOperation.cs b/csharp/ExcelAddIn/operations/SnapshotOperation.cs new file mode 100644 index 00000000000..70b577de0e3 --- /dev/null +++ b/csharp/ExcelAddIn/operations/SnapshotOperation.cs @@ -0,0 +1,81 @@ +using Deephaven.DeephavenClient; +using Deephaven.DeephavenClient.ExcelAddIn.Util; +using Deephaven.ExcelAddIn.ExcelDna; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; +using ExcelDna.Integration; + +namespace Deephaven.ExcelAddIn.Operations; + +internal class SnapshotOperation : IExcelObservable, IObserver> { + private readonly TableTriple _tableDescriptor; + private readonly string _filter; + private readonly bool _wantHeaders; + private readonly StateManager _stateManager; + private readonly ObserverContainer> _observers = new(); + private readonly WorkerThread _workerThread; + private IDisposable? _filteredTableDisposer = null; + + public SnapshotOperation(TableTriple tableDescriptor, string filter, bool wantHeaders, + StateManager stateManager) { + _tableDescriptor = tableDescriptor; + _filter = filter; + _wantHeaders = wantHeaders; + _stateManager = stateManager; + // Convenience + _workerThread = _stateManager.WorkerThread; + } + + public IDisposable Subscribe(IExcelObserver observer) { + var wrappedObserver = ExcelDnaHelpers.WrapExcelObserver(observer); + _workerThread.Invoke(() => { + _observers.Add(wrappedObserver, out var isFirst); + + if (isFirst) { + _filteredTableDisposer = _stateManager.SubscribeToTableTriple(_tableDescriptor, _filter, this); + } + }); + + return ActionAsDisposable.Create(() => { + _workerThread.Invoke(() => { + _observers.Remove(wrappedObserver, out var wasLast); + if (!wasLast) { + return; + } + + Utility.Exchange(ref _filteredTableDisposer, null)?.Dispose(); + }); + }); + } + + public void OnNext(StatusOr soth) { + if (_workerThread.InvokeIfRequired(() => OnNext(soth))) { + return; + } + + if (!soth.GetValueOrStatus(out var tableHandle, out var status)) { + _observers.SendStatus(status); + return; + } + + _observers.SendStatus($"Snapshotting \"{_tableDescriptor.TableName}\""); + + try { + using var ct = tableHandle.ToClientTable(); + var result = Renderer.Render(ct, _wantHeaders); + _observers.SendValue(result); + } catch (Exception ex) { + _observers.SendStatus(ex.Message); + } + } + + void IObserver>.OnCompleted() { + // TODO(kosak): TODO + throw new NotImplementedException(); + } + + void IObserver>.OnError(Exception error) { + // TODO(kosak): TODO + throw new NotImplementedException(); + } +} diff --git a/csharp/ExcelAddIn/operations/SubscribeOperation.cs b/csharp/ExcelAddIn/operations/SubscribeOperation.cs new file mode 100644 index 00000000000..e451861546a --- /dev/null +++ b/csharp/ExcelAddIn/operations/SubscribeOperation.cs @@ -0,0 +1,121 @@ +using Deephaven.DeephavenClient; +using Deephaven.DeephavenClient.ExcelAddIn.Util; +using Deephaven.ExcelAddIn.ExcelDna; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Providers; +using Deephaven.ExcelAddIn.Util; +using ExcelDna.Integration; + +namespace Deephaven.ExcelAddIn.Operations; + +internal class SubscribeOperation : IExcelObservable, IObserver> { + private readonly TableTriple _tableDescriptor; + private readonly string _filter; + private readonly bool _wantHeaders; + private readonly StateManager _stateManager; + private readonly ObserverContainer> _observers = new(); + private readonly WorkerThread _workerThread; + private IDisposable? _filteredTableDisposer = null; + private TableHandle? _currentTableHandle = null; + private SubscriptionHandle? _currentSubHandle = null; + + public SubscribeOperation(TableTriple tableDescriptor, string filter, bool wantHeaders, + StateManager stateManager) { + _tableDescriptor = tableDescriptor; + _filter = filter; + _wantHeaders = wantHeaders; + _stateManager = stateManager; + // Convenience + _workerThread = _stateManager.WorkerThread; + } + + public IDisposable Subscribe(IExcelObserver observer) { + var wrappedObserver = ExcelDnaHelpers.WrapExcelObserver(observer); + _workerThread.Invoke(() => { + _observers.Add(wrappedObserver, out var isFirst); + + if (isFirst) { + _filteredTableDisposer = _stateManager.SubscribeToTableTriple(_tableDescriptor, _filter, this); + } + }); + + return ActionAsDisposable.Create(() => { + _workerThread.Invoke(() => { + _observers.Remove(wrappedObserver, out var wasLast); + if (!wasLast) { + return; + } + + var temp = _filteredTableDisposer; + _filteredTableDisposer = null; + temp?.Dispose(); + }); + }); + } + + public void OnNext(StatusOr soth) { + if (_workerThread.InvokeIfRequired(() => OnNext(soth))) { + return; + } + + // First tear down old state + if (_currentTableHandle != null) { + _currentTableHandle.Unsubscribe(_currentSubHandle!); + _currentSubHandle!.Dispose(); + _currentTableHandle = null; + _currentSubHandle = null; + } + + if (!soth.GetValueOrStatus(out var tableHandle, out var status)) { + _observers.SendStatus(status); + return; + } + + _observers.SendStatus($"Subscribing to \"{_tableDescriptor.TableName}\""); + + _currentTableHandle = tableHandle; + _currentSubHandle = _currentTableHandle.Subscribe(new MyTickingCallback(_observers, _wantHeaders)); + + try { + using var ct = tableHandle.ToClientTable(); + var result = Renderer.Render(ct, _wantHeaders); + _observers.SendValue(result); + } catch (Exception ex) { + _observers.SendStatus(ex.Message); + } + } + + void IObserver>.OnCompleted() { + // TODO(kosak): TODO + throw new NotImplementedException(); + } + + void IObserver>.OnError(Exception error) { + // TODO(kosak): TODO + throw new NotImplementedException(); + } + + private class MyTickingCallback : ITickingCallback { + private readonly ObserverContainer> _observers; + private readonly bool _wantHeaders; + + public MyTickingCallback(ObserverContainer> observers, + bool wantHeaders) { + _observers = observers; + _wantHeaders = wantHeaders; + } + + public void OnTick(TickingUpdate update) { + try { + var results = Renderer.Render(update.Current, _wantHeaders); + _observers.SendValue(results); + } catch (Exception e) { + _observers.SendStatus(e.Message); + } + } + + public void OnFailure(string errorText) { + _observers.SendStatus(errorText); + } + } +} diff --git a/csharp/ExcelAddIn/providers/CorePlusClientProvider.cs b/csharp/ExcelAddIn/providers/CorePlusClientProvider.cs new file mode 100644 index 00000000000..f7f9f90c7f5 --- /dev/null +++ b/csharp/ExcelAddIn/providers/CorePlusClientProvider.cs @@ -0,0 +1,73 @@ +using Deephaven.DeephavenClient.ExcelAddIn.Util; +using Deephaven.DeephavenClient; +using Deephaven.DheClient.Session; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Providers; + +/// +/// This Observable provides StatusOr<Client> objects for Core+. +/// If it can successfully connect to a PQ on Core+, it will send a Client. +/// In the future it will be an Observer of PQ up/down messages. +/// +internal class CorePlusClientProvider : IObservable>, IDisposable { + public static CorePlusClientProvider Create(WorkerThread workerThread, SessionManager sessionManager, + PersistentQueryId persistentQueryId) { + var self = new CorePlusClientProvider(workerThread); + workerThread.Invoke(() => { + try { + var dndClient = sessionManager.ConnectToPqByName(persistentQueryId.Id, false); + self._client = StatusOr.OfValue(dndClient); + } catch (Exception ex) { + self._client = StatusOr.OfStatus(ex.Message); + } + }); + return self; + } + + private readonly WorkerThread _workerThread; + private readonly ObserverContainer> _observers = new(); + private StatusOr _client = StatusOr.OfStatus("Not connected"); + + private CorePlusClientProvider(WorkerThread workerThread) { + _workerThread = workerThread; + } + + public IDisposable Subscribe(IObserver> observer) { + _workerThread.Invoke(() => { + // New observer gets added to the collection and then notified of the current status. + _observers.Add(observer, out _); + observer.OnNext(_client); + }); + + return ActionAsDisposable.Create(() => { + _workerThread.Invoke(() => { + _observers.Remove(observer, out _); + }); + }); + } + + public void Dispose() { + if (_workerThread.InvokeIfRequired(Dispose)) { + return; + } + + _ = _client.GetValueOrStatus(out var c, out _); + _client = StatusOr.OfStatus("Disposed"); + c?.Dispose(); + } + + public void DisposeIfEmpty(Action onEmpty) { + if (_workerThread.InvokeIfRequired(() => DisposeIfEmpty(onEmpty))) { + return; + } + + if (_observers.Count != 0) { + return; + } + + Dispose(); + onEmpty(); + } +} diff --git a/csharp/ExcelAddIn/providers/DefaultSessionProvider.cs b/csharp/ExcelAddIn/providers/DefaultSessionProvider.cs new file mode 100644 index 00000000000..f38f8bc8a8e --- /dev/null +++ b/csharp/ExcelAddIn/providers/DefaultSessionProvider.cs @@ -0,0 +1,88 @@ +using Deephaven.DeephavenClient.ExcelAddIn.Util; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Providers; + +internal class DefaultSessionProvider(WorkerThread workerThread) : + IObserver>, IObserver>, + IObservable>, IObservable> { + private StatusOr _credentials = StatusOr.OfStatus("[Not set]"); + private StatusOr _session = StatusOr.OfStatus("[Not connected]"); + private readonly ObserverContainer> _credentialsObservers = new(); + private readonly ObserverContainer> _sessionObservers = new(); + private SessionProvider? _parent = null; + private IDisposable? _credentialsSubDisposer = null; + private IDisposable? _sessionSubDisposer = null; + + public IDisposable Subscribe(IObserver> observer) { + workerThread.Invoke(() => { + // New observer gets added to the collection and then notified of the current status. + _credentialsObservers.Add(observer, out _); + observer.OnNext(_credentials); + }); + + return ActionAsDisposable.Create(() => { + workerThread.Invoke(() => { + _credentialsObservers.Remove(observer, out _); + }); + }); + } + + public IDisposable Subscribe(IObserver> observer) { + workerThread.Invoke(() => { + // New observer gets added to the collection and then notified of the current status. + _sessionObservers.Add(observer, out _); + observer.OnNext(_session); + }); + + return ActionAsDisposable.Create(() => { + workerThread.Invoke(() => { + _sessionObservers.Remove(observer, out _); + }); + }); + } + + public void OnNext(StatusOr value) { + if (workerThread.InvokeIfRequired(() => OnNext(value))) { + return; + } + _credentials = value; + _credentialsObservers.OnNext(_credentials); + } + + public void OnNext(StatusOr value) { + if (workerThread.InvokeIfRequired(() => OnNext(value))) { + return; + } + _session = value; + _sessionObservers.OnNext(_session); + } + + public void OnCompleted() { + // TODO(kosak) + throw new NotImplementedException(); + } + + public void OnError(Exception error) { + // TODO(kosak) + throw new NotImplementedException(); + } + + public void SetParent(SessionProvider? newParent) { + if (workerThread.InvokeIfRequired(() => SetParent(newParent))) { + return; + } + + _parent = newParent; + Utility.Exchange(ref _credentialsSubDisposer, null)?.Dispose(); + Utility.Exchange(ref _sessionSubDisposer, null)?.Dispose(); + + if (_parent == null) { + return; + } + + _credentialsSubDisposer = _parent.Subscribe((IObserver>)this); + _sessionSubDisposer = _parent.Subscribe((IObserver>)this); + } +} diff --git a/csharp/ExcelAddIn/providers/SessionProvider.cs b/csharp/ExcelAddIn/providers/SessionProvider.cs new file mode 100644 index 00000000000..35e98926360 --- /dev/null +++ b/csharp/ExcelAddIn/providers/SessionProvider.cs @@ -0,0 +1,115 @@ +using Deephaven.DeephavenClient.ExcelAddIn.Util; +using Deephaven.ExcelAddIn.Factories; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; +using System.Net; + +namespace Deephaven.ExcelAddIn.Providers; + +internal class SessionProvider(WorkerThread workerThread) : IObservable>, IObservable>, IDisposable { + private StatusOr _credentials = StatusOr.OfStatus("[Not set]"); + private StatusOr _session = StatusOr.OfStatus("[Not connected]"); + private readonly ObserverContainer> _credentialsObservers = new(); + private readonly ObserverContainer> _sessionObservers = new(); + + public void Dispose() { + // Get on the worker thread if not there already. + if (workerThread.InvokeIfRequired(Dispose)) { + return; + } + + // TODO(kosak) + // I feel like we should send an OnComplete to any remaining observers + + if (!_session.GetValueOrStatus(out var sess, out _)) { + return; + } + + _sessionObservers.SetAndSendStatus(ref _session, "Disposing"); + sess.Dispose(); + } + + /// + /// Subscribe to credentials changes + /// + /// + /// + public IDisposable Subscribe(IObserver> observer) { + workerThread.Invoke(() => { + // New observer gets added to the collection and then notified of the current status. + _credentialsObservers.Add(observer, out _); + observer.OnNext(_credentials); + }); + + return ActionAsDisposable.Create(() => { + workerThread.Invoke(() => { + _credentialsObservers.Remove(observer, out _); + }); + }); + } + + /// + /// Subscribe to session changes + /// + /// + /// + public IDisposable Subscribe(IObserver> observer) { + workerThread.Invoke(() => { + // New observer gets added to the collection and then notified of the current status. + _sessionObservers.Add(observer, out _); + observer.OnNext(_session); + }); + + return ActionAsDisposable.Create(() => { + workerThread.Invoke(() => { + _sessionObservers.Remove(observer, out _); + }); + }); + } + + public void SetCredentials(CredentialsBase credentials) { + // Get on the worker thread if not there already. + if (workerThread.InvokeIfRequired(() => SetCredentials(credentials))) { + return; + } + + // Dispose existing session + if (_session.GetValueOrStatus(out var sess, out _)) { + _sessionObservers.SetAndSendStatus(ref _session, "Disposing session"); + sess.Dispose(); + } + + _credentialsObservers.SetAndSendValue(ref _credentials, credentials); + + _sessionObservers.SetAndSendStatus(ref _session, "Trying to connect"); + + try { + var sb = SessionBaseFactory.Create(credentials, workerThread); + _sessionObservers.SetAndSendValue(ref _session, sb); + } catch (Exception ex) { + _sessionObservers.SetAndSendStatus(ref _session, ex.Message); + } + } + + public void Reconnect() { + // Get on the worker thread if not there already. + if (workerThread.InvokeIfRequired(Reconnect)) { + return; + } + + // We implement this as a SetCredentials call, with credentials we already have. + if (_credentials.GetValueOrStatus(out var creds, out _)) { + SetCredentials(creds); + } + } + + public void OnCompleted() { + // TODO(kosak) + throw new NotImplementedException(); + } + + public void OnError(Exception error) { + // TODO(kosak) + throw new NotImplementedException(); + } +} diff --git a/csharp/ExcelAddIn/providers/SessionProviders.cs b/csharp/ExcelAddIn/providers/SessionProviders.cs new file mode 100644 index 00000000000..5e5db2366e3 --- /dev/null +++ b/csharp/ExcelAddIn/providers/SessionProviders.cs @@ -0,0 +1,108 @@ +using Deephaven.DeephavenClient.ExcelAddIn.Util; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Providers; + +internal class SessionProviders(WorkerThread workerThread) : IObservable> { + private readonly DefaultSessionProvider _defaultProvider = new(workerThread); + private readonly Dictionary _providerMap = new(); + private readonly ObserverContainer> _endpointsObservers = new(); + + public IDisposable Subscribe(IObserver> observer) { + IDisposable? disposable = null; + // We need to run this on our worker thread because we want to protect + // access to our dictionary. + workerThread.Invoke(() => { + _endpointsObservers.Add(observer, out _); + // To avoid any further possibility of reentrancy while iterating over the dict, + // make a copy of the keys + var keys = _providerMap.Keys.ToArray(); + foreach (var endpointId in keys) { + observer.OnNext(AddOrRemove.OfAdd(endpointId)); + } + }); + + return ActionAsDisposable.Create(() => { + workerThread.Invoke(() => { + Utility.Exchange(ref disposable, null)?.Dispose(); + }); + }); + } + + public IDisposable SubscribeToSession(EndpointId id, IObserver> observer) { + IDisposable? disposable = null; + ApplyTo(id, sp => disposable = sp.Subscribe(observer)); + + return ActionAsDisposable.Create(() => { + workerThread.Invoke(() => { + Utility.Exchange(ref disposable, null)?.Dispose(); + }); + }); + } + + public IDisposable SubscribeToCredentials(EndpointId id, IObserver> observer) { + IDisposable? disposable = null; + ApplyTo(id, sp => disposable = sp.Subscribe(observer)); + + return ActionAsDisposable.Create(() => { + workerThread.Invoke(() => { + Utility.Exchange(ref disposable, null)?.Dispose(); + }); + }); + } + + public IDisposable SubscribeToDefaultSession(IObserver> observer) { + IDisposable? disposable = null; + workerThread.Invoke(() => { + disposable = _defaultProvider.Subscribe(observer); + }); + + return ActionAsDisposable.Create(() => { + workerThread.Invoke(() => { + Utility.Exchange(ref disposable, null)?.Dispose(); + }); + }); + } + + public IDisposable SubscribeToDefaultCredentials(IObserver> observer) { + IDisposable? disposable = null; + workerThread.Invoke(() => { + disposable = _defaultProvider.Subscribe(observer); + }); + + return ActionAsDisposable.Create(() => { + workerThread.Invoke(() => { + Utility.Exchange(ref disposable, null)?.Dispose(); + }); + }); + } + + public void SetCredentials(CredentialsBase credentials) { + ApplyTo(credentials.Id, sp => { + sp.SetCredentials(credentials); + }); + } + + public void SetDefaultCredentials(CredentialsBase credentials) { + ApplyTo(credentials.Id, _defaultProvider.SetParent); + } + + public void Reconnect(EndpointId id) { + ApplyTo(id, sp => sp.Reconnect()); + } + + private void ApplyTo(EndpointId id, Action action) { + if (workerThread.InvokeIfRequired(() => ApplyTo(id, action))) { + return; + } + + if (!_providerMap.TryGetValue(id, out var sp)) { + sp = new SessionProvider(workerThread); + _providerMap.Add(id, sp); + _endpointsObservers.OnNext(AddOrRemove.OfAdd(id)); + } + + action(sp); + } +} diff --git a/csharp/ExcelAddIn/providers/TableHandleProvider.cs b/csharp/ExcelAddIn/providers/TableHandleProvider.cs new file mode 100644 index 00000000000..e45ee7ce5df --- /dev/null +++ b/csharp/ExcelAddIn/providers/TableHandleProvider.cs @@ -0,0 +1,142 @@ +using Deephaven.DeephavenClient; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; +using Deephaven.DeephavenClient.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Providers; + +internal class TableHandleProvider( + WorkerThread workerThread, + TableTriple descriptor, + string filter) : IObserver>, IObserver>, + IObservable>, IDisposable { + + private readonly ObserverContainer> _observers = new(); + private IDisposable? _pqDisposable = null; + private StatusOr _tableHandle = StatusOr.OfStatus("[no TableHandle]"); + + public IDisposable Subscribe(IObserver> observer) { + // We need to run this on our worker thread because we want to protect + // access to our dictionary. + workerThread.Invoke(() => { + _observers.Add(observer, out _); + observer.OnNext(_tableHandle); + }); + + return ActionAsDisposable.Create(() => { + workerThread.Invoke(() => { + _observers.Remove(observer, out _); + }); + }); + } + + public void Dispose() { + // Get onto the worker thread if we're not already on it. + if (workerThread.InvokeIfRequired(Dispose)) { + return; + } + + DisposePqAndThState(); + } + + public void OnNext(StatusOr session) { + // Get onto the worker thread if we're not already on it. + if (workerThread.InvokeIfRequired(() => OnNext(session))) { + return; + } + + try { + // Dispose whatever state we had before. + DisposePqAndThState(); + + // If the new state is just a status message, make that our status and transmit to our observers + if (!session.GetValueOrStatus(out var sb, out var status)) { + _observers.SetAndSendStatus(ref _tableHandle, status); + return; + } + + // New state is a Core or CorePlus Session. + _ = sb.Visit(coreSession => { + // It's a Core session so just forward its client field to our own OnNext(Client) method. + // We test against null in the unlikely/impossible case that the session is Disposed + if (coreSession.Client != null) { + OnNext(StatusOr.OfValue(coreSession.Client)); + } + + return Unit.Instance; // Essentially a "void" value that is ignored. + }, corePlusSession => { + // It's a CorePlus session so subscribe us to its PQ observer for the appropriate PQ ID + // If no PQ id was provided, that's a problem + var pqid = descriptor.PersistentQueryId; + if (pqid == null) { + throw new Exception("PQ id is required"); + } + _observers.SetAndSendStatus(ref _tableHandle, $"Subscribing to PQ \"{pqid}\""); + _pqDisposable = corePlusSession.SubscribeToPq(pqid, this); + return Unit.Instance; + }); + } catch (Exception ex) { + _observers.SetAndSendStatus(ref _tableHandle, ex.Message); + } + } + + public void OnNext(StatusOr client) { + // Get onto the worker thread if we're not already on it. + if (workerThread.InvokeIfRequired(() => OnNext(client))) { + return; + } + + try { + // Dispose whatever state we had before. + DisposePqAndThState(); + + // If the new state is just a status message, make that our state and transmit to our observers + if (!client.GetValueOrStatus(out var cli, out var status)) { + _observers.SetAndSendStatus(ref _tableHandle, status); + return; + } + + // It's a real client so start fetching the table. First notify our observers. + _observers.SetAndSendStatus(ref _tableHandle, $"Fetching \"{descriptor.TableName}\""); + + // Now fetch the table. This might block but we're on the worker thread. In the future + // we might move this to yet another thread. + var th = cli.Manager.FetchTable(descriptor.TableName); + if (filter != "") { + // If there's a filter, take this table handle and surround it with a Where. + var temp = th; + th = temp.Where(filter); + temp.Dispose(); + } + + // Success! Make this our state and send the table handle to our observers. + _observers.SetAndSendValue(ref _tableHandle, th); + } catch (Exception ex) { + // Some exception. Make the exception message our state and send it to our observers. + _observers.SetAndSendStatus(ref _tableHandle, ex.Message); + } + } + + private void DisposePqAndThState() { + _ = _tableHandle.GetValueOrStatus(out var oldTh, out var _); + var oldPq = Utility.Exchange(ref _pqDisposable, null); + + if (oldTh != null) { + _observers.SetAndSendStatus(ref _tableHandle, "Disposing TableHandle"); + oldTh.Dispose(); + } + + if (oldPq != null) { + _observers.SetAndSendStatus(ref _tableHandle, "Disposing PQ"); + oldPq.Dispose(); + } + } + + public void OnCompleted() { + throw new NotImplementedException(); + } + + public void OnError(Exception error) { + throw new NotImplementedException(); + } +} diff --git a/csharp/ExcelAddIn/util/ActionAsDisposable.cs b/csharp/ExcelAddIn/util/ActionAsDisposable.cs new file mode 100644 index 00000000000..e195d2aa7f6 --- /dev/null +++ b/csharp/ExcelAddIn/util/ActionAsDisposable.cs @@ -0,0 +1,21 @@ +namespace Deephaven.DeephavenClient.ExcelAddIn.Util; + +internal class ActionAsDisposable : IDisposable { + public static IDisposable Create(Action action) { + return new ActionAsDisposable(action); + } + + private Action? _action; + + private ActionAsDisposable(Action action) => _action = action; + + public void Dispose() { + var temp = _action; + if (temp == null) { + return; + } + + _action = null; + temp(); + } +} diff --git a/csharp/ExcelAddIn/util/ObserverContainer.cs b/csharp/ExcelAddIn/util/ObserverContainer.cs new file mode 100644 index 00000000000..e1d0e98cf42 --- /dev/null +++ b/csharp/ExcelAddIn/util/ObserverContainer.cs @@ -0,0 +1,52 @@ +namespace Deephaven.ExcelAddIn.Util; + +public sealed class ObserverContainer : IObserver { + private readonly object _sync = new(); + private readonly HashSet> _observers = new(); + + public int Count { + get { + lock (_sync) { + return _observers.Count; + } + } + } + + public void Add(IObserver observer, out bool isFirst) { + lock (_sync) { + isFirst = _observers.Count == 0; + _observers.Add(observer); + } + } + + public void Remove(IObserver observer, out bool wasLast) { + lock (_sync) { + var removed = _observers.Remove(observer); + wasLast = removed && _observers.Count == 0; + } + } + + public void OnNext(T result) { + foreach (var observer in SafeCopyObservers()) { + observer.OnNext(result); + } + } + + public void OnError(Exception ex) { + foreach (var observer in SafeCopyObservers()) { + observer.OnError(ex); + } + } + + public void OnCompleted() { + foreach (var observer in SafeCopyObservers()) { + observer.OnCompleted(); + } + } + + private IObserver[] SafeCopyObservers() { + lock (_sync) { + return _observers.ToArray(); + } + } +} diff --git a/csharp/ExcelAddIn/util/Renderer.cs b/csharp/ExcelAddIn/util/Renderer.cs new file mode 100644 index 00000000000..b0fba4fecc7 --- /dev/null +++ b/csharp/ExcelAddIn/util/Renderer.cs @@ -0,0 +1,33 @@ +using Deephaven.DeephavenClient; + +namespace Deephaven.ExcelAddIn.Util; + +internal static class Renderer { + public static object?[,] Render(ClientTable table, bool wantHeaders) { + var numRows = table.NumRows; + var numCols = table.NumCols; + var effectiveNumRows = wantHeaders ? numRows + 1 : numRows; + var result = new object?[effectiveNumRows, numCols]; + + var headers = table.Schema.Names; + for (var colIndex = 0; colIndex != numCols; ++colIndex) { + var destIndex = 0; + if (wantHeaders) { + result[destIndex++, colIndex] = headers[colIndex]; + } + + var (col, nulls) = table.GetColumn(colIndex); + for (var i = 0; i != numRows; ++i) { + var temp = nulls[i] ? null : col.GetValue(i); + // sad hack, wrong place, inefficient + if (temp is DhDateTime dh) { + temp = dh.DateTime.ToString("s", System.Globalization.CultureInfo.InvariantCulture); + } + + result[destIndex++, colIndex] = temp; + } + } + + return result; + } +} diff --git a/csharp/ExcelAddIn/util/StatusOr.cs b/csharp/ExcelAddIn/util/StatusOr.cs new file mode 100644 index 00000000000..4aeae778e98 --- /dev/null +++ b/csharp/ExcelAddIn/util/StatusOr.cs @@ -0,0 +1,57 @@ +using System.Diagnostics.CodeAnalysis; + +namespace Deephaven.ExcelAddIn.Util; + +public sealed class StatusOr { + private readonly string? _status; + private readonly T? _value; + + public static StatusOr OfStatus(string status) { + return new StatusOr(status, default); + } + + public static StatusOr OfValue(T value) { + return new StatusOr(null, value); + } + + private StatusOr(string? status, T? value) { + _status = status; + _value = value; + } + + public bool GetValueOrStatus( + [NotNullWhen(true)]out T? value, + [NotNullWhen(false)]out string? status) { + status = _status; + value = _value; + return value != null; + } + + public U AcceptVisitor(Func onValue, Func onStatus) { + return _value != null ? onValue(_value) : onStatus(_status!); + } +} + +public static class ObserverStatusOr_Extensions { + public static void SendStatus(this IObserver> observer, string message) { + var so = StatusOr.OfStatus(message); + observer.OnNext(so); + } + + public static void SetAndSendStatus(this IObserver> observer, ref StatusOr sor, + string message) { + sor = StatusOr.OfStatus(message); + observer.OnNext(sor); + } + + public static void SendValue(this IObserver> observer, T value) { + var so = StatusOr.OfValue(value); + observer.OnNext(so); + } + + public static void SetAndSendValue(this IObserver> observer, ref StatusOr sor, + T value) { + sor = StatusOr.OfValue(value); + observer.OnNext(sor); + } +} diff --git a/csharp/ExcelAddIn/util/TableDescriptor.cs b/csharp/ExcelAddIn/util/TableDescriptor.cs new file mode 100644 index 00000000000..caa52c74e2a --- /dev/null +++ b/csharp/ExcelAddIn/util/TableDescriptor.cs @@ -0,0 +1,2 @@ +namespace Deephaven.ExcelAddIn.Util; + diff --git a/csharp/ExcelAddIn/util/Utility.cs b/csharp/ExcelAddIn/util/Utility.cs new file mode 100644 index 00000000000..0fce1f7a6f9 --- /dev/null +++ b/csharp/ExcelAddIn/util/Utility.cs @@ -0,0 +1,17 @@ + +namespace Deephaven.ExcelAddIn.Util; + +internal static class Utility { + public static T Exchange(ref T item, T newValue) { + var result = item; + item = newValue; + return result; + } +} + +public class Unit { + public static readonly Unit Instance = new Unit(); + + private Unit() { + } +} diff --git a/csharp/ExcelAddIn/util/WorkerThread.cs b/csharp/ExcelAddIn/util/WorkerThread.cs new file mode 100644 index 00000000000..dc1d2858754 --- /dev/null +++ b/csharp/ExcelAddIn/util/WorkerThread.cs @@ -0,0 +1,64 @@ +using System.Diagnostics; + +namespace Deephaven.ExcelAddIn.Util; + +public class WorkerThread { + public static WorkerThread Create() { + var result = new WorkerThread(); + var t = new Thread(result.Doit) { IsBackground = true }; + result._thisThread = t; + t.Start(); + return result; + } + + private readonly object _sync = new(); + private readonly Queue _queue = new(); + private Thread? _thisThread; + + private WorkerThread() { + } + + public void Invoke(Action action) { + if (!InvokeIfRequired(action)) { + action(); + } + } + + public bool InvokeIfRequired(Action action) { + if (ReferenceEquals(Thread.CurrentThread, _thisThread)) { + // Appending to thread queue was not required. Return false. + return false; + } + + lock (_sync) { + _queue.Enqueue(action); + if (_queue.Count == 1) { + // Only need to pulse on transition from 0 to 1, because the + // Doit method only Waits if the queue is empty. + Monitor.PulseAll(_sync); + } + } + + // Appending to thread queue was required. + return true; + } + + private void Doit() { + while (true) { + Action action; + lock (_sync) { + while (_queue.Count == 0) { + Monitor.Wait(_sync); + } + + action = _queue.Dequeue(); + } + + try { + action(); + } catch (Exception ex) { + Debug.WriteLine($"Swallowing exception {ex}"); + } + } + } +} diff --git a/csharp/ExcelAddIn/viewmodels/ConnectionManagerDialogRow.cs b/csharp/ExcelAddIn/viewmodels/ConnectionManagerDialogRow.cs new file mode 100644 index 00000000000..0e544131282 --- /dev/null +++ b/csharp/ExcelAddIn/viewmodels/ConnectionManagerDialogRow.cs @@ -0,0 +1,126 @@ +using Deephaven.ExcelAddIn.Factories; +using Deephaven.ExcelAddIn.ViewModels; +using System.ComponentModel; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.Viewmodels; + +public sealed class ConnectionManagerDialogRow(string id, StateManager stateManager) : + IObserver>, IObserver>, + INotifyPropertyChanged { + public event PropertyChangedEventHandler? PropertyChanged; + + private readonly object _sync = new(); + private StatusOr _credentials = StatusOr.OfStatus("[Not set]"); + private StatusOr _session = StatusOr.OfStatus("[Not connected]"); + private StatusOr _defaultCredentials = StatusOr.OfStatus("[Not set]"); + + public string Id { get; init; } = id; + + public string Status { + get { + var session = GetSessionSynced(); + // If we have a valid session, return "[Connected]", otherwise pass through the status text we have. + return session.AcceptVisitor( + _ => "[Connected]", + status => status); + } + } + + public string ServerType { + get { + var creds = GetCredentialsSynced(); + // Nested AcceptVisitor!! + // If we have valid credentials, determine whether they are for Core or Core+ and return the appropriate string. + // Otherwise (if we have invalid credentials), ignore their status text and just say "[Unknown]". + return creds.AcceptVisitor( + crs => crs.AcceptVisitor(_ => "Core", _ => "Core+"), + _ => "[Unknown]"); + + } + } + + public bool IsDefault => + _credentials.GetValueOrStatus(out var creds1, out _) && + _defaultCredentials.GetValueOrStatus(out var creds2, out _) && + creds1.Id == creds2.Id; + + public void SettingsClicked() { + var creds = GetCredentialsSynced(); + // If we have valid credentials, + var cvm = creds.AcceptVisitor( + crs => CredentialsDialogViewModel.OfIdAndCredentials(Id, crs), + _ => CredentialsDialogViewModel.OfIdButOtherwiseEmpty(Id)); + var cd = CredentialsDialogFactory.Create(stateManager, cvm); + cd.Show(); + } + + public void ReconnectClicked() { + stateManager.Reconnect(new EndpointId(Id)); + } + + public void IsDefaultClicked() { + // If the box is already checked, do nothing. + if (IsDefault) { + return; + } + + // If we don't have credentials, then we can't make them the default. + if (!_credentials.GetValueOrStatus(out var creds, out _)) { + return; + } + + stateManager.SetDefaultCredentials(creds); + } + + public void OnNext(StatusOr value) { + lock (_sync) { + _credentials = value; + } + + OnPropertyChanged(nameof(ServerType)); + OnPropertyChanged(nameof(IsDefault)); + } + + public void OnNext(StatusOr value) { + lock (_sync) { + _session = value; + } + + OnPropertyChanged(nameof(Status)); + } + + public void SetDefaultCredentials(StatusOr creds) { + lock (_sync) { + _defaultCredentials = creds; + } + OnPropertyChanged(nameof(IsDefault)); + } + + public void OnCompleted() { + // TODO(kosak) + throw new NotImplementedException(); + } + + public void OnError(Exception error) { + // TODO(kosak) + throw new NotImplementedException(); + } + + private StatusOr GetCredentialsSynced() { + lock (_sync) { + return _credentials; + } + } + + private StatusOr GetSessionSynced() { + lock (_sync) { + return _session; + } + } + + private void OnPropertyChanged(string name) { + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(name)); + } +} diff --git a/csharp/ExcelAddIn/viewmodels/CredentialsDialogViewModel.cs b/csharp/ExcelAddIn/viewmodels/CredentialsDialogViewModel.cs new file mode 100644 index 00000000000..c25f6b2faa7 --- /dev/null +++ b/csharp/ExcelAddIn/viewmodels/CredentialsDialogViewModel.cs @@ -0,0 +1,207 @@ +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using Deephaven.ExcelAddIn.Models; +using Deephaven.ExcelAddIn.Util; + +namespace Deephaven.ExcelAddIn.ViewModels; + +public sealed class CredentialsDialogViewModel : INotifyPropertyChanged { + public static CredentialsDialogViewModel OfEmpty() { + return new CredentialsDialogViewModel(); + } + + public static CredentialsDialogViewModel OfIdButOtherwiseEmpty(string id) { + return new CredentialsDialogViewModel { Id = id }; + } + + public static CredentialsDialogViewModel OfIdAndCredentials(string id, CredentialsBase credentials) { + var result = new CredentialsDialogViewModel { Id = id }; + _ = credentials.AcceptVisitor( + core => { + result._isCorePlus = false; + result.ConnectionString = core.ConnectionString; + return Unit.Instance; + }, + corePlus => { + result._isCorePlus = true; + result.JsonUrl = corePlus.JsonUrl; + result.UserId = corePlus.User; + result.Password = corePlus.Password; + result.OperateAs = corePlus.OperateAs; + return Unit.Instance; + }); + + return result; + } + + private string _id = ""; + private bool _isDefault = false; + private bool _isCorePlus = true; + + // Core properties + private string _connectionString = ""; + + // Core+ properties + private string _jsonUrl = ""; + private string _userId = ""; + private string _password = ""; + private string _operateAs = ""; + + public event PropertyChangedEventHandler? PropertyChanged; + + public bool TryMakeCredentials([NotNullWhen(true)] out CredentialsBase? result, + [NotNullWhen(false)] out string? errorText) { + result = null; + errorText = null; + + var missingFields = new List(); + void CheckMissing(string field, string name) { + if (field.Length == 0) { + missingFields.Add(name); + } + } + + CheckMissing(_id, "Connection Id"); + + if (!_isCorePlus) { + CheckMissing(_connectionString, "Connection String"); + } else { + CheckMissing(_jsonUrl, "JSON URL"); + CheckMissing(_userId, "User Id"); + CheckMissing(_password, "Password"); + CheckMissing(_operateAs, "Operate As"); + } + + if (missingFields.Count > 0) { + errorText = string.Join(", ", missingFields); + return false; + } + + var epId = new EndpointId(_id); + result = _isCorePlus + ? CredentialsBase.OfCorePlus(epId, _jsonUrl, _userId, _password, _operateAs) + : CredentialsBase.OfCore(epId, _connectionString); + return true; + } + + public string Id { + get => _id; + set { + if (value == _id) { + return; + } + + _id = value; + OnPropertyChanged(); + } + } + + public bool IsDefault { + get => _isDefault; + set { + if (value == _isDefault) { + return; + } + + _isDefault = value; + OnPropertyChanged(); + } + } + + /** + * I don't know if I have to do it this way, but I bind IsCore and IsCorePlus to the + * same underlying variable. The property "IsCore" maps to the inverse of the variable + * _isCorePlus, meanwhile the property "IsCorePlus" maps to the normal sense of the + * variable. Setters on either one trigger property change events for both. + */ + public bool IsCore { + get => !_isCorePlus; + set { + if (_isCorePlus == !value) { + return; + } + + _isCorePlus = !value; + OnPropertyChanged(); + OnPropertyChanged(nameof(IsCorePlus)); + } + } + + public bool IsCorePlus { + get => _isCorePlus; + set { + if (_isCorePlus == value) { + return; + } + + _isCorePlus = value; + OnPropertyChanged(); + OnPropertyChanged(nameof(IsCore)); + } + } + + public string ConnectionString { + get => _connectionString; + set { + if (_connectionString == value) { + return; + } + + _connectionString = value; + OnPropertyChanged(); + } + } + + public string JsonUrl { + get => _jsonUrl; + set { + if (_jsonUrl == value) { + return; + } + + _jsonUrl = value; + OnPropertyChanged(); + } + } + + public string UserId { + get => _userId; + set { + if (_userId == value) { + return; + } + + _userId = value; + OnPropertyChanged(); + } + } + + public string Password { + get => _password; + set { + if (_password == value) { + return; + } + + _password = value; + OnPropertyChanged(); + } + } + + public string OperateAs { + get => _operateAs; + set { + if (_operateAs == value) { + return; + } + + _operateAs = value; + OnPropertyChanged(); + } + } + + private void OnPropertyChanged([CallerMemberName] string? name = null) { + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(name)); + } +} diff --git a/csharp/ExcelAddIn/views/ConnectionManagerDialog.Designer.cs b/csharp/ExcelAddIn/views/ConnectionManagerDialog.Designer.cs new file mode 100644 index 00000000000..fee2096228f --- /dev/null +++ b/csharp/ExcelAddIn/views/ConnectionManagerDialog.Designer.cs @@ -0,0 +1,87 @@ +namespace Deephaven.ExcelAddIn.Views { + partial class ConnectionManagerDialog { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) { + if (disposing && (components != null)) { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() { + colorDialog1 = new ColorDialog(); + dataGridView1 = new DataGridView(); + newButton = new Button(); + connectionsLabel = new Label(); + ((System.ComponentModel.ISupportInitialize)dataGridView1).BeginInit(); + SuspendLayout(); + // + // dataGridView1 + // + dataGridView1.AllowUserToAddRows = false; + dataGridView1.AllowUserToDeleteRows = false; + dataGridView1.ColumnHeadersHeightSizeMode = DataGridViewColumnHeadersHeightSizeMode.AutoSize; + dataGridView1.Location = new Point(68, 83); + dataGridView1.Name = "dataGridView1"; + dataGridView1.ReadOnly = true; + dataGridView1.RowHeadersWidth = 62; + dataGridView1.Size = new Size(979, 454); + dataGridView1.TabIndex = 0; + // + // newButton + // + newButton.Location = new Point(869, 560); + newButton.Name = "newButton"; + newButton.Size = new Size(178, 34); + newButton.TabIndex = 1; + newButton.Text = "New Connection"; + newButton.UseVisualStyleBackColor = true; + newButton.Click += newButton_Click; + // + // connectionsLabel + // + connectionsLabel.AutoSize = true; + connectionsLabel.Font = new Font("Segoe UI", 12F, FontStyle.Regular, GraphicsUnit.Point, 0); + connectionsLabel.Location = new Point(68, 33); + connectionsLabel.Name = "connectionsLabel"; + connectionsLabel.Size = new Size(147, 32); + connectionsLabel.TabIndex = 2; + connectionsLabel.Text = "Connections"; + // + // ConnectionManagerDialog + // + AutoScaleDimensions = new SizeF(10F, 25F); + AutoScaleMode = AutoScaleMode.Font; + ClientSize = new Size(1115, 615); + Controls.Add(connectionsLabel); + Controls.Add(newButton); + Controls.Add(dataGridView1); + Name = "ConnectionManagerDialog"; + Text = "Connection Manager"; + ((System.ComponentModel.ISupportInitialize)dataGridView1).EndInit(); + ResumeLayout(false); + PerformLayout(); + } + + #endregion + + private ColorDialog colorDialog1; + private DataGridView dataGridView1; + private Button newButton; + private Label connectionsLabel; + } +} \ No newline at end of file diff --git a/csharp/ExcelAddIn/views/ConnectionManagerDialog.cs b/csharp/ExcelAddIn/views/ConnectionManagerDialog.cs new file mode 100644 index 00000000000..750fd39e0de --- /dev/null +++ b/csharp/ExcelAddIn/views/ConnectionManagerDialog.cs @@ -0,0 +1,75 @@ +using Deephaven.ExcelAddIn.Viewmodels; + +namespace Deephaven.ExcelAddIn.Views; + +public partial class ConnectionManagerDialog : Form { + private const string IsDefaultColumnName = "IsDefault"; + private const string SettingsButtonColumnName = "settings_button_column"; + private const string ReconnectButtonColumnName = "reconnect_button_column"; + private readonly Action _onNewButtonClicked; + private readonly BindingSource _bindingSource = new(); + + public ConnectionManagerDialog(Action onNewButtonClicked) { + _onNewButtonClicked = onNewButtonClicked; + + InitializeComponent(); + + _bindingSource.DataSource = typeof(ConnectionManagerDialogRow); + dataGridView1.DataSource = _bindingSource; + + var settingsButtonColumn = new DataGridViewButtonColumn { + Name = SettingsButtonColumnName, + HeaderText = "Credentials", + Text = "Edit", + UseColumnTextForButtonValue = true + }; + + var reconnectButtonColumn = new DataGridViewButtonColumn { + Name = ReconnectButtonColumnName, + HeaderText = "Reconnect", + Text = "Reconnect", + UseColumnTextForButtonValue = true + }; + + dataGridView1.Columns.Add(settingsButtonColumn); + dataGridView1.Columns.Add(reconnectButtonColumn); + + dataGridView1.CellClick += DataGridView1_CellClick; + } + + public void AddRow(ConnectionManagerDialogRow row) { + _bindingSource.Add(row); + } + + private void DataGridView1_CellClick(object? sender, DataGridViewCellEventArgs e) { + if (e.RowIndex < 0) { + return; + } + + if (_bindingSource[e.RowIndex] is not ConnectionManagerDialogRow row) { + return; + } + var name = dataGridView1.Columns[e.ColumnIndex].Name; + + switch (name) { + case SettingsButtonColumnName: { + row.SettingsClicked(); + break; + } + + case ReconnectButtonColumnName: { + row.ReconnectClicked(); + break; + } + + case IsDefaultColumnName: { + row.IsDefaultClicked(); + break; + } + } + } + + private void newButton_Click(object sender, EventArgs e) { + _onNewButtonClicked(); + } +} diff --git a/csharp/ExcelAddIn/views/ConnectionManagerDialog.resx b/csharp/ExcelAddIn/views/ConnectionManagerDialog.resx new file mode 100644 index 00000000000..b3e33e7e100 --- /dev/null +++ b/csharp/ExcelAddIn/views/ConnectionManagerDialog.resx @@ -0,0 +1,123 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + 17, 17 + + \ No newline at end of file diff --git a/csharp/ExcelAddIn/views/CredentialsDialog.Designer.cs b/csharp/ExcelAddIn/views/CredentialsDialog.Designer.cs new file mode 100644 index 00000000000..a554e61e2b2 --- /dev/null +++ b/csharp/ExcelAddIn/views/CredentialsDialog.Designer.cs @@ -0,0 +1,330 @@ +namespace ExcelAddIn.views { + partial class CredentialsDialog { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) { + if (disposing && (components != null)) { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() { + flowLayoutPanel1 = new FlowLayoutPanel(); + corePlusPanel = new Panel(); + operateAsBox = new TextBox(); + passwordBox = new TextBox(); + label5 = new Label(); + label4 = new Label(); + userIdBox = new TextBox(); + userIdLabel = new Label(); + jsonUrlBox = new TextBox(); + label3 = new Label(); + corePanel = new Panel(); + connectionStringBox = new TextBox(); + label2 = new Label(); + finalPanel = new Panel(); + testResultsTextBox = new TextBox(); + testResultsLabel = new Label(); + testCredentialsButton = new Button(); + setCredentialsButton = new Button(); + isCorePlusRadioButton = new RadioButton(); + isCoreRadioButton = new RadioButton(); + endpointIdBox = new TextBox(); + label1 = new Label(); + connectionTypeGroup = new GroupBox(); + makeDefaultCheckBox = new CheckBox(); + flowLayoutPanel1.SuspendLayout(); + corePlusPanel.SuspendLayout(); + corePanel.SuspendLayout(); + finalPanel.SuspendLayout(); + connectionTypeGroup.SuspendLayout(); + SuspendLayout(); + // + // flowLayoutPanel1 + // + flowLayoutPanel1.Controls.Add(corePlusPanel); + flowLayoutPanel1.Controls.Add(corePanel); + flowLayoutPanel1.Controls.Add(finalPanel); + flowLayoutPanel1.Location = new Point(28, 160); + flowLayoutPanel1.Name = "flowLayoutPanel1"; + flowLayoutPanel1.Size = new Size(994, 531); + flowLayoutPanel1.TabIndex = 200; + // + // corePlusPanel + // + corePlusPanel.Controls.Add(operateAsBox); + corePlusPanel.Controls.Add(passwordBox); + corePlusPanel.Controls.Add(label5); + corePlusPanel.Controls.Add(label4); + corePlusPanel.Controls.Add(userIdBox); + corePlusPanel.Controls.Add(userIdLabel); + corePlusPanel.Controls.Add(jsonUrlBox); + corePlusPanel.Controls.Add(label3); + corePlusPanel.Location = new Point(3, 3); + corePlusPanel.Name = "corePlusPanel"; + corePlusPanel.Size = new Size(991, 242); + corePlusPanel.TabIndex = 210; + // + // operateAsBox + // + operateAsBox.Location = new Point(189, 183); + operateAsBox.Name = "operateAsBox"; + operateAsBox.Size = new Size(768, 31); + operateAsBox.TabIndex = 214; + // + // passwordBox + // + passwordBox.Location = new Point(189, 130); + passwordBox.Name = "passwordBox"; + passwordBox.PasswordChar = '●'; + passwordBox.Size = new Size(768, 31); + passwordBox.TabIndex = 213; + // + // label5 + // + label5.AutoSize = true; + label5.Location = new Point(18, 189); + label5.Name = "label5"; + label5.Size = new Size(96, 25); + label5.TabIndex = 5; + label5.Text = "OperateAs"; + // + // label4 + // + label4.AutoSize = true; + label4.Location = new Point(18, 136); + label4.Name = "label4"; + label4.Size = new Size(87, 25); + label4.TabIndex = 4; + label4.Text = "Password"; + // + // userIdBox + // + userIdBox.Location = new Point(189, 82); + userIdBox.Name = "userIdBox"; + userIdBox.Size = new Size(768, 31); + userIdBox.TabIndex = 212; + // + // userIdLabel + // + userIdLabel.AutoSize = true; + userIdLabel.Location = new Point(18, 88); + userIdLabel.Name = "userIdLabel"; + userIdLabel.Size = new Size(63, 25); + userIdLabel.TabIndex = 2; + userIdLabel.Text = "UserId"; + // + // jsonUrlBox + // + jsonUrlBox.Location = new Point(189, 33); + jsonUrlBox.Name = "jsonUrlBox"; + jsonUrlBox.Size = new Size(768, 31); + jsonUrlBox.TabIndex = 211; + // + // label3 + // + label3.AutoSize = true; + label3.Location = new Point(18, 33); + label3.Name = "label3"; + label3.Size = new Size(91, 25); + label3.TabIndex = 0; + label3.Text = "JSON URL"; + // + // corePanel + // + corePanel.Controls.Add(connectionStringBox); + corePanel.Controls.Add(label2); + corePanel.Location = new Point(3, 251); + corePanel.Name = "corePanel"; + corePanel.Size = new Size(991, 76); + corePanel.TabIndex = 220; + // + // connectionStringBox + // + connectionStringBox.Location = new Point(189, 20); + connectionStringBox.Name = "connectionStringBox"; + connectionStringBox.Size = new Size(768, 31); + connectionStringBox.TabIndex = 221; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new Point(18, 26); + label2.Name = "label2"; + label2.Size = new Size(153, 25); + label2.TabIndex = 0; + label2.Text = "Connection String"; + // + // finalPanel + // + finalPanel.Controls.Add(makeDefaultCheckBox); + finalPanel.Controls.Add(testResultsTextBox); + finalPanel.Controls.Add(testResultsLabel); + finalPanel.Controls.Add(testCredentialsButton); + finalPanel.Controls.Add(setCredentialsButton); + finalPanel.Location = new Point(3, 333); + finalPanel.Name = "finalPanel"; + finalPanel.Size = new Size(991, 132); + finalPanel.TabIndex = 230; + // + // testResultsTextBox + // + testResultsTextBox.Location = new Point(189, 17); + testResultsTextBox.Name = "testResultsTextBox"; + testResultsTextBox.ReadOnly = true; + testResultsTextBox.Size = new Size(768, 31); + testResultsTextBox.TabIndex = 7; + // + // testResultsLabel + // + testResultsLabel.AutoSize = true; + testResultsLabel.Location = new Point(125, 47); + testResultsLabel.Name = "testResultsLabel"; + testResultsLabel.Size = new Size(0, 25); + testResultsLabel.TabIndex = 6; + // + // testCredentialsButton + // + testCredentialsButton.Location = new Point(8, 17); + testCredentialsButton.Name = "testCredentialsButton"; + testCredentialsButton.Size = new Size(175, 34); + testCredentialsButton.TabIndex = 231; + testCredentialsButton.Text = "Test Credentials"; + testCredentialsButton.UseVisualStyleBackColor = true; + testCredentialsButton.Click += testCredentialsButton_Click; + // + // setCredentialsButton + // + setCredentialsButton.Location = new Point(757, 65); + setCredentialsButton.Name = "setCredentialsButton"; + setCredentialsButton.Size = new Size(200, 34); + setCredentialsButton.TabIndex = 232; + setCredentialsButton.Text = "Set Credentials"; + setCredentialsButton.UseVisualStyleBackColor = true; + setCredentialsButton.Click += setCredentialsButton_Click; + // + // isCorePlusRadioButton + // + isCorePlusRadioButton.AutoSize = true; + isCorePlusRadioButton.Location = new Point(6, 39); + isCorePlusRadioButton.Name = "isCorePlusRadioButton"; + isCorePlusRadioButton.Size = new Size(169, 29); + isCorePlusRadioButton.TabIndex = 110; + isCorePlusRadioButton.TabStop = true; + isCorePlusRadioButton.Text = "Enterprise Core+"; + isCorePlusRadioButton.UseVisualStyleBackColor = true; + // + // isCoreRadioButton + // + isCoreRadioButton.AutoSize = true; + isCoreRadioButton.Location = new Point(195, 39); + isCoreRadioButton.Name = "isCoreRadioButton"; + isCoreRadioButton.Size = new Size(172, 29); + isCoreRadioButton.TabIndex = 111; + isCoreRadioButton.TabStop = true; + isCoreRadioButton.Text = "Community Core"; + isCoreRadioButton.UseVisualStyleBackColor = true; + // + // endpointIdBox + // + endpointIdBox.Location = new Point(220, 19); + endpointIdBox.Name = "endpointIdBox"; + endpointIdBox.Size = new Size(768, 31); + endpointIdBox.TabIndex = 1; + // + // label1 + // + label1.AutoSize = true; + label1.Location = new Point(34, 22); + label1.Name = "label1"; + label1.Size = new Size(125, 25); + label1.TabIndex = 5; + label1.Text = "Connection ID"; + // + // connectionTypeGroup + // + connectionTypeGroup.Controls.Add(isCorePlusRadioButton); + connectionTypeGroup.Controls.Add(isCoreRadioButton); + connectionTypeGroup.Location = new Point(28, 74); + connectionTypeGroup.Name = "connectionTypeGroup"; + connectionTypeGroup.Size = new Size(588, 80); + connectionTypeGroup.TabIndex = 100; + connectionTypeGroup.TabStop = false; + connectionTypeGroup.Text = "Connection Type"; + // + // makeDefaultCheckBox + // + makeDefaultCheckBox.AutoSize = true; + makeDefaultCheckBox.Location = new Point(599, 70); + makeDefaultCheckBox.Name = "makeDefaultCheckBox"; + makeDefaultCheckBox.Size = new Size(143, 29); + makeDefaultCheckBox.TabIndex = 234; + makeDefaultCheckBox.Text = "Make Default"; + makeDefaultCheckBox.UseVisualStyleBackColor = true; + // + // CredentialsDialog + // + AutoScaleDimensions = new SizeF(10F, 25F); + AutoScaleMode = AutoScaleMode.Font; + ClientSize = new Size(1086, 714); + Controls.Add(connectionTypeGroup); + Controls.Add(label1); + Controls.Add(endpointIdBox); + Controls.Add(flowLayoutPanel1); + Name = "CredentialsDialog"; + Text = "Credentials Editor"; + flowLayoutPanel1.ResumeLayout(false); + corePlusPanel.ResumeLayout(false); + corePlusPanel.PerformLayout(); + corePanel.ResumeLayout(false); + corePanel.PerformLayout(); + finalPanel.ResumeLayout(false); + finalPanel.PerformLayout(); + connectionTypeGroup.ResumeLayout(false); + connectionTypeGroup.PerformLayout(); + ResumeLayout(false); + PerformLayout(); + } + + #endregion + + private FlowLayoutPanel flowLayoutPanel1; + private RadioButton isCorePlusRadioButton; + private RadioButton isCoreRadioButton; + private Button setCredentialsButton; + private TextBox endpointIdBox; + private Label label1; + private GroupBox connectionTypeGroup; + private Panel corePlusPanel; + private Panel corePanel; + private Label label3; + private Label label2; + private TextBox jsonUrlBox; + private TextBox connectionStringBox; + private Label label4; + private TextBox userIdBox; + private Label userIdLabel; + private TextBox operateAsBox; + private TextBox passwordBox; + private Label label5; + private Panel finalPanel; + private Button testCredentialsButton; + private Label testResultsLabel; + private TextBox testResultsTextBox; + private CheckBox makeDefaultCheckBox; + } +} \ No newline at end of file diff --git a/csharp/ExcelAddIn/views/CredentialsDialog.cs b/csharp/ExcelAddIn/views/CredentialsDialog.cs new file mode 100644 index 00000000000..1453b8703c4 --- /dev/null +++ b/csharp/ExcelAddIn/views/CredentialsDialog.cs @@ -0,0 +1,59 @@ +using Deephaven.ExcelAddIn.ViewModels; + +namespace ExcelAddIn.views { + public partial class CredentialsDialog : Form { + private readonly Action _onSetCredentialsButtonClicked; + private readonly Action _onTestCredentialsButtonClicked; + + public CredentialsDialog(CredentialsDialogViewModel vm, Action onSetCredentialsButtonClicked, + Action onTestCredentialsButtonClicked) { + _onSetCredentialsButtonClicked = onSetCredentialsButtonClicked; + _onTestCredentialsButtonClicked = onTestCredentialsButtonClicked; + + InitializeComponent(); + // Need to fire these bindings on property changed rather than simply on validation, + // because on validation is not responsive enough. Also, painful technical note: + // being a member of connectionTypeGroup *also* ensures that at most one of these buttons + // are checked. So you might think databinding is not necessary. However being in + // a group does nothing for the initial conditions. So the group doesn't care if + // *neither* of them are checked. + isCorePlusRadioButton.DataBindings.Add(new Binding(nameof(isCorePlusRadioButton.Checked), + vm, nameof(vm.IsCorePlus), false, DataSourceUpdateMode.OnPropertyChanged)); + isCoreRadioButton.DataBindings.Add(new Binding(nameof(isCorePlusRadioButton.Checked), + vm, nameof(vm.IsCore), false, DataSourceUpdateMode.OnPropertyChanged)); + + // Make one of the two panels visible, according to the setting of the radio box. + corePlusPanel.DataBindings.Add(nameof(corePlusPanel.Visible), vm, nameof(vm.IsCorePlus)); + corePanel.DataBindings.Add(nameof(corePanel.Visible), vm, nameof(vm.IsCore)); + + // Bind the ID + endpointIdBox.DataBindings.Add(nameof(endpointIdBox.Text), vm, nameof(vm.Id)); + + // Bind the Core+ properties + jsonUrlBox.DataBindings.Add(nameof(jsonUrlBox.Text), vm, nameof(vm.JsonUrl)); + userIdBox.DataBindings.Add(nameof(userIdBox.Text), vm, nameof(vm.UserId)); + passwordBox.DataBindings.Add(nameof(passwordBox.Text), vm, nameof(vm.Password)); + operateAsBox.DataBindings.Add(nameof(operateAsBox.Text), vm, nameof(vm.OperateAs)); + + // Bind the Core property (there's just one) + connectionStringBox.DataBindings.Add(nameof(connectionStringBox.Text), + vm, nameof(vm.ConnectionString)); + + // Bind the IsDefault property + makeDefaultCheckBox.DataBindings.Add(nameof(makeDefaultCheckBox.Checked), + vm, nameof(vm.IsDefault)); + } + + public void SetTestResultsBox(string painState) { + Invoke(() => testResultsTextBox.Text = painState); + } + + private void setCredentialsButton_Click(object sender, EventArgs e) { + _onSetCredentialsButtonClicked(); + } + + private void testCredentialsButton_Click(object sender, EventArgs e) { + _onTestCredentialsButtonClicked(); + } + } +} diff --git a/csharp/ExcelAddIn/views/CredentialsDialog.resx b/csharp/ExcelAddIn/views/CredentialsDialog.resx new file mode 100644 index 00000000000..b92c16350e8 --- /dev/null +++ b/csharp/ExcelAddIn/views/CredentialsDialog.resx @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + \ No newline at end of file From afbb84bca4f1b2d50dac462da59b1a2c6ba50923 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Sat, 31 Aug 2024 14:59:02 -0400 Subject: [PATCH 07/15] feat(csharp/ExcelAddIn): allow insecure JSON URL for Core+; choose python/groovy for Core (#6008) Add GUI elements (and wire them up) 1. ... for allowing insecure JSON URL fetches for Enterprise Core+ 2. ...for selecting the session type (groovy vs python) for Community Core 3. ...for closing the "Set Credentials" dialog box upon setting the credentials --- .../factories/CredentialsDialogFactory.cs | 2 + .../factories/SessionBaseFactory.cs | 13 ++- csharp/ExcelAddIn/models/Credentials.cs | 17 ++-- .../viewmodels/CredentialsDialogViewModel.cs | 50 +++++++++- .../views/CredentialsDialog.Designer.cs | 97 +++++++++++++++---- csharp/ExcelAddIn/views/CredentialsDialog.cs | 7 ++ .../ExcelAddIn/views/CredentialsDialog.resx | 4 +- .../dhe_client/session/SessionManager.cs | 11 +++ 8 files changed, 168 insertions(+), 33 deletions(-) diff --git a/csharp/ExcelAddIn/factories/CredentialsDialogFactory.cs b/csharp/ExcelAddIn/factories/CredentialsDialogFactory.cs index 4212c250c31..adada638e9c 100644 --- a/csharp/ExcelAddIn/factories/CredentialsDialogFactory.cs +++ b/csharp/ExcelAddIn/factories/CredentialsDialogFactory.cs @@ -19,6 +19,8 @@ void OnSetCredentialsButtonClicked() { if (cvm.IsDefault) { sm.SetDefaultCredentials(newCreds); } + + credentialsDialog!.Close(); } void OnTestCredentialsButtonClicked() { diff --git a/csharp/ExcelAddIn/factories/SessionBaseFactory.cs b/csharp/ExcelAddIn/factories/SessionBaseFactory.cs index 0719e58fc2f..44886d6a16d 100644 --- a/csharp/ExcelAddIn/factories/SessionBaseFactory.cs +++ b/csharp/ExcelAddIn/factories/SessionBaseFactory.cs @@ -9,12 +9,21 @@ internal static class SessionBaseFactory { public static SessionBase Create(CredentialsBase credentials, WorkerThread workerThread) { return credentials.AcceptVisitor( core => { - var client = Client.Connect(core.ConnectionString, new ClientOptions()); + var options = new ClientOptions(); + options.SetSessionType(core.SessionTypeIsPython ? "python" : "groovy"); + var client = Client.Connect(core.ConnectionString, options); return new CoreSession(client); }, corePlus => { - var session = SessionManager.FromUrl("Deephaven Excel", corePlus.JsonUrl); + var handler = new HttpClientHandler(); + if (!corePlus.ValidateCertificate) { + handler.ClientCertificateOptions = ClientCertificateOption.Manual; + handler.ServerCertificateCustomValidationCallback = (_, _, _, _) => true; + } + var hc = new HttpClient(handler); + var json = hc.GetStringAsync(corePlus.JsonUrl).Result; + var session = SessionManager.FromJson("Deephaven Excel", json); if (!session.PasswordAuthentication(corePlus.User, corePlus.Password, corePlus.OperateAs)) { throw new Exception("Authentication failed"); } diff --git a/csharp/ExcelAddIn/models/Credentials.cs b/csharp/ExcelAddIn/models/Credentials.cs index 3368a59de04..0a2ffdc6b60 100644 --- a/csharp/ExcelAddIn/models/Credentials.cs +++ b/csharp/ExcelAddIn/models/Credentials.cs @@ -3,21 +3,25 @@ public abstract class CredentialsBase(EndpointId id) { public readonly EndpointId Id = id; - public static CredentialsBase OfCore(EndpointId id, string connectionString) { - return new CoreCredentials(id, connectionString); + public static CredentialsBase OfCore(EndpointId id, string connectionString, bool sessionTypeIsPython) { + return new CoreCredentials(id, connectionString, sessionTypeIsPython); } public static CredentialsBase OfCorePlus(EndpointId id, string jsonUrl, string userId, - string password, string operateAs) { - return new CorePlusCredentials(id, jsonUrl, userId, password, operateAs); + string password, string operateAs, bool validateCertificate) { + return new CorePlusCredentials(id, jsonUrl, userId, password, operateAs, validateCertificate); } public abstract T AcceptVisitor(Func ofCore, Func ofCorePlus); } -public sealed class CoreCredentials(EndpointId id, string connectionString) : CredentialsBase(id) { +public sealed class CoreCredentials( + EndpointId id, + string connectionString, + bool sessionTypeIsPython) : CredentialsBase(id) { public readonly string ConnectionString = connectionString; + public readonly bool SessionTypeIsPython = sessionTypeIsPython; public override T AcceptVisitor(Func ofCore, Func ofCorePlus) { return ofCore(this); @@ -25,11 +29,12 @@ public override T AcceptVisitor(Func ofCore, Func(Func ofCore, Func ofCorePlus) { return ofCorePlus(this); diff --git a/csharp/ExcelAddIn/viewmodels/CredentialsDialogViewModel.cs b/csharp/ExcelAddIn/viewmodels/CredentialsDialogViewModel.cs index c25f6b2faa7..6634640031f 100644 --- a/csharp/ExcelAddIn/viewmodels/CredentialsDialogViewModel.cs +++ b/csharp/ExcelAddIn/viewmodels/CredentialsDialogViewModel.cs @@ -16,11 +16,14 @@ public static CredentialsDialogViewModel OfIdButOtherwiseEmpty(string id) { } public static CredentialsDialogViewModel OfIdAndCredentials(string id, CredentialsBase credentials) { - var result = new CredentialsDialogViewModel { Id = id }; + var result = new CredentialsDialogViewModel { + Id = credentials.Id.Id + }; _ = credentials.AcceptVisitor( core => { result._isCorePlus = false; result.ConnectionString = core.ConnectionString; + result.SessionTypeIsPython = core.SessionTypeIsPython; return Unit.Instance; }, corePlus => { @@ -29,6 +32,7 @@ public static CredentialsDialogViewModel OfIdAndCredentials(string id, Credentia result.UserId = corePlus.User; result.Password = corePlus.Password; result.OperateAs = corePlus.OperateAs; + result.ValidateCertificate = corePlus.ValidateCertificate; return Unit.Instance; }); @@ -38,12 +42,14 @@ public static CredentialsDialogViewModel OfIdAndCredentials(string id, Credentia private string _id = ""; private bool _isDefault = false; private bool _isCorePlus = true; + private bool _sessionTypeIsPython = true; // Core properties private string _connectionString = ""; // Core+ properties private string _jsonUrl = ""; + private bool _validateCertificate = true; private string _userId = ""; private string _password = ""; private string _operateAs = ""; @@ -80,8 +86,8 @@ void CheckMissing(string field, string name) { var epId = new EndpointId(_id); result = _isCorePlus - ? CredentialsBase.OfCorePlus(epId, _jsonUrl, _userId, _password, _operateAs) - : CredentialsBase.OfCore(epId, _connectionString); + ? CredentialsBase.OfCorePlus(epId, _jsonUrl, _userId, _password, _operateAs, _validateCertificate) + : CredentialsBase.OfCore(epId, _connectionString, _sessionTypeIsPython); return true; } @@ -201,6 +207,44 @@ public string OperateAs { } } + public bool ValidateCertificate { + get => _validateCertificate; + set { + if (_validateCertificate == value) { + return; + } + + _validateCertificate = value; + OnPropertyChanged(); + } + } + + public bool SessionTypeIsPython { + get => _sessionTypeIsPython; + set { + if (_sessionTypeIsPython == value) { + return; + } + + _sessionTypeIsPython = value; + OnPropertyChanged(); + OnPropertyChanged(nameof(SessionTypeIsGroovy)); + } + } + + public bool SessionTypeIsGroovy { + get => !_sessionTypeIsPython; + set { + if (!_sessionTypeIsPython == value) { + return; + } + + _sessionTypeIsPython = !value; + OnPropertyChanged(); + OnPropertyChanged(nameof(SessionTypeIsPython)); + } + } + private void OnPropertyChanged([CallerMemberName] string? name = null) { PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(name)); } diff --git a/csharp/ExcelAddIn/views/CredentialsDialog.Designer.cs b/csharp/ExcelAddIn/views/CredentialsDialog.Designer.cs index a554e61e2b2..44d21cf57d1 100644 --- a/csharp/ExcelAddIn/views/CredentialsDialog.Designer.cs +++ b/csharp/ExcelAddIn/views/CredentialsDialog.Designer.cs @@ -25,6 +25,7 @@ protected override void Dispose(bool disposing) { private void InitializeComponent() { flowLayoutPanel1 = new FlowLayoutPanel(); corePlusPanel = new Panel(); + validateCertCheckBox = new CheckBox(); operateAsBox = new TextBox(); passwordBox = new TextBox(); label5 = new Label(); @@ -36,7 +37,11 @@ private void InitializeComponent() { corePanel = new Panel(); connectionStringBox = new TextBox(); label2 = new Label(); + groupBox1 = new GroupBox(); + sessionTypeIsGroovyButton = new RadioButton(); + sessionTypeIsPythonButton = new RadioButton(); finalPanel = new Panel(); + makeDefaultCheckBox = new CheckBox(); testResultsTextBox = new TextBox(); testResultsLabel = new Label(); testCredentialsButton = new Button(); @@ -46,10 +51,10 @@ private void InitializeComponent() { endpointIdBox = new TextBox(); label1 = new Label(); connectionTypeGroup = new GroupBox(); - makeDefaultCheckBox = new CheckBox(); flowLayoutPanel1.SuspendLayout(); corePlusPanel.SuspendLayout(); corePanel.SuspendLayout(); + groupBox1.SuspendLayout(); finalPanel.SuspendLayout(); connectionTypeGroup.SuspendLayout(); SuspendLayout(); @@ -61,11 +66,12 @@ private void InitializeComponent() { flowLayoutPanel1.Controls.Add(finalPanel); flowLayoutPanel1.Location = new Point(28, 160); flowLayoutPanel1.Name = "flowLayoutPanel1"; - flowLayoutPanel1.Size = new Size(994, 531); + flowLayoutPanel1.Size = new Size(994, 676); flowLayoutPanel1.TabIndex = 200; // // corePlusPanel // + corePlusPanel.Controls.Add(validateCertCheckBox); corePlusPanel.Controls.Add(operateAsBox); corePlusPanel.Controls.Add(passwordBox); corePlusPanel.Controls.Add(label5); @@ -76,9 +82,20 @@ private void InitializeComponent() { corePlusPanel.Controls.Add(label3); corePlusPanel.Location = new Point(3, 3); corePlusPanel.Name = "corePlusPanel"; - corePlusPanel.Size = new Size(991, 242); + corePlusPanel.Size = new Size(991, 299); corePlusPanel.TabIndex = 210; // + // validateCertCheckBox + // + validateCertCheckBox.AutoSize = true; + validateCertCheckBox.Location = new Point(192, 240); + validateCertCheckBox.Name = "validateCertCheckBox"; + validateCertCheckBox.RightToLeft = RightToLeft.No; + validateCertCheckBox.Size = new Size(183, 29); + validateCertCheckBox.TabIndex = 216; + validateCertCheckBox.Text = "Validate Certificate"; + validateCertCheckBox.UseVisualStyleBackColor = true; + // // operateAsBox // operateAsBox.Location = new Point(189, 183); @@ -148,9 +165,10 @@ private void InitializeComponent() { // corePanel.Controls.Add(connectionStringBox); corePanel.Controls.Add(label2); - corePanel.Location = new Point(3, 251); + corePanel.Controls.Add(groupBox1); + corePanel.Location = new Point(3, 308); corePanel.Name = "corePanel"; - corePanel.Size = new Size(991, 76); + corePanel.Size = new Size(991, 170); corePanel.TabIndex = 220; // // connectionStringBox @@ -169,6 +187,39 @@ private void InitializeComponent() { label2.TabIndex = 0; label2.Text = "Connection String"; // + // groupBox1 + // + groupBox1.Controls.Add(sessionTypeIsGroovyButton); + groupBox1.Controls.Add(sessionTypeIsPythonButton); + groupBox1.Location = new Point(18, 80); + groupBox1.Name = "groupBox1"; + groupBox1.Size = new Size(970, 71); + groupBox1.TabIndex = 231; + groupBox1.TabStop = false; + groupBox1.Text = "Session Type"; + // + // sessionTypeIsGroovyButton + // + sessionTypeIsGroovyButton.AutoSize = true; + sessionTypeIsGroovyButton.Location = new Point(338, 30); + sessionTypeIsGroovyButton.Name = "sessionTypeIsGroovyButton"; + sessionTypeIsGroovyButton.Size = new Size(95, 29); + sessionTypeIsGroovyButton.TabIndex = 1; + sessionTypeIsGroovyButton.TabStop = true; + sessionTypeIsGroovyButton.Text = "Groovy"; + sessionTypeIsGroovyButton.UseVisualStyleBackColor = true; + // + // sessionTypeIsPythonButton + // + sessionTypeIsPythonButton.AutoSize = true; + sessionTypeIsPythonButton.Location = new Point(192, 30); + sessionTypeIsPythonButton.Name = "sessionTypeIsPythonButton"; + sessionTypeIsPythonButton.Size = new Size(93, 29); + sessionTypeIsPythonButton.TabIndex = 0; + sessionTypeIsPythonButton.TabStop = true; + sessionTypeIsPythonButton.Text = "Python"; + sessionTypeIsPythonButton.UseVisualStyleBackColor = true; + // // finalPanel // finalPanel.Controls.Add(makeDefaultCheckBox); @@ -176,11 +227,21 @@ private void InitializeComponent() { finalPanel.Controls.Add(testResultsLabel); finalPanel.Controls.Add(testCredentialsButton); finalPanel.Controls.Add(setCredentialsButton); - finalPanel.Location = new Point(3, 333); + finalPanel.Location = new Point(3, 484); finalPanel.Name = "finalPanel"; - finalPanel.Size = new Size(991, 132); + finalPanel.Size = new Size(991, 152); finalPanel.TabIndex = 230; // + // makeDefaultCheckBox + // + makeDefaultCheckBox.AutoSize = true; + makeDefaultCheckBox.Location = new Point(599, 70); + makeDefaultCheckBox.Name = "makeDefaultCheckBox"; + makeDefaultCheckBox.Size = new Size(143, 29); + makeDefaultCheckBox.TabIndex = 234; + makeDefaultCheckBox.Text = "Make Default"; + makeDefaultCheckBox.UseVisualStyleBackColor = true; + // // testResultsTextBox // testResultsTextBox.Location = new Point(189, 17); @@ -220,7 +281,7 @@ private void InitializeComponent() { // isCorePlusRadioButton // isCorePlusRadioButton.AutoSize = true; - isCorePlusRadioButton.Location = new Point(6, 39); + isCorePlusRadioButton.Location = new Point(192, 30); isCorePlusRadioButton.Name = "isCorePlusRadioButton"; isCorePlusRadioButton.Size = new Size(169, 29); isCorePlusRadioButton.TabIndex = 110; @@ -231,7 +292,7 @@ private void InitializeComponent() { // isCoreRadioButton // isCoreRadioButton.AutoSize = true; - isCoreRadioButton.Location = new Point(195, 39); + isCoreRadioButton.Location = new Point(388, 30); isCoreRadioButton.Name = "isCoreRadioButton"; isCoreRadioButton.Size = new Size(172, 29); isCoreRadioButton.TabIndex = 111; @@ -266,21 +327,11 @@ private void InitializeComponent() { connectionTypeGroup.TabStop = false; connectionTypeGroup.Text = "Connection Type"; // - // makeDefaultCheckBox - // - makeDefaultCheckBox.AutoSize = true; - makeDefaultCheckBox.Location = new Point(599, 70); - makeDefaultCheckBox.Name = "makeDefaultCheckBox"; - makeDefaultCheckBox.Size = new Size(143, 29); - makeDefaultCheckBox.TabIndex = 234; - makeDefaultCheckBox.Text = "Make Default"; - makeDefaultCheckBox.UseVisualStyleBackColor = true; - // // CredentialsDialog // AutoScaleDimensions = new SizeF(10F, 25F); AutoScaleMode = AutoScaleMode.Font; - ClientSize = new Size(1086, 714); + ClientSize = new Size(1086, 887); Controls.Add(connectionTypeGroup); Controls.Add(label1); Controls.Add(endpointIdBox); @@ -292,6 +343,8 @@ private void InitializeComponent() { corePlusPanel.PerformLayout(); corePanel.ResumeLayout(false); corePanel.PerformLayout(); + groupBox1.ResumeLayout(false); + groupBox1.PerformLayout(); finalPanel.ResumeLayout(false); finalPanel.PerformLayout(); connectionTypeGroup.ResumeLayout(false); @@ -326,5 +379,9 @@ private void InitializeComponent() { private Label testResultsLabel; private TextBox testResultsTextBox; private CheckBox makeDefaultCheckBox; + private CheckBox validateCertCheckBox; + private GroupBox groupBox1; + private RadioButton sessionTypeIsGroovyButton; + private RadioButton sessionTypeIsPythonButton; } } \ No newline at end of file diff --git a/csharp/ExcelAddIn/views/CredentialsDialog.cs b/csharp/ExcelAddIn/views/CredentialsDialog.cs index 1453b8703c4..4ca2260aec1 100644 --- a/csharp/ExcelAddIn/views/CredentialsDialog.cs +++ b/csharp/ExcelAddIn/views/CredentialsDialog.cs @@ -34,11 +34,18 @@ public CredentialsDialog(CredentialsDialogViewModel vm, Action onSetCredentialsB userIdBox.DataBindings.Add(nameof(userIdBox.Text), vm, nameof(vm.UserId)); passwordBox.DataBindings.Add(nameof(passwordBox.Text), vm, nameof(vm.Password)); operateAsBox.DataBindings.Add(nameof(operateAsBox.Text), vm, nameof(vm.OperateAs)); + validateCertCheckBox.DataBindings.Add(nameof(validateCertCheckBox.Checked), vm, nameof(vm.ValidateCertificate)); // Bind the Core property (there's just one) connectionStringBox.DataBindings.Add(nameof(connectionStringBox.Text), vm, nameof(vm.ConnectionString)); + // Bind the SessionType checkboxes + sessionTypeIsPythonButton.DataBindings.Add(nameof(sessionTypeIsPythonButton.Checked), + vm, nameof(vm.SessionTypeIsPython)); + sessionTypeIsGroovyButton.DataBindings.Add(nameof(sessionTypeIsGroovyButton.Checked), + vm, nameof(vm.SessionTypeIsGroovy)); + // Bind the IsDefault property makeDefaultCheckBox.DataBindings.Add(nameof(makeDefaultCheckBox.Checked), vm, nameof(vm.IsDefault)); diff --git a/csharp/ExcelAddIn/views/CredentialsDialog.resx b/csharp/ExcelAddIn/views/CredentialsDialog.resx index b92c16350e8..4f24d55cd6b 100644 --- a/csharp/ExcelAddIn/views/CredentialsDialog.resx +++ b/csharp/ExcelAddIn/views/CredentialsDialog.resx @@ -1,7 +1,7 @@ 

The resulting output of flattened objects is not always clear, but the following can hopefully serve as a quick reference as well as these specs when using _ as the delimiter:

Input:

{
    "apples": "tree",
    "bananas": {
        "truthiness": true
    }
}

Output:

{
    "apples": "tree",
    "bananas_truthiness": "true"
}

Notice that bananas_truthiness is also stringified in this process, as part of updating values to match the expected inputs of Workflow Builder!

Changes

In addition to the changes above, the following lists all of the changes since the prior version with the complete changelog changes found here: https://github.com/slackapi/slack-github-action/compare/v1.26.0...v1.27.0

🎁 Enhancements

... (truncated)

Commits
  • 37ebaef Automatic compilation
  • 5d1fb07 chore(release): tag version 1.27.0
  • 3bc0671 chore(deps): bump axios to 1.7.5 (#332)
  • b452451 feat: make the payload delimiter configurable for workflow webhook triggers (...
  • c50e848 build(deps-dev): bump mocha from 10.5.2 to 10.7.0 (#328)
  • e4a9c4b build(deps): bump @​slack/web-api from 7.2.0 to 7.3.2 (#327)
  • 9a7f0fa build(deps-dev): bump chai from 4.4.1 to 4.5.0 (#326)
  • 73b7062 build(deps-dev): bump eslint-plugin-jsdoc from 48.5.0 to 48.10.2 (#325)
  • 3d5207b build(deps): bump https-proxy-agent from 7.0.4 to 7.0.5 (#320)
  • 4e15b6a build(deps): bump @​slack/web-api from 7.0.4 to 7.2.0 (#323)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=slackapi/slack-github-action&package-manager=github_actions&previous-version=1.26.0&new-version=1.27.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/create-docs-issues.yml | 2 +- .github/workflows/nightly-check-ci.yml | 2 +- .github/workflows/nightly-image-check.yml | 2 +- .github/workflows/nightly-publish-ci.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/create-docs-issues.yml b/.github/workflows/create-docs-issues.yml index 77f52569deb..6e6fc938ea5 100644 --- a/.github/workflows/create-docs-issues.yml +++ b/.github/workflows/create-docs-issues.yml @@ -26,7 +26,7 @@ jobs: await script({github, context}); - name: Slack Failure Message - uses: slackapi/slack-github-action@v1.26.0 + uses: slackapi/slack-github-action@v1.27.0 id: slack-failure-message if: failure() && github.ref == 'refs/heads/main' && github.repository_owner == 'deephaven' env: diff --git a/.github/workflows/nightly-check-ci.yml b/.github/workflows/nightly-check-ci.yml index d8b5c93fec7..2e5c37cec73 100644 --- a/.github/workflows/nightly-check-ci.yml +++ b/.github/workflows/nightly-check-ci.yml @@ -100,7 +100,7 @@ jobs: report_paths: '**/build/test-results/*/TEST-*.xml' - name: Slack Nightly Failure - uses: slackapi/slack-github-action@v1.26.0 + uses: slackapi/slack-github-action@v1.27.0 id: slack-nightly-failure if: ${{ failure() && github.repository_owner == 'deephaven' }} with: diff --git a/.github/workflows/nightly-image-check.yml b/.github/workflows/nightly-image-check.yml index 1e868280782..e2f75b3a15f 100644 --- a/.github/workflows/nightly-image-check.yml +++ b/.github/workflows/nightly-image-check.yml @@ -35,7 +35,7 @@ jobs: run: ./gradlew --continue pullImage compareImage - name: Notify Slack - uses: slackapi/slack-github-action@v1.26.0 + uses: slackapi/slack-github-action@v1.27.0 id: notify-slack if: ${{ failure() }} env: diff --git a/.github/workflows/nightly-publish-ci.yml b/.github/workflows/nightly-publish-ci.yml index a23f9e04a2d..927c0b2009a 100644 --- a/.github/workflows/nightly-publish-ci.yml +++ b/.github/workflows/nightly-publish-ci.yml @@ -55,7 +55,7 @@ jobs: ORG_GRADLE_PROJECT_signingRequired: true - name: Slack Nightly Failure - uses: slackapi/slack-github-action@v1.26.0 + uses: slackapi/slack-github-action@v1.27.0 id: slack-nightly-failure if: failure() with: From 992fc48c7eda10581698abc423f17f2ddfed0bb1 Mon Sep 17 00:00:00 2001 From: Shivam Malhotra Date: Tue, 3 Sep 2024 19:07:54 -0500 Subject: [PATCH 13/15] fix: For failing nightly check test (#6018) --- .../table/ParquetTableReadWriteTest.java | 225 ++++++++++-------- 1 file changed, 125 insertions(+), 100 deletions(-) diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java index 74eca341e24..5d104ec8113 100644 --- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java +++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java @@ -15,6 +15,13 @@ import io.deephaven.engine.primitive.function.FloatConsumer; import io.deephaven.engine.primitive.function.ShortConsumer; import io.deephaven.engine.primitive.iterator.CloseableIterator; +import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfByte; +import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfChar; +import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfDouble; +import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfFloat; +import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfInt; +import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfLong; +import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfShort; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.PartitionedTable; @@ -4113,18 +4120,20 @@ private void assertByteVectorColumnStatistics(SerialObjectColumnIterator { + itemCount.increment(); + if (value == NULL_BYTE) { + nullCount.increment(); + } else { + if (min.get() == NULL_BYTE || value < min.get()) { + min.set(value); + } + if (max.get() == NULL_BYTE || value > max.get()) { + max.set(value); + } } - if (max.get() == NULL_BYTE || value > max.get()) { - max.set(value); - } - } + }); } }); @@ -4222,18 +4231,20 @@ private void assertCharVectorColumnStatistics(SerialObjectColumnIterator max.get()) { - max.set(value); + try (final CloseablePrimitiveIteratorOfChar valuesIterator = values.iterator()) { + valuesIterator.forEachRemaining((final char value) -> { + itemCount.increment(); + if (value == NULL_CHAR) { + nullCount.increment(); + } else { + if (min.get() == NULL_CHAR || value < min.get()) { + min.set(value); + } + if (max.get() == NULL_CHAR || value > max.get()) { + max.set(value); + } } - } + }); } }); @@ -4331,18 +4342,20 @@ private void assertShortVectorColumnStatistics(SerialObjectColumnIterator max.get()) { - max.set(value); + try (final CloseablePrimitiveIteratorOfShort valuesIterator = values.iterator()) { + valuesIterator.forEachRemaining((final short value) -> { + itemCount.increment(); + if (value == NULL_SHORT) { + nullCount.increment(); + } else { + if (min.get() == NULL_SHORT || value < min.get()) { + min.set(value); + } + if (max.get() == NULL_SHORT || value > max.get()) { + max.set(value); + } } - } + }); } }); @@ -4440,18 +4453,20 @@ private void assertIntVectorColumnStatistics(SerialObjectColumnIterator max.get()) { - max.set(value); + try (final CloseablePrimitiveIteratorOfInt valuesIterator = values.iterator()) { + valuesIterator.forEachRemaining((final int value) -> { + itemCount.increment(); + if (value == NULL_INT) { + nullCount.increment(); + } else { + if (min.get() == NULL_INT || value < min.get()) { + min.set(value); + } + if (max.get() == NULL_INT || value > max.get()) { + max.set(value); + } } - } + }); } }); @@ -4549,18 +4564,20 @@ private void assertLongVectorColumnStatistics(SerialObjectColumnIterator { + itemCount.increment(); + if (value == NULL_LONG) { + nullCount.increment(); + } else { + if (min.get() == NULL_LONG || value < min.get()) { + min.set(value); + } + if (max.get() == NULL_LONG || value > max.get()) { + max.set(value); + } } - if (max.get() == NULL_LONG || value > max.get()) { - max.set(value); - } - } + }); } }); @@ -4660,18 +4677,20 @@ private void assertFloatVectorColumnStatistics(SerialObjectColumnIterator { + itemCount.increment(); + if (value == NULL_FLOAT) { + nullCount.increment(); + } else { + if (min.floatValue() == NULL_FLOAT || value < min.floatValue()) { + min.setValue(value); + } + if (max.floatValue() == NULL_FLOAT || value > max.floatValue()) { + max.setValue(value); + } } - if (max.floatValue() == NULL_FLOAT || value > max.floatValue()) { - max.setValue(value); - } - } + }); } }); @@ -4772,18 +4791,20 @@ private void assertDoubleVectorColumnStatistics(SerialObjectColumnIterator { + itemCount.increment(); + if (value == NULL_DOUBLE) { + nullCount.increment(); + } else { + if (min.doubleValue() == NULL_DOUBLE || value < min.doubleValue()) { + min.setValue(value); + } + if (max.doubleValue() == NULL_DOUBLE || value > max.doubleValue()) { + max.setValue(value); + } } - if (max.doubleValue() == NULL_DOUBLE || value > max.doubleValue()) { - max.setValue(value); - } - } + }); } }); @@ -4883,18 +4904,20 @@ private void assertStringVectorColumnStatistics(SerialObjectColumnIterator 0) { - max.setValue(value); + try (final CloseableIterator valuesIterator = values.iterator()) { + valuesIterator.forEachRemaining((final String value) -> { + itemCount.increment(); + if (value == null) { + nullCount.increment(); + } else { + if (min.getValue() == null || value.compareTo(min.getValue()) < 0) { + min.setValue(value); + } + if (max.getValue() == null || value.compareTo(max.getValue()) > 0) { + max.setValue(value); + } } - } + }); } }); @@ -4995,19 +5018,21 @@ private void assertInstantVectorColumnStatistics(SerialObjectColumnIterator max.get()) { - max.set(DateTimeUtils.epochNanos(value)); + try (final CloseableIterator valuesIterator = values.iterator()) { + valuesIterator.forEachRemaining((final Instant value) -> { + itemCount.increment(); + if (value == null) { + nullCount.increment(); + } else { + // DateTimeUtils.epochNanos() is the correct conversion for Instant to long. + if (min.get() == NULL_LONG || DateTimeUtils.epochNanos(value) < min.get()) { + min.set(DateTimeUtils.epochNanos(value)); + } + if (max.get() == NULL_LONG || DateTimeUtils.epochNanos(value) > max.get()) { + max.set(DateTimeUtils.epochNanos(value)); + } } - } + }); } }); From a457ff976e5f45f29118bfc9640d8be61f08fb33 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 4 Sep 2024 10:33:08 -0500 Subject: [PATCH 14/15] fix: Python embedded server should shut down any jvm threads/work it can (#6006) Fixes #6005 --- py/embedded-server/build.gradle | 4 ++-- py/embedded-server/deephaven_server/server.py | 4 ++++ .../io/deephaven/python/server/EmbeddedServer.java | 10 ++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/py/embedded-server/build.gradle b/py/embedded-server/build.gradle index e0bc16be9db..2479a7d0b38 100644 --- a/py/embedded-server/build.gradle +++ b/py/embedded-server/build.gradle @@ -27,7 +27,7 @@ dependencies { } } -def testPyClient = Docker.registerDockerTask(project, 'testPyClient') { +def testEmbeddedServer = Docker.registerDockerTask(project, 'testEmbeddedServer') { copyIn { from('tests') { into 'project/tests' @@ -62,4 +62,4 @@ def testPyClient = Docker.registerDockerTask(project, 'testPyClient') { } } -tasks.getByName('check').dependsOn(testPyClient) +tasks.getByName('check').dependsOn(testEmbeddedServer) diff --git a/py/embedded-server/deephaven_server/server.py b/py/embedded-server/deephaven_server/server.py index 6bae0ef5ee6..f888dad73d2 100644 --- a/py/embedded-server/deephaven_server/server.py +++ b/py/embedded-server/deephaven_server/server.py @@ -5,6 +5,7 @@ from typing import List, Optional import sys +import atexit from .start_jvm import start_jvm @@ -167,6 +168,9 @@ def __init__( # Keep a reference to the server so we know it is running Server.instance = self + # On halt, prevent the JVM from writing to sys.out and sys.err + atexit.register(self.j_server.prepareForShutdown) + def start(self): """ Starts the server. Presently once the server is started, it cannot be stopped until the diff --git a/py/embedded-server/java-runtime/src/main/java/io/deephaven/python/server/EmbeddedServer.java b/py/embedded-server/java-runtime/src/main/java/io/deephaven/python/server/EmbeddedServer.java index 7e18ccca1bb..3bc427f8904 100644 --- a/py/embedded-server/java-runtime/src/main/java/io/deephaven/python/server/EmbeddedServer.java +++ b/py/embedded-server/java-runtime/src/main/java/io/deephaven/python/server/EmbeddedServer.java @@ -36,6 +36,7 @@ import io.deephaven.server.session.ObfuscatingErrorTransformerModule; import io.deephaven.server.session.SslConfigModule; import io.deephaven.time.calendar.CalendarsFromConfigurationModule; +import io.deephaven.util.process.ProcessEnvironment; import org.jpy.PyModule; import org.jpy.PyObject; @@ -167,6 +168,15 @@ public void start() throws Exception { Bootstrap.printf("Server started on port %d%n", getPort()); } + public void prepareForShutdown() { + // Stop any DH-started threads and work + ProcessEnvironment.getGlobalShutdownManager().maybeInvokeTasks(); + + // Shut down our wrapped stdout/stderr instances + System.out.close(); + System.err.close(); + } + public int getPort() { return server.server().getPort(); } From ea0dcb2b44cb72e88409ef1fa8b63c647e4f1628 Mon Sep 17 00:00:00 2001 From: Alex Peters <80283343+alexpeters1208@users.noreply.github.com> Date: Wed, 4 Sep 2024 14:50:33 -0500 Subject: [PATCH 15/15] feat: Support `remove_blink` to remove blink table semantics in server-side Python (#5958) `removeBlink` is used by the Java API to disable the specialized aggregation semantics that blink tables implement. This wrapper adds the functionality to disable those semantics from the Python API. --- py/server/deephaven/table.py | 4 ++++ py/server/tests/test_table.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/py/server/deephaven/table.py b/py/server/deephaven/table.py index abb22e1031c..e02307f647e 100644 --- a/py/server/deephaven/table.py +++ b/py/server/deephaven/table.py @@ -816,6 +816,10 @@ def flatten(self) -> Table: """Returns a new version of this table with a flat row set, i.e. from 0 to number of rows - 1.""" return Table(j_table=self.j_table.flatten()) + def remove_blink(self) -> Table: + """Returns a non-blink child table, or this table if it is not a blink table.""" + return Table(j_table=self.j_table.removeBlink()) + def snapshot(self) -> Table: """Returns a static snapshot table. diff --git a/py/server/tests/test_table.py b/py/server/tests/test_table.py index 06e0ab2355a..b37b89c8c14 100644 --- a/py/server/tests/test_table.py +++ b/py/server/tests/test_table.py @@ -930,6 +930,12 @@ def test_attributes(self): self.assertEqual(len(attrs), len(rt_attrs) + 1) self.assertIn("BlinkTable", set(attrs.keys()) - set(rt_attrs.keys())) + def test_remove_blink(self): + t_blink = time_table("PT1s", blink_table=True) + t_no_blink = t_blink.remove_blink() + self.assertEqual(t_blink.is_blink, True) + self.assertEqual(t_no_blink.is_blink, False) + def test_grouped_column_as_arg(self): t1 = empty_table(100).update( ["id = i % 10", "Person = random() > 0.5 ? true : random() > 0.5 ? false : true"]).sort(