From e71913691fddbf46b125e94e0224e0385554a848 Mon Sep 17 00:00:00 2001 From: Ryan Caudy Date: Thu, 30 Nov 2023 13:00:11 -0500 Subject: [PATCH] Clean up some legacy code and improve build (#4674) * Avoid extra copy of TypeScript/JavaScript files in proto/raw-js-openapi/Dockerfile * Delete FishUtil. All usages have been eliminated, trivially inlined, or the files have migrated to more appropriate modules. * Refactor TrackedFileHandleFactory to use a ScheduledExecutorService instead of Scheduler * Refactor Liveness to use a ScheduledExecutorService instead of Scheduler for (currently unused) scheduleCountReport * Replace Scheduler usage in PeriodicUpdateGraph with ScheduledExecutorService. Standardize on nanoseconds for timing. Eliminate unnecessary initializations. * Replace Scheduler usage in StatsDriver with ScheduledExecutorService * Delete :Net. Delete io.sched package. Delete gradle :Net and fishnet projects/configurations. Delete dependencies on same. * Delete legacy utility that used reflection to reach into the guts of SelectorImpl --------- Co-authored-by: Colin Alworth --- .../base/stats/ItemUpdateListener.java | 11 +- .../base/stats/HistogramPower2Test.java | 2 +- .../base/stats/HistogramStateTest.java | 2 +- FishUtil/build.gradle | 14 - FishUtil/cpp/MicroTimer.cpp | 89 -- FishUtil/cpp/MicroTimer.h | 22 - FishUtil/cpp/SignalUtils.cpp | 9 - FishUtil/cpp/SignalUtils.h | 21 - FishUtil/gradle.properties | 1 - .../main/java/io/deephaven/util/DateUtil.java | 955 ----------------- .../java/io/deephaven/util/ExceptionUtil.java | 22 - .../main/java/io/deephaven/util/Mailer.java | 19 - .../deephaven/util/ThreadSafeDateFormat.java | 155 --- .../main/java/io/deephaven/util/Validate.java | 110 -- .../io/deephaven/util/formatters/ISO8601.java | 120 --- .../deephaven/util/signals/SignalSender.java | 111 -- .../deephaven/util/signals/SignalUtils.java | 110 -- .../io/deephaven/util/threads/ThreadDump.java | 48 - IO/src/main/java/io/deephaven/io/NioUtil.java | 109 -- .../main/java/io/deephaven/io/sched/Job.java | 66 -- .../java/io/deephaven/io/sched/JobState.java | 65 -- .../io/sched/JobStateTimeoutQueue.java | 211 ---- .../java/io/deephaven/io/sched/Scheduler.java | 199 ---- .../java/io/deephaven/io/sched/TimedJob.java | 30 - .../deephaven/io/sched/YASchedulerImpl.java | 979 ------------------ .../io/sched/TestJobStateTimeoutQueue.java | 124 --- Net/build.gradle | 30 - Net/gradle.properties | 1 - .../main/java/io/deephaven/net/CommBase.java | 111 -- .../deephaven/net/impl/nio/FastNIODriver.java | 285 ----- .../io/deephaven/net/impl/nio/NIODriver.java | 295 ------ Stats/build.gradle | 2 - .../io/deephaven/stats/StatsCPUCollector.java | 2 +- .../java/io/deephaven/stats/StatsDriver.java | 60 +- .../java/io/deephaven/stats}/util/OSUtil.java | 12 +- TableLogger/TableLogger.gradle | 6 +- .../util/process/BaseProcessEnvironment.java | 0 .../process/DefaultFatalErrorReporter.java | 0 .../process/DefaultProcessEnvironment.java | 0 .../util/process/FatalErrorReporter.java | 0 .../util/process/FatalErrorReporterBase.java | 0 .../util/process/LoggerShutdownTask.java | 0 .../util/process/OnetimeShutdownTask.java | 0 .../util/process/ProcessEnvironment.java | 0 .../util/process/ShutdownManager.java | 0 .../util/process/ShutdownManagerImpl.java | 2 +- ...eephaven.java-classpath-conventions.gradle | 8 +- engine/table/build.gradle | 2 - .../updategraph/impl/PeriodicUpdateGraph.java | 104 +- .../util/file/TrackedFileHandleFactory.java | 75 +- .../file/TestTrackedFileHandleFactory.java | 16 +- engine/test-utils/build.gradle | 1 - engine/time/build.gradle | 1 - engine/updategraph/build.gradle | 2 - .../deephaven/engine/liveness/Liveness.java | 25 +- proto/raw-js-openapi/Dockerfile | 5 - server/build.gradle | 1 - settings.gradle | 4 - 58 files changed, 173 insertions(+), 4481 deletions(-) delete mode 100644 FishUtil/build.gradle delete mode 100644 FishUtil/cpp/MicroTimer.cpp delete mode 100644 FishUtil/cpp/MicroTimer.h delete mode 100644 FishUtil/cpp/SignalUtils.cpp delete mode 100644 FishUtil/cpp/SignalUtils.h delete mode 100644 FishUtil/gradle.properties delete mode 100644 FishUtil/src/main/java/io/deephaven/util/DateUtil.java delete mode 100644 FishUtil/src/main/java/io/deephaven/util/ExceptionUtil.java delete mode 100644 FishUtil/src/main/java/io/deephaven/util/Mailer.java delete mode 100644 FishUtil/src/main/java/io/deephaven/util/ThreadSafeDateFormat.java delete mode 100644 FishUtil/src/main/java/io/deephaven/util/Validate.java delete mode 100644 FishUtil/src/main/java/io/deephaven/util/formatters/ISO8601.java delete mode 100644 FishUtil/src/main/java/io/deephaven/util/signals/SignalSender.java delete mode 100644 FishUtil/src/main/java/io/deephaven/util/signals/SignalUtils.java delete mode 100644 FishUtil/src/main/java/io/deephaven/util/threads/ThreadDump.java delete mode 100644 IO/src/main/java/io/deephaven/io/NioUtil.java delete mode 100644 IO/src/main/java/io/deephaven/io/sched/Job.java delete mode 100644 IO/src/main/java/io/deephaven/io/sched/JobState.java delete mode 100644 IO/src/main/java/io/deephaven/io/sched/JobStateTimeoutQueue.java delete mode 100644 IO/src/main/java/io/deephaven/io/sched/Scheduler.java delete mode 100644 IO/src/main/java/io/deephaven/io/sched/TimedJob.java delete mode 100644 IO/src/main/java/io/deephaven/io/sched/YASchedulerImpl.java delete mode 100644 IO/src/test/java/io/deephaven/io/sched/TestJobStateTimeoutQueue.java delete mode 100644 Net/build.gradle delete mode 100644 Net/gradle.properties delete mode 100644 Net/src/main/java/io/deephaven/net/CommBase.java delete mode 100644 Net/src/main/java/io/deephaven/net/impl/nio/FastNIODriver.java delete mode 100644 Net/src/main/java/io/deephaven/net/impl/nio/NIODriver.java rename {FishUtil/src/main/java/io/deephaven => Stats/src/main/java/io/deephaven/stats}/util/OSUtil.java (85%) rename {FishUtil => Util}/src/main/java/io/deephaven/util/process/BaseProcessEnvironment.java (100%) rename {FishUtil => Util}/src/main/java/io/deephaven/util/process/DefaultFatalErrorReporter.java (100%) rename {FishUtil => Util}/src/main/java/io/deephaven/util/process/DefaultProcessEnvironment.java (100%) rename {FishUtil => Util}/src/main/java/io/deephaven/util/process/FatalErrorReporter.java (100%) rename {FishUtil => Util}/src/main/java/io/deephaven/util/process/FatalErrorReporterBase.java (100%) rename {FishUtil => Util}/src/main/java/io/deephaven/util/process/LoggerShutdownTask.java (100%) rename {FishUtil => Util}/src/main/java/io/deephaven/util/process/OnetimeShutdownTask.java (100%) rename {FishUtil => Util}/src/main/java/io/deephaven/util/process/ProcessEnvironment.java (100%) rename {FishUtil => Util}/src/main/java/io/deephaven/util/process/ShutdownManager.java (100%) rename {FishUtil => Util}/src/main/java/io/deephaven/util/process/ShutdownManagerImpl.java (99%) diff --git a/Base/src/main/java/io/deephaven/base/stats/ItemUpdateListener.java b/Base/src/main/java/io/deephaven/base/stats/ItemUpdateListener.java index 4d092026ba6..cc9007f5e72 100644 --- a/Base/src/main/java/io/deephaven/base/stats/ItemUpdateListener.java +++ b/Base/src/main/java/io/deephaven/base/stats/ItemUpdateListener.java @@ -4,13 +4,10 @@ package io.deephaven.base.stats; public interface ItemUpdateListener { - public void handleItemUpdated(Item item, long now, long appNow, int intervalIndex, long intervalMillis, - String intervalName); - public static final ItemUpdateListener NULL = new ItemUpdateListener() { - public void handleItemUpdated(Item item, long now, long appNow, int intervalIndex, long intervalMillis, - String intervalName) { - // empty - } + void handleItemUpdated( + Item item, long now, long appNow, int intervalIndex, long intervalMillis, String intervalName); + + ItemUpdateListener NULL = (item, now, appNow, intervalIndex, intervalMillis, intervalName) -> { }; } diff --git a/Base/src/test/java/io/deephaven/base/stats/HistogramPower2Test.java b/Base/src/test/java/io/deephaven/base/stats/HistogramPower2Test.java index 6958e16994b..bd24cf5c052 100644 --- a/Base/src/test/java/io/deephaven/base/stats/HistogramPower2Test.java +++ b/Base/src/test/java/io/deephaven/base/stats/HistogramPower2Test.java @@ -33,7 +33,7 @@ public void testSample() throws Exception { // should have a count of 1 in bin[1]..bin[63]; bin[0]=2 Stats.update(new ItemUpdateListener() { - public void handleItemUpdated(Item item, long now, long appNow, int intervalIndex, long intervalMillis, + public void handleItemUpdated(Item item, long now, long appNow, int intervalIndex, long intervalMillis, String intervalName) { // Value v = item.getValue(); HistogramPower2 nh; diff --git a/Base/src/test/java/io/deephaven/base/stats/HistogramStateTest.java b/Base/src/test/java/io/deephaven/base/stats/HistogramStateTest.java index ea63a9100fd..2a23844c93f 100644 --- a/Base/src/test/java/io/deephaven/base/stats/HistogramStateTest.java +++ b/Base/src/test/java/io/deephaven/base/stats/HistogramStateTest.java @@ -23,7 +23,7 @@ public void testSample() throws Exception { // This should print 10 invocations every time Stats.update(new ItemUpdateListener() { - public void handleItemUpdated(Item item, long now, long appNow, int intervalIndex, long intervalMillis, + public void handleItemUpdated(Item item, long now, long appNow, int intervalIndex, long intervalMillis, String intervalName) { Value v = item.getValue(); History history = v.getHistory(); diff --git a/FishUtil/build.gradle b/FishUtil/build.gradle deleted file mode 100644 index bcadd9989db..00000000000 --- a/FishUtil/build.gradle +++ /dev/null @@ -1,14 +0,0 @@ -plugins { - id 'io.deephaven.project.register' - id 'java-library' -} - -dependencies { - implementation project(':Base') - implementation project(':DataStructures') - implementation project(':IO') - implementation project(':Configuration') - implementation project(':log-factory') - - testImplementation project(path: ':Base', configuration: 'tests') -} \ No newline at end of file diff --git a/FishUtil/cpp/MicroTimer.cpp b/FishUtil/cpp/MicroTimer.cpp deleted file mode 100644 index 1455d872f8e..00000000000 --- a/FishUtil/cpp/MicroTimer.cpp +++ /dev/null @@ -1,89 +0,0 @@ -#include "MicroTimer.h" - -#ifdef _WIN32 -#include -#include -static jdouble scale; -LARGE_INTEGER startupTick; -LARGE_INTEGER startupTime; -const double MICROS_IN_SEC = 1000000.0; - -JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM * vm, void * reserved) { - LARGE_INTEGER freq; - QueryPerformanceFrequency (&freq); - scale = freq.QuadPart / MICROS_IN_SEC; - - QueryPerformanceCounter(&startupTick); - - struct timeb startupTimeMillis; - ftime(&startupTimeMillis); - startupTime.QuadPart = startupTimeMillis.time; - startupTime.QuadPart *= 1000; - startupTime.QuadPart += startupTimeMillis.millitm; - startupTime.QuadPart *= 1000; - - return JNI_VERSION_1_2; -} - -JNIEXPORT jlong JNICALL Java_io_deephaven_util_clock_MicroTimer_currentTimeMicrosNative - (JNIEnv * env, jclass cls) { - LARGE_INTEGER now; - QueryPerformanceCounter (&now); - LARGE_INTEGER diff; - diff.QuadPart = (now.QuadPart - startupTick.QuadPart) / scale; - return startupTime.QuadPart + diff.QuadPart; -} - -extern "C" JNIEXPORT jlong JNICALL Java_io_deephaven_util_clock_MicroTimer_clockRealtimeNative - (JNIEnv * env, jclass cls) { - jlong micros = Java_io_deephaven_util_clock_MicroTimer_currentTimeMicrosNative(env, cls); - return micros * 1000L; -} - -extern "C" JNIEXPORT jlong JNICALL Java_io_deephaven_util_clock_MicroTimer_clockMonotonicNative - (JNIEnv * env, jclass cls) { - jlong micros = Java_io_deephaven_util_clock_MicroTimer_currentTimeMicrosNative(env, cls); - return micros * 1000L; -} - - -#else -#include -#include -#include -const uint64_t MICROS_IN_SEC = 1000000L; - -JNIEXPORT jlong JNICALL Java_io_deephaven_util_clock_MicroTimer_currentTimeMicrosNative - (JNIEnv * env, jclass cls) { - timeval now; - gettimeofday(&now, NULL); - return ((uint64_t) now.tv_sec * 1000000L) + now.tv_usec; -} - -extern "C" JNIEXPORT jlong JNICALL Java_io_deephaven_util_clock_MicroTimer_clockRealtimeNative - (JNIEnv * env, jclass cls) { - timespec now; - clock_gettime(CLOCK_REALTIME, &now); - return ((uint64_t) now.tv_sec * 1000000000L) + now.tv_nsec; -} - -extern "C" JNIEXPORT jlong JNICALL Java_io_deephaven_util_clock_MicroTimer_clockMonotonicNative - (JNIEnv * env, jclass cls) { - timespec now; - clock_gettime(CLOCK_MONOTONIC, &now); - return ((uint64_t) now.tv_sec * 1000000000L) + now.tv_nsec; -} - -#endif - -static __inline__ unsigned long long rdtsc(void) { - unsigned hi, lo; - __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi)); - return ((unsigned long long)lo)|(((unsigned long long)hi)<<32); -} - -extern "C" JNIEXPORT jlong JNICALL Java_io_deephaven_util_clock_MicroTimer_rdtscNative - (JNIEnv * env, jclass cls) { - return (jlong) rdtsc(); -} - diff --git a/FishUtil/cpp/MicroTimer.h b/FishUtil/cpp/MicroTimer.h deleted file mode 100644 index e20fbca170f..00000000000 --- a/FishUtil/cpp/MicroTimer.h +++ /dev/null @@ -1,22 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include -/* Header for class io_deephaven_util_clock_MicroTimer */ - -#ifndef _Included_io_deephaven_util_clock_MicroTimer -#define _Included_io_deephaven_util_clock_MicroTimer - -#ifdef __cplusplus -extern "C" { -#endif -/* - * Class: io_deephaven_util_clock_MicroTimer - * Method: currentTimeMicrosNative - * Signature: ()J - */ -JNIEXPORT jlong JNICALL Java_io_deephaven_util_clock_MicroTimer_currentTimeMicrosNative - (JNIEnv *, jclass); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/FishUtil/cpp/SignalUtils.cpp b/FishUtil/cpp/SignalUtils.cpp deleted file mode 100644 index 5d4f2a75ca4..00000000000 --- a/FishUtil/cpp/SignalUtils.cpp +++ /dev/null @@ -1,9 +0,0 @@ -#include "SignalUtils.h" - -#include -#include - -extern "C" JNIEXPORT jint JNICALL Java_io_deephaven_util_signals_SignalUtils_sendSignalNative - (JNIEnv * env, jclass cls, jint pid, jint sig) { - return kill(pid, sig); -} diff --git a/FishUtil/cpp/SignalUtils.h b/FishUtil/cpp/SignalUtils.h deleted file mode 100644 index 281c9a8a34f..00000000000 --- a/FishUtil/cpp/SignalUtils.h +++ /dev/null @@ -1,21 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include -/* Header for class io_deephaven_util_signals_SignalUtils */ - -#ifndef _Included_io_deephaven_util_signals_SignalUtils -#define _Included_io_deephaven_util_signals_SignalUtils -#ifdef __cplusplus -extern "C" { -#endif -/* - * Class: io_deephaven_util_signals_SignalUtils - * Method: sendSignalNative - * Signature: (II)I - */ -JNIEXPORT jint JNICALL Java_io_deephaven_util_signals_SignalUtils_sendSignalNative - (JNIEnv *, jclass, jint, jint); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/FishUtil/gradle.properties b/FishUtil/gradle.properties deleted file mode 100644 index c186bbfdde1..00000000000 --- a/FishUtil/gradle.properties +++ /dev/null @@ -1 +0,0 @@ -io.deephaven.project.ProjectType=JAVA_PUBLIC diff --git a/FishUtil/src/main/java/io/deephaven/util/DateUtil.java b/FishUtil/src/main/java/io/deephaven/util/DateUtil.java deleted file mode 100644 index c5c5b66ee90..00000000000 --- a/FishUtil/src/main/java/io/deephaven/util/DateUtil.java +++ /dev/null @@ -1,955 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.util; - -import io.deephaven.base.verify.Require; -import io.deephaven.base.verify.RequirementFailure; -import io.deephaven.configuration.PropertyFile; - -import java.io.File; -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.*; - -// -------------------------------------------------------------------- -/** - * Useful methods for working with dates. Not for use in the critical path. - */ -public class DateUtil { - - public static final boolean DAYMASK_STRICT = true; - public static final boolean DAYMASK_NOT_STRICT = false; - - public static final int DAY_VALID = '1'; - public static final int DAY_INVALID = '0'; - public static final int DAY_OPTIONAL = '2'; - - public static final String DAYMASK_NORMAL_BUSINESS_WEEK = "0111110"; - - public static final long NANOS_PER_MICRO = 1000; - public static final long NANOS_PER_MILLI = NANOS_PER_MICRO * 1000; - public static final long NANOS_PER_SECOND = NANOS_PER_MILLI * 1000; - - public static final long MICROS_PER_MILLI = 1000; - public static final long MICROS_PER_SECOND = MICROS_PER_MILLI * 1000; - - public static final long MILLIS_PER_SECOND = 1000; - public static final long MILLIS_PER_MINUTE = MILLIS_PER_SECOND * 60; - public static final long MILLIS_PER_HOUR = MILLIS_PER_MINUTE * 60; - public static final long MILLIS_PER_DAY = MILLIS_PER_HOUR * 24; - - public static final int SECONDS_PER_MINUTE = 60; - public static final int SECONDS_PER_HOUR = SECONDS_PER_MINUTE * 60; - public static final int SECONDS_PER_DAY = SECONDS_PER_HOUR * 24; - - public static final int DAYS_PER_WEEK = 7; - - public static final long[] THOUSANDS = {1, 1000, 1000000, 1000000000}; - - /** Number of days in each month. (Jan==1, Feb is non-leap-year) */ - public static final int[] DAYS_PER_MONTH = {0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; - - /** Three letter abbreviations of month names. (Jan==1, title case) */ - public static final String[] MONTH_ABBREVIATIONS_3T = - {"Xxx", "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; - - /** Three letter abbreviations of month names. (Jan==1, upper case) */ - public static final String[] MONTH_ABBREVIATIONS_3U = - {"XXX", "JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP", "OCT", "NOV", "DEC"}; - - /** Three letter abbreviations of month names. (Jan==1, lower case) */ - public static final String[] MONTH_ABBREVIATIONS_3L = - {"xxx", "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"}; - - // some useful formatting objects - /** Formats a year in YYYY format. */ - private static final DateFormat ms_dateFormatYear = new ThreadSafeDateFormat(new SimpleDateFormat("yyyy")); - /** Formats a month in MM format. */ - private static final DateFormat ms_dateFormatMonth = new ThreadSafeDateFormat(new SimpleDateFormat("MM")); - /** Formats a day in DD format. */ - private static final DateFormat ms_dateFormatDay = new ThreadSafeDateFormat(new SimpleDateFormat("dd")); - - private static final DateFormat ms_dateFormatHour = new ThreadSafeDateFormat(new SimpleDateFormat("HH")); - private static final DateFormat ms_dateFormatMinute = new ThreadSafeDateFormat(new SimpleDateFormat("mm")); - private static final DateFormat ms_dateFormatSecond = new ThreadSafeDateFormat(new SimpleDateFormat("ss")); - - - /** - * An easy way to get the OS-specific directory name component separator. - */ - private static final String DIR_SEP = File.separator; - - /** - * The "local" time zone. We make it explicit for testing purposes. - */ - private static TimeZone ms_localTimeZone = TimeZone.getDefault(); - - // ---------------------------------------------------------------- - /** Gets the "local" time zone. */ - public static TimeZone getLocalTimeZone() { - return ms_localTimeZone; - } - - // ---------------------------------------------------------------- - /** Gets the "local" time zone. */ - public static void setLocalTimeZone(TimeZone localTimeZone) { - Require.neqNull(localTimeZone, "localTimeZone"); - ms_localTimeZone = localTimeZone; - ms_dateFormatDay.setTimeZone(localTimeZone); - ms_dateFormatMonth.setTimeZone(localTimeZone); - ms_dateFormatYear.setTimeZone(localTimeZone); - } - - // ---------------------------------------------------------------- - public static boolean isLeapYear(int nYear) { - return 0 == nYear % 4 && (0 != nYear % 100 || 0 == nYear % 400) && 0 != nYear; - } - - // ---------------------------------------------------------------- - public static int getDaysInMonth(int nMonth, int nYear) { - Require.geq(nMonth, "nMonth", 1); - Require.leq(nMonth, "nMonth", 12); - return DAYS_PER_MONTH[nMonth] + (2 == nMonth && isLeapYear(nYear) ? 1 : 0); - } - - // ---------------------------------------------------------------- - /** - * Converts the given date (local timezone) to a string in MMDD format. - */ - public static String getDateAsMMDD(Date date) { - Require.neqNull(date, "date"); - return ms_dateFormatMonth.format(date) + ms_dateFormatDay.format(date); - } - - // ---------------------------------------------------------------- - /** - * Converts the given date (local timezone) to a string in MM format. - */ - public static String getDateAsMM(Date date) { - Require.neqNull(date, "date"); - return ms_dateFormatMonth.format(date); - } - - // ---------------------------------------------------------------- - /** - * Converts the given date (local timezone) to a string in DD format. - */ - public static String getDateAsDD(Date date) { - Require.neqNull(date, "date"); - return ms_dateFormatDay.format(date); - } - - // ---------------------------------------------------------------- - /** - * Converts the given date (local timezone) to a string in YYYYMM format. - */ - public static String getDateAsYYYYMM(Date date) { - Require.neqNull(date, "date"); - return ms_dateFormatYear.format(date) + ms_dateFormatMonth.format(date); - } - - // ---------------------------------------------------------------- - /** - * Converts the given date (local timezone) to a string in YYYYMMDD format. - */ - public static String getDateAsYYYYMMDD(Date date) { - Require.neqNull(date, "date"); - return ms_dateFormatYear.format(date) + ms_dateFormatMonth.format(date) + ms_dateFormatDay.format(date); - } - - // ---------------------------------------------------------------- - /** - * Converts the given date (local timezone) to a string in YYYYMMDD format. - */ - public static String getDateAsYYYYMMDD(long timeInMillis) { - Date date = new Date(timeInMillis); - return getDateAsYYYYMMDD(date); - } - - // ---------------------------------------------------------------- - /** - * Converts the given date (local timezone) to a string in YYYYMMDDTHH:MM:SS format. - */ - public static String getDateAsYYYYdMMdDDTHHcMMcSS(Date date) { - return ms_dateFormatYear.format(date) + "-" + ms_dateFormatMonth.format(date) + "-" - + ms_dateFormatDay.format(date) + "T" + ms_dateFormatHour.format(date) + ":" - + ms_dateFormatMinute.format(date) + ":" + ms_dateFormatSecond.format(date); - } - - // ---------------------------------------------------------------- - /** - * Converts the given date (local timezone) to a string in YYYYMMDDTHH:MM:SS format. - */ - public static String getDateAsYYYYdMMdDDTHHcMMcSS(long timeInMillis) { - Date date = new Date(timeInMillis); - return getDateAsYYYYdMMdDDTHHcMMcSS(date); - } - - - // ---------------------------------------------------------------- - /** - * Converts the given date (local timezone) to a string in MMDDYYYY format. - */ - public static String getDateAsMMDDYYYY(Date date) { - Require.neqNull(date, "date"); - return ms_dateFormatMonth.format(date) + ms_dateFormatDay.format(date) + ms_dateFormatYear.format(date); - } - - // ---------------------------------------------------------------- - /** - * Converts the given date (local timezone) to a string in YYYY/YYYYMM/YYYYMMDD format. - */ - public static String getDateAsPath(Date date) { - Require.neqNull(date, "date"); - String sYear = ms_dateFormatYear.format(date); - String sMonth = ms_dateFormatMonth.format(date); - String sDay = ms_dateFormatDay.format(date); - return sYear + DIR_SEP + sYear + sMonth + DIR_SEP + sYear + sMonth + sDay; - } - - // ---------------------------------------------------------------- - /** - * Converts the given integer in YYYYMMDD format to a string in YYYY/YYYYMM/YYYYMMDD format. - */ - public static String getYyyymmddIntAsPath(int nDateYyyymmdd) { - String sYyyymmdd = "00000000" + Integer.toString(nDateYyyymmdd); - sYyyymmdd = sYyyymmdd.substring(sYyyymmdd.length() - 8); - String sYear = sYyyymmdd.substring(0, 4); - String sMonth = sYyyymmdd.substring(4, 6); - String sDay = sYyyymmdd.substring(6, 8); - return sYear + DIR_SEP + sYear + sMonth + DIR_SEP + sYear + sMonth + sDay; - } - - // ---------------------------------------------------------------- - /** - * Gets the download path, in [DownloadBaseDir]/sDataSubdir/YYYY/YYYYMM/YYYYMMDD format given a date (local - * timezone). - */ - public static String getDateDownloadPath(PropertyFile configuration, String sDataSubdir, Date date) { - Require.nonempty(sDataSubdir, "sDataSubdir"); - Require.neqNull(date, "date"); - return configuration.getProperty("DownloadBaseDir") + DIR_SEP + sDataSubdir + DIR_SEP + getDateAsPath(date); - } - - // ---------------------------------------------------------------- - /** - * Gets the download path, in [DownloadBaseDir]/sDataSubdir/YYYY/YYYYMM/YYYYMMDD format given an integer in YYYYMMDD - * format. - */ - public static String getYyyymmddIntDownloadPath(PropertyFile configuration, String sDataSubdir, int nDateYyyymmdd) { - Require.nonempty(sDataSubdir, "sDataSubdir"); - return configuration.getProperty("DownloadBaseDir") + DIR_SEP + sDataSubdir + DIR_SEP - + getYyyymmddIntAsPath(nDateYyyymmdd); - } - - // ---------------------------------------------------------------- - /** Gets a date object representing the time 24 hours ago. */ - public static Date getDateYesterday() { - long timeNow = System.currentTimeMillis(); - long timeYesterday = timeNow - MILLIS_PER_DAY; - return new Date(timeYesterday); - } - - // ---------------------------------------------------------------- - /** - * Gets a date object representing the next day at the same hour (which may not be exactly 24 hours in the future). - */ - public static Date getNextDaySameTime(Date baseline, TimeZone zone) { - Require.neqNull(baseline, "baseline"); - Require.neqNull(zone, "zone"); - Calendar calendar = Calendar.getInstance(zone); - calendar.setTime(baseline); - calendar.add(Calendar.DATE, 1); - return calendar.getTime(); - } - - // ---------------------------------------------------------------- - /** - * Subtracts zero or more 24hr periods from the given date until the day of week for the resulting date (local - * timezone) is a valid day according to the mask. If the strict flag is true, optional days are not considered - * valid. - *

- * See {@link #validateDayOfWeekMask}. - */ - public static Date getMostRecentValidDate(Date date, String sValidDaysMask, boolean bStrict) { - Require.neqNull(date, "date"); - validateDayOfWeekMask(sValidDaysMask, bStrict); - - Calendar calendar = Calendar.getInstance(ms_localTimeZone); - while (true) { - calendar.setTime(date); - char chDayType = sValidDaysMask.charAt(calendar.get(Calendar.DAY_OF_WEEK) - Calendar.SUNDAY); - if (DAY_VALID == chDayType || (!bStrict && DAY_OPTIONAL == chDayType)) { - break; - } - date = new Date(date.getTime() - MILLIS_PER_DAY); - } - return date; - } - - // ---------------------------------------------------------------- - /** - * Adds one or more 24hr periods from the given date until the day of week for the resulting date (local timezone) - * is a valid day according to the mask. If the strict flag is true, optional days are not considered valid. - *

- * See {@link #validateDayOfWeekMask}. - */ - public static Date getNextValidDate(Date date, String sValidDaysMask, boolean bStrict) { - Require.neqNull(date, "date"); - validateDayOfWeekMask(sValidDaysMask, bStrict); - - Calendar calendar = Calendar.getInstance(ms_localTimeZone); - while (true) { - date = new Date(date.getTime() + MILLIS_PER_DAY); - calendar.setTime(date); - char chDayType = sValidDaysMask.charAt(calendar.get(Calendar.DAY_OF_WEEK) - Calendar.SUNDAY); - if (DAY_VALID == chDayType || (!bStrict && DAY_OPTIONAL == chDayType)) { - break; - } - } - return date; - } - - // ---------------------------------------------------------------- - /** - * Returns the validity flag from the mask for the given date (local timezone). - *

- * See {@link #validateDayOfWeekMask}. - */ - public static int getDayValidity(Date date, String sValidDaysMask) { - Require.neqNull(date, "date"); - validateDayOfWeekMask(sValidDaysMask, DAYMASK_NOT_STRICT); - return sValidDaysMask.charAt(getDayOfWeek(date)); - } - - // ---------------------------------------------------------------- - /** - * Throws a requirement exception if the given day of week mask is not valid. There must be at least one valid day - * in the mask. If the strict flag is set, optional days are not considered valid. - *

- * See {@link #DAY_VALID}, {@link #DAY_INVALID}, {@link #DAY_OPTIONAL}, {@link #DAYMASK_STRICT}, - * {@link #DAYMASK_NOT_STRICT} - */ - public static void validateDayOfWeekMask(String sValidDaysMask, boolean bStrict) { - Require.neqNull(sValidDaysMask, "sValidDaysMask", 1); - Require.eq(sValidDaysMask.length(), "sValidDaysMask.length()", DAYS_PER_WEEK, 1); - int nValidDaysFound = 0; - for (int nIndex = 0; nIndex < DAYS_PER_WEEK; nIndex++) { - char chDayType = sValidDaysMask.charAt(nIndex); - Require.requirement(DAY_INVALID == chDayType || DAY_VALID == chDayType || DAY_OPTIONAL == chDayType, - "DAY_INVALID==chDayType || DAY_VALID==chDayType || DAY_OPTIONAL==chDayType", 1); - if (DAY_VALID == chDayType || (!bStrict && DAY_OPTIONAL == chDayType)) { - nValidDaysFound++; - } - } - Require.gtZero(nValidDaysFound, "nValidDaysFound", 1); - } - - // ---------------------------------------------------------------- - /** - * Gets the day of the week (Su == 0) for the given date (local timezone). - */ - public static int getDayOfWeek(Date date) { - Require.neqNull(date, "date"); - Calendar calendar = Calendar.getInstance(ms_localTimeZone); - calendar.setTime(date); - return calendar.get(Calendar.DAY_OF_WEEK) - Calendar.SUNDAY; - } - - // ---------------------------------------------------------------- - /** - * Gets the current date (local timezone) as an integer, in YYYYMMDD format. - */ - public static int getDateTodayAsYyyymmddInt() { - return getDateAsYyyymmddInt(new Date()); - } - - // ---------------------------------------------------------------- - /** - * Gets the given date (local timezone) as an integer, in YYYYMMDD format. - */ - public static int getDateAsYyyymmddInt(Date date) { - Require.neqNull(date, "date"); - Calendar calendar = Calendar.getInstance(ms_localTimeZone); - calendar.setTime(date); - return calendar.get(Calendar.YEAR) * 10000 + (calendar.get(Calendar.MONTH) + 1) * 100 - + calendar.get(Calendar.DAY_OF_MONTH); - } - - // ---------------------------------------------------------------- - /** Converts an integer in YYYYMMDD format into "YYYY-MM-DD". */ - public static String formatYyyymmddIntAsIso(int nDateYyyymmdd) { - return formatYyyymmddStringAsIso(Integer.toString(nDateYyyymmdd)); - } - - // ---------------------------------------------------------------- - /** Converts an integer in YYYYMMDD format into "MM/DD/YYYY". */ - public static String formatYyyymmddIntAsUs(int nDateYyyymmdd) { - return formatYyyymmddStringAsUs(Integer.toString(nDateYyyymmdd)); - } - - // ---------------------------------------------------------------- - /** Converts a String in YYYYMMDD format into "YYYY-MM-DD". */ - public static String formatYyyymmddStringAsIso(String sDateYyyymmdd) { - Require.neqNull(sDateYyyymmdd, "sDateYyyymmdd"); - Require.eq(sDateYyyymmdd.length(), "sDateYyyymmdd.length()", 8); - return sDateYyyymmdd.substring(0, 4) + "-" + sDateYyyymmdd.substring(4, 6) + "-" - + sDateYyyymmdd.substring(6, 8); - } - - // ---------------------------------------------------------------- - /** Converts a String in YYYYMMDD format into "MM/DD/YYYY". */ - public static String formatYyyymmddStringAsUs(String sDateYyyymmdd) { - Require.neqNull(sDateYyyymmdd, "sDateYyyymmdd"); - Require.eq(sDateYyyymmdd.length(), "sDateYyyymmdd.length()", 8); - return sDateYyyymmdd.substring(4, 6) + "/" + sDateYyyymmdd.substring(6, 8) + "/" - + sDateYyyymmdd.substring(0, 4); - } - - // ---------------------------------------------------------------- - /** Converts a String in (M|MM)/(D|DD)/(YY|YYYY) format into "YYYY-MM-DD". */ - public static String formatMmddyyyyStringAsIso(String sDateMmddyyyy) { - Require.neqNull(sDateMmddyyyy, "sDateMmddyyyy"); - String[] date = sDateMmddyyyy.split("/"); - String res; - - res = ((date[2].length() == 2) ? "20" + date[2] : date[2]); - res += "-" + ((date[0].length() == 1) ? "0" + date[0] : date[0]); - res += "-" + ((date[1].length() == 1) ? "0" + date[1] : date[1]); - - Require.eq(res.length(), "sDateMmddyyyy.length()", 10); - return res; - } - - // ---------------------------------------------------------------- - /** Converts a String in (M|MM)/(D|DD)/YYYY format into "YYYY-MM-DD". */ - public static String formatMmddyyyyStringAsIsoAllowNull(String sDateMmddyyyy) { - if (null == sDateMmddyyyy || sDateMmddyyyy.length() == 0) { - return ""; - } - Require.neqNull(sDateMmddyyyy, "sDateMmddyyyy"); - String[] date = sDateMmddyyyy.split("/"); - String res; - - res = date[2]; - res += "-" + ((date[0].length() == 1) ? "0" + date[0] : date[0]); - res += "-" + ((date[1].length() == 1) ? "0" + date[1] : date[1]); - - Require.eq(res.length(), "sDateMmddyyyy.length()", 10); - return res; - } - - - // ---------------------------------------------------------------- - /** Converts a String in DDM3UYYYY format into "YYYY-MM-DD". */ - public static String formatddM3UyyyyStringAsIso(String sDateddM3Uyyyy) { - Require.neqNull(sDateddM3Uyyyy, "sDateddM3Uyyyy"); - String res; - - res = sDateddM3Uyyyy.substring(5); - int monthValue = Arrays.asList(MONTH_ABBREVIATIONS_3U).indexOf(sDateddM3Uyyyy.substring(2, 5)); - res += "-" + ((monthValue < 10) ? "0" + monthValue : monthValue); - res += "-" + (sDateddM3Uyyyy.substring(0, 2)); - - Require.eq(res.length(), "sDateddM3Uyyyy.length()", 10); - return res; - } - - // ---------------------------------------------------------------- - /** Converts a String in DD-MMM-YYYY format into "YYYY-MM-DD". */ - public static String formatddMMMyyyyStringAsIso(String sDateddMMMyyyy) { - Require.neqNull(sDateddMMMyyyy, "sDateddMMMyyyy"); - String[] date = sDateddMMMyyyy.split("-"); - String res; - - res = date[2]; - int monthValue = Arrays.asList(MONTH_ABBREVIATIONS_3U).indexOf(date[1]); - res += "-" + ((monthValue < 10) ? "0" + monthValue : monthValue); - res += "-" + ((date[0].length() == 1) ? "0" + date[0] : date[0]); - - Require.eq(res.length(), "sDateddmmmyyyy.length()", 10); - return res; - } - - // ---------------------------------------------------------------- - /** Converts a String in DD-MMM-YY format into "YYYY-MM-DD". */ - public static String formatddMMMyyStringAsIso(String sDateddMMMyy) { - Require.neqNull(sDateddMMMyy, "sDateddMMMyy"); - String[] date = sDateddMMMyy.split("-"); - String res; - - res = date[2]; - int monthValue = Arrays.asList(MONTH_ABBREVIATIONS_3U).indexOf(date[1].toUpperCase()); - res += "-" + ((monthValue < 10) ? "0" + monthValue : monthValue); - res += "-" + ((date[0].length() == 1) ? "0" + date[0] : date[0]); - - Require.eq(res.length(), "sDateddmmmyyyy.length()", 10); - return res; - } - - - // ------------------------------------------------------------------ - /** Converts a String in "Mmm dd, YYYY" format int "YYYY-MM-DD". */ - public static String formatMmmddcYYYYStringAsIso(String sDateMmmddcYYYY) { - Require.neqNull(sDateMmmddcYYYY, "sDateMmmddcYYYY"); - String[] date = sDateMmmddcYYYY.split("[ ,]"); - String res; - - res = date[3]; - int monthValue = Arrays.asList(MONTH_ABBREVIATIONS_3T).indexOf(date[0]); - res += "-" + ((monthValue < 10) ? "0" + monthValue : monthValue); - res += "-" + date[1]; - - Require.eq(res.length(), "sDateMmmddcYYYY.length()", 10); - return res; - } - - // ------------------------------------------------------------------ - /** Converts a String in "YYYY-MM-DD" format into "MM/DD/YYYY" format. */ - public static String formatIsoAsMMsDDsYYYYString(String sDateYYYYdMMdDD) { - Require.neqNull(sDateYYYYdMMdDD, "sDateYYYYdMMdDD"); - String[] date = sDateYYYYdMMdDD.split("-"); - String res = date[1] + "/" + date[2] + "/" + date[0]; - Require.eq(res.length(), "sDateYYYYdMMdDD.length()", 10); - return res; - } - - /** - * Converts a date string into a date. - * - * @param date - * @param sourceFormat - * @param resultFormat - * @return date - * @throws ParseException - */ - public static String formatDateFromStringToString(String date, String sourceFormat, String resultFormat) { - final DateFormat sourceDateFormat = new SimpleDateFormat(sourceFormat); - final DateFormat resultDateFormat = new SimpleDateFormat(resultFormat); - return formatDateFromFormatToFormat(date, sourceDateFormat, resultDateFormat); - } - - /** - * Converts a date string into a date. - * - * @param date - * @param sourceDateFormat - * @param resultDateFormat - * @return date - * @throws ParseException - */ - public static String formatDateFromFormatToFormat(String date, DateFormat sourceDateFormat, - DateFormat resultDateFormat) { - try { - return resultDateFormat.format(sourceDateFormat.parse(date)); - } catch (ParseException e) { - throw new RuntimeException(e); - } - } - - // ################################################################ - - // ---------------------------------------------------------------- - /** - * Returns the absolute timestamp of the most recent occurrence (before or exactly on the - * referenceTimestamp) of a daily event. The time of day is taken from - * sPropertyNameRoot.time in "h:mm a" format. The time zone for calculations is taken from - * sPropertyNameRoot.timeZone. - */ - public static Date getTimestampOfMostRecentDailyEvent(PropertyFile configuration, String sPropertyNameRoot, - Date referenceTimestamp) { - Require.nonempty(sPropertyNameRoot, "sPropertyNameRoot"); - Require.neqNull(referenceTimestamp, "referenceTimestamp"); - - // get the time zone of the event from the system properties - TimeZone timeZone = getTimeZoneOfEvent(configuration, sPropertyNameRoot); - - // get the time of day of the event from the system properties - Calendar eventTimestampCalendar = buildEventTimestampCalendar(timeZone, sPropertyNameRoot, configuration); - - // determine the exact timestamp of when the event happens today - Calendar referenceTimestampCalendar = Calendar.getInstance(timeZone); - referenceTimestampCalendar.setTime(referenceTimestamp); - eventTimestampCalendar.set( - referenceTimestampCalendar.get(Calendar.YEAR), - referenceTimestampCalendar.get(Calendar.MONTH), - referenceTimestampCalendar.get(Calendar.DAY_OF_MONTH)); - - // if the event happens in the future, then the most recent occurrence was the one that happened one day ago - if (eventTimestampCalendar.getTimeInMillis() > referenceTimestampCalendar.getTimeInMillis()) { - eventTimestampCalendar.add(Calendar.DAY_OF_MONTH, -1); - } - - return eventTimestampCalendar.getTime(); - } - - // ---------------------------------------------------------------- - /** - * Returns the absolute timestamp of the occurrence of a daily event that happens in the same "day" as right now. - * The time of day of the event is taken from sPropertyNameRoot.time in "h:mm a" format. The - * time zone for calculations (and for determining the boundaries of "today") is taken from - * sPropertyNameRoot.timeZone. - */ - public static Date getTimestampOfEventToday(PropertyFile configuration, String sPropertyNameRoot) { - Require.nonempty(sPropertyNameRoot, "sPropertyNameRoot"); - - // get the time zone of the event from the system properties - TimeZone timeZone = getTimeZoneOfEvent(configuration, sPropertyNameRoot); - - // get the time of day of the event from the system properties - Calendar eventTimestampCalendar = buildEventTimestampCalendar(timeZone, sPropertyNameRoot, configuration); - - // determine the exact timestamp of when the event happens today - Calendar referenceTimestampCalendar = Calendar.getInstance(timeZone); - eventTimestampCalendar.set( - referenceTimestampCalendar.get(Calendar.YEAR), - referenceTimestampCalendar.get(Calendar.MONTH), - referenceTimestampCalendar.get(Calendar.DAY_OF_MONTH)); - - return eventTimestampCalendar.getTime(); - } - - // ---------------------------------------------------------------- - /** - * Returns the absolute timestamp of the occurrence of a daily event that happens in the same "day" as right now. - * The time of day of the event is taken from sPropertyNameRoot.time in "h:mm a" format. The - * time zone for calculations (and for determining the boundaries of "today") is taken from - * sPropertyNameRoot.timeZone. - */ - public static Date getTimestampOfEventToday(PropertyFile configuration, String sPropertyNameRoot, long nNowMillis) { - Require.nonempty(sPropertyNameRoot, "sPropertyNameRoot"); - - // get the time zone of the event from the system properties - TimeZone timeZone = getTimeZoneOfEvent(configuration, sPropertyNameRoot); - - // get the time of day of the event from the system properties - Calendar eventTimestampCalendar = buildEventTimestampCalendar(timeZone, sPropertyNameRoot, configuration); - - // determine the exact timestamp of when the event happens today - Calendar referenceTimestampCalendar = Calendar.getInstance(timeZone); - referenceTimestampCalendar.setTimeInMillis(nNowMillis); - eventTimestampCalendar.set( - referenceTimestampCalendar.get(Calendar.YEAR), - referenceTimestampCalendar.get(Calendar.MONTH), - referenceTimestampCalendar.get(Calendar.DAY_OF_MONTH)); - - return eventTimestampCalendar.getTime(); - } - - // ---------------------------------------------------------------- - private static Calendar buildEventTimestampCalendar(TimeZone timeZone, String sPropertyNameRoot, - PropertyFile configuration) { - String sTimeProperty = sPropertyNameRoot + ".time"; - String sTime = configuration.getProperty(sTimeProperty); - Calendar eventTimestampCalendar = Calendar.getInstance(timeZone); - SimpleDateFormat timeFormat = new SimpleDateFormat("h:mm:ss a"); - timeFormat.setCalendar(eventTimestampCalendar); - try { - timeFormat.parse(sTime); - } catch (ParseException e) { - timeFormat = new SimpleDateFormat("h:mm a"); - timeFormat.setCalendar(eventTimestampCalendar); - try { - timeFormat.parse(sTime); - } catch (ParseException e2) { - throw Require.exceptionNeverCaught("Value of property " + sTimeProperty + " (\"" + sTime - + "\") not in proper format (\"" + timeFormat.toPattern() + "\").", e2); - } - } - return eventTimestampCalendar; - } - - // ---------------------------------------------------------------- - /** - * Gets the timestamp of an event based upon a daily event and a date (retrieved from properties) - */ - public static Date getTimestampOfEvent(PropertyFile configuration, String sEventPropertyRoot, - String sDateProperty) { - Require.nonempty(sEventPropertyRoot, "sEventPropertyRoot"); - Require.nonempty(sDateProperty, "sDateProperty"); - - // get the time zone of the event from the system properties - TimeZone timeZone = getTimeZoneOfEvent(configuration, sEventPropertyRoot); - - // get the time of day of the event from the system properties - Calendar eventTimestampCalendar = buildEventTimestampCalendar(timeZone, sEventPropertyRoot, configuration); - - // parse the date string and set the year, month, and day of the timestamp we are building - // note: time zone is irrelevant for the next step because we just want the numbers - we could use a regexp. - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - String sDate = configuration.getProperty(sDateProperty); - try { - dateFormat.parse(sDate); - } catch (ParseException e) { - throw Require.exceptionNeverCaught( - sDateProperty + " (\"" + sDate + "\") not in \"" + dateFormat.toPattern() + "\" format.", e); - } - Calendar dateCalendar = dateFormat.getCalendar(); - - // set the year, month, and day - eventTimestampCalendar.set(dateCalendar.get(Calendar.YEAR), dateCalendar.get(Calendar.MONTH), - dateCalendar.get(Calendar.DAY_OF_MONTH)); - - return eventTimestampCalendar.getTime(); - } - - // ---------------------------------------------------------------- - /** - * Gets the timestamp of an event based upon a daily event and a date specified by year, month (jan=1), day - */ - public static Date getTimestampOfEvent(PropertyFile configuration, String sEventPropertyRoot, int nYear, int nMonth, - int nDay) { - Require.nonempty(sEventPropertyRoot, "sEventPropertyRoot"); - - // get the time zone of the event from the system properties - TimeZone timeZone = getTimeZoneOfEvent(configuration, sEventPropertyRoot); - - // get the time of day of the event from the system properties - Calendar eventTimestampCalendar = buildEventTimestampCalendar(timeZone, sEventPropertyRoot, configuration); - - // set the year, month, and day - eventTimestampCalendar.set(nYear, nMonth - 1, nDay); - - return eventTimestampCalendar.getTime(); - } - - // ---------------------------------------------------------------- - /** - * Gets the timestamp of an event based upon a daily event and a date in YYYYMMDD format - */ - public static Date getTimestampOfEvent(PropertyFile configuration, String sEventPropertyRoot, int nYYYYMMDD) { - Require.nonempty(sEventPropertyRoot, "sEventPropertyRoot"); - return getTimestampOfEvent(configuration, sEventPropertyRoot, nYYYYMMDD / 10000, (nYYYYMMDD / 100) % 100, - nYYYYMMDD % 100); - } - - // ---------------------------------------------------------------- - /** Gets the time zone associated with a particular daily event. */ - public static TimeZone getTimeZoneOfEvent(PropertyFile configuration, String sPropertyNameRoot) { - Require.nonempty(sPropertyNameRoot, "sPropertyNameRoot"); - return TimeZone.getTimeZone(configuration.getProperty(sPropertyNameRoot + ".timeZone")); - } - - // ---------------------------------------------------------------- - /** - * Returns a date (noon in the local time zone) which is the date of the most recent occurrence (before or exactly - * on the referenceTimestamp) of the specified event, in the event's timezone. - */ - public static Date getDateOfMostRecentDailyEvent(PropertyFile configuration, String sPropertyNameRoot, - Date referenceTimestamp) { - Require.nonempty(sPropertyNameRoot, "sPropertyNameRoot"); - Require.neqNull(referenceTimestamp, "referenceTimestamp"); - Date eventTimestamp = getTimestampOfMostRecentDailyEvent(configuration, sPropertyNameRoot, referenceTimestamp); - Calendar sourceCalendar = Calendar.getInstance(getTimeZoneOfEvent(configuration, sPropertyNameRoot)); - sourceCalendar.setTime(eventTimestamp); - Calendar targetCalendar = Calendar.getInstance(ms_localTimeZone); - targetCalendar.clear(); - targetCalendar.set(sourceCalendar.get(Calendar.YEAR), sourceCalendar.get(Calendar.MONTH), - sourceCalendar.get(Calendar.DAY_OF_MONTH), 12, 0, 0); - return targetCalendar.getTime(); - } - - // ---------------------------------------------------------------- - /** - * Returns a date (noon in the local time zone) which is the date of the most recent occurrence (before or exactly - * on the referenceTimestamp) of the specified event, in the event's timezone. If the (strict) valid - * days mask indicates that the date is not valid, days will be subtracted until the date is valid. - *

- * See {@link #validateDayOfWeekMask}. - */ - public static Date getDateOfMostRecentDailyEvent(PropertyFile configuration, String sPropertyNameRoot, - Date referenceTimestamp, String sValidDaysMask) { - Require.nonempty(sPropertyNameRoot, "sPropertyNameRoot"); - Require.neqNull(referenceTimestamp, "referenceTimestamp"); - validateDayOfWeekMask(sValidDaysMask, DAYMASK_STRICT); - Calendar calendar = Calendar.getInstance(ms_localTimeZone); - calendar.setTime(getDateOfMostRecentDailyEvent(configuration, sPropertyNameRoot, referenceTimestamp)); - while (true) { - char chDayType = sValidDaysMask.charAt(calendar.get(Calendar.DAY_OF_WEEK) - Calendar.SUNDAY); - if (DAY_VALID == chDayType) { - break; - } - calendar.add(Calendar.DATE, -1); - } - return calendar.getTime(); - } - - // ---------------------------------------------------------------- - /** - * Wraps a "daily event" as an object. The time of day of the event is taken from - * sPropertyNameRoot.time in "h:mm a" format. The time zone for calculations (and for - * determining the boundaries of "today") is taken from sPropertyNameRoot.timeZone. - */ - public static class DailyEvent { - - private final PropertyFile m_configuration; - private final String m_sPropertyNameRoot; - - // ------------------------------------------------------------ - public DailyEvent(PropertyFile configuration, String sPropertyNameRoot) { - Require.neqNull(configuration, "configuration"); - Require.nonempty(sPropertyNameRoot, "sPropertyNameRoot"); - try { - buildEventTimestampCalendar(getTimeZoneOfEvent(configuration, sPropertyNameRoot), sPropertyNameRoot, - configuration); - } catch (RequirementFailure e) { - throw e.adjustForDelegatingMethod(); - } - m_configuration = configuration; - m_sPropertyNameRoot = sPropertyNameRoot; - } - - // ------------------------------------------------------------ - public long getTimestampOfEventToday(long nNow) { - return DateUtil.getTimestampOfEventToday(m_configuration, m_sPropertyNameRoot, nNow).getTime(); - } - - // ------------------------------------------------------------ - @Override - public String toString() { - return m_configuration.getProperty(m_sPropertyNameRoot + ".time") + ", " - + m_configuration.getProperty(m_sPropertyNameRoot + ".timeZone"); - } - } - - // ################################################################ - - // ---------------------------------------------------------------- - /** Parse the given string into a date with the given format. */ - public static long parse(String sTime, String sFormat) throws ParseException { - SimpleDateFormat simpleDateFormat = new SimpleDateFormat(sFormat); - simpleDateFormat.setTimeZone(ms_localTimeZone); - return simpleDateFormat.parse(sTime).getTime(); - } - - // ---------------------------------------------------------------- - /** - * Determines if two dates are on the same calendar day. - * - * @param d1 first date. - * @param d2 second date. - * @param tz timezone for the calendar. - * @return true if the dates are on the same calendar day, and false otherwise. - */ - public static boolean isSameDay(Date d1, Date d2, TimeZone tz) { - Calendar calendar1 = new GregorianCalendar(tz); - calendar1.setTime(d1); - Calendar calendar2 = new GregorianCalendar(tz); - calendar2.setTime(d2); - - if (calendar1.get(Calendar.YEAR) != calendar2.get(Calendar.YEAR)) { - return false; - } else if (calendar1.get(Calendar.DAY_OF_YEAR) != calendar2.get(Calendar.DAY_OF_YEAR)) { - return false; - } else { - return true; - } - } - - // ################################################################ - - // ---------------------------------------------------------------- - /** - * Returns a string in "0d 0h 0m 0.000'000'000s" format from a time interval in nanoseconds. - */ - public static String formatIntervalNanos(long tsInterval) { - return internalFormatInterval(tsInterval, 3); - } - - // ---------------------------------------------------------------- - /** - * Returns a string in "0d 0h 0m 0.000'000s" format from a time interval in microseconds. - */ - public static String formatIntervalMicros(long tsInterval) { - return internalFormatInterval(tsInterval, 2); - } - - // ---------------------------------------------------------------- - /** - * Returns a string in "0d 0h 0m 0.000s" format from a time interval in milliseconds. - */ - public static String formatIntervalMillis(long tsInterval) { - return internalFormatInterval(tsInterval, 1); - } - - // ---------------------------------------------------------------- - private static String internalFormatInterval(long tsInterval, int nThousands) { - - StringBuilder stringBuilder = new StringBuilder(); - if (tsInterval < 0) { - stringBuilder.append("-"); - tsInterval = -tsInterval; - } - - long tsSeconds = tsInterval / THOUSANDS[nThousands]; - - boolean bNeedUnit = false; - if (tsSeconds > SECONDS_PER_DAY) { - long nDays = tsSeconds / SECONDS_PER_DAY; - tsSeconds %= SECONDS_PER_DAY; - stringBuilder.append(nDays).append("d "); - bNeedUnit = true; - } - if (tsSeconds > SECONDS_PER_HOUR || bNeedUnit) { - long nHours = tsSeconds / SECONDS_PER_HOUR; - tsSeconds %= SECONDS_PER_HOUR; - stringBuilder.append(nHours).append("h "); - bNeedUnit = true; - } - if (tsSeconds > SECONDS_PER_MINUTE || bNeedUnit) { - long nMinutes = tsSeconds / SECONDS_PER_MINUTE; - tsSeconds %= SECONDS_PER_MINUTE; - stringBuilder.append(nMinutes).append("m "); - } - stringBuilder.append(tsSeconds).append('.'); - - long tsFractions = tsInterval % THOUSANDS[nThousands]; - - for (int nIndex = nThousands; nIndex > 0; nIndex--) { - // if (nIndex!=nThousands) { stringBuilder.append('\''); } - long tsThousand = tsFractions / THOUSANDS[nIndex - 1]; - tsFractions %= THOUSANDS[nIndex - 1]; - - String sLeadingZeros; - if (tsThousand >= 100) { - sLeadingZeros = ""; - } else if (tsThousand >= 10) { - sLeadingZeros = "0"; - } else { - sLeadingZeros = "00"; - } - stringBuilder.append(sLeadingZeros).append(tsThousand); - } - return stringBuilder.append("s").toString(); - } - - // ---------------------------------------------------------------- - /** - * Formats the given microsecond timestamp with the given date formatter and then appends the last three microsend - * digits. - */ - public static String formatWithTrailingMicros(DateFormat dateFormat, long nTimestampMicros) { - return dateFormat.format(nTimestampMicros / DateUtil.MICROS_PER_MILLI) - + DateUtil.formatTrailingMicros(nTimestampMicros); - } - - // ---------------------------------------------------------------- - /** - * Returns the last three digits of the given microsecond timestamp as a string, suitable for appending to a - * timestamp formatted to millisecond precision. - */ - public static String formatTrailingMicros(long nTimestampMicros) { - nTimestampMicros = nTimestampMicros % 1000; - String sLeadingZeros; - if (nTimestampMicros >= 100) { - sLeadingZeros = ""; - } else if (nTimestampMicros >= 10) { - sLeadingZeros = "0"; - } else { - sLeadingZeros = "00"; - } - return sLeadingZeros + nTimestampMicros; - } -} diff --git a/FishUtil/src/main/java/io/deephaven/util/ExceptionUtil.java b/FishUtil/src/main/java/io/deephaven/util/ExceptionUtil.java deleted file mode 100644 index 759729e5f32..00000000000 --- a/FishUtil/src/main/java/io/deephaven/util/ExceptionUtil.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.util; - -import org.jetbrains.annotations.NotNull; - -/** - * Some utilities for inspecting exceptions - */ -public class ExceptionUtil { - public static boolean causedBy(@NotNull Throwable t, Class cause) { - Throwable curr = t; - while (curr != null) { - if (cause.isAssignableFrom(curr.getClass())) { - return true; - } - curr = curr.getCause(); - } - return false; - } -} diff --git a/FishUtil/src/main/java/io/deephaven/util/Mailer.java b/FishUtil/src/main/java/io/deephaven/util/Mailer.java deleted file mode 100644 index c520f647054..00000000000 --- a/FishUtil/src/main/java/io/deephaven/util/Mailer.java +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.util; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -public interface Mailer { - void sendEmail(String sender, String[] recipients, String subject, String msg) throws IOException; - - void sendEmail(String sender, String recipient, String subject, String msg) throws IOException; - - void sendHTMLEmail(String sender, String recipient, String subject, String msg) throws IOException; - - void sendEmail(String sender, String recipient, String subject, String msg, - List> extraHeaderEntries) throws IOException; -} diff --git a/FishUtil/src/main/java/io/deephaven/util/ThreadSafeDateFormat.java b/FishUtil/src/main/java/io/deephaven/util/ThreadSafeDateFormat.java deleted file mode 100644 index 18cf9f4a550..00000000000 --- a/FishUtil/src/main/java/io/deephaven/util/ThreadSafeDateFormat.java +++ /dev/null @@ -1,155 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.util; - -import io.deephaven.base.verify.Require; - -import java.text.AttributedCharacterIterator; -import java.text.DateFormat; -import java.text.FieldPosition; -import java.text.NumberFormat; -import java.text.ParseException; -import java.text.ParsePosition; -import java.util.Calendar; -import java.util.Date; -import java.util.TimeZone; - -// -------------------------------------------------------------------- -/** - * Wraps a {@link DateFormat} to provide a minimal level of thread safety that DateFormat is lacking (namely, preventing - * simultaneous calls to {@link #format} from separate threads from interfering with each other). - */ -public class ThreadSafeDateFormat extends DateFormat { - private final DateFormat m_dateFormat; - - public ThreadSafeDateFormat(DateFormat dateFormat) { - m_dateFormat = dateFormat; - } - - @Override - public Date parse(String source, ParsePosition pos) { - synchronized (m_dateFormat) { - return m_dateFormat.parse(source, pos); - } - } - - @Override - public StringBuffer format(Date date, StringBuffer toAppendTo, FieldPosition fieldPosition) { - synchronized (m_dateFormat) { - return m_dateFormat.format(date, toAppendTo, fieldPosition); - } - } - - @Override - public boolean isLenient() { - synchronized (m_dateFormat) { - return m_dateFormat.isLenient(); - } - } - - @Override - public void setLenient(boolean lenient) { - synchronized (m_dateFormat) { - m_dateFormat.setLenient(lenient); - } - } - - @Override - public NumberFormat getNumberFormat() { - synchronized (m_dateFormat) { - return m_dateFormat.getNumberFormat(); - } - } - - @Override - public void setNumberFormat(NumberFormat newNumberFormat) { - synchronized (m_dateFormat) { - m_dateFormat.setNumberFormat(newNumberFormat); - } - } - - @Override - public Calendar getCalendar() { - synchronized (m_dateFormat) { - return m_dateFormat.getCalendar(); - } - } - - @Override - public void setCalendar(Calendar newCalendar) { - synchronized (m_dateFormat) { - m_dateFormat.setCalendar(newCalendar); - } - } - - @Override - public TimeZone getTimeZone() { - synchronized (m_dateFormat) { - return m_dateFormat.getTimeZone(); - } - } - - @Override - public void setTimeZone(TimeZone zone) { - synchronized (m_dateFormat) { - m_dateFormat.setTimeZone(zone); - } - } - - @Override - public Date parse(String source) throws ParseException { - synchronized (m_dateFormat) { - return m_dateFormat.parse(source); - } - } - - @Override - public Object parseObject(String source, ParsePosition pos) { - synchronized (m_dateFormat) { - return m_dateFormat.parseObject(source, pos); - } - } - - @Override - public Object parseObject(String source) throws ParseException { - synchronized (m_dateFormat) { - return m_dateFormat.parseObject(source); - } - } - - @Override - public AttributedCharacterIterator formatToCharacterIterator(Object obj) { - synchronized (m_dateFormat) { - return m_dateFormat.formatToCharacterIterator(obj); - } - } - - @Override - public String toString() { - synchronized (m_dateFormat) { - return m_dateFormat.toString(); - } - } - - // ################################################################ - - @Override - public boolean equals(Object obj) { - Require.statementNeverExecuted(); - return super.equals(obj); - } - - @Override - public int hashCode() { - Require.statementNeverExecuted(); - return super.hashCode(); - } - - @Override - public Object clone() { - Require.statementNeverExecuted(); - return super.clone(); - } - -} diff --git a/FishUtil/src/main/java/io/deephaven/util/Validate.java b/FishUtil/src/main/java/io/deephaven/util/Validate.java deleted file mode 100644 index bb03259c765..00000000000 --- a/FishUtil/src/main/java/io/deephaven/util/Validate.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.util; - -public class Validate { - public static int validatePositiveInteger(String name, String s) throws NumberFormatException { - int i = 0; - - try { - i = Integer.parseInt(s); - } catch (NumberFormatException e) { - throw new NumberFormatException(name + " is not a valid number"); - } - - if (i <= 0) { - throw new NumberFormatException(name + " must be greater than zero"); - } - - return i; - } - - public static int validateInteger(String name, String s) throws NumberFormatException { - int i = 0; - - try { - i = Integer.parseInt(s); - } catch (NumberFormatException e) { - throw new NumberFormatException(name + " is not a valid number"); - } - - return i; - } - - public static double validatePositiveDouble(String name, String s) throws NumberFormatException { - double d = 0; - - try { - d = Double.parseDouble(s); - } catch (NumberFormatException e) { - throw new NumberFormatException(name + " is not a valid number"); - } - - if (d <= 0) { - throw new NumberFormatException(name + " must be greater than zero"); - } - - return d; - } - - public static double validateDouble(String name, String s) throws NumberFormatException { - double d = 0; - - try { - d = Double.parseDouble(s); - } catch (NumberFormatException e) { - throw new NumberFormatException(name + " is not a valid number"); - } - - return d; - } - - public static void validate(boolean b, String errorMsg) throws Exception { - if (!b) { - throw new Exception(errorMsg); - } - } - - public static void validateDouble(String name, double value, double min, double max, boolean inclusiveMin, - boolean inclusiveMax) throws Exception { - if (Double.isNaN(value)) { - throw new Exception(name + " may not be NaN"); - } - - if (inclusiveMin && value < min) { - throw new Exception(name + " must be greater than or equal to " + min); - } - - if (!inclusiveMin && value <= min) { - throw new Exception(name + " must be greater than " + min); - } - - if (inclusiveMax && value > max) { - throw new Exception(name + " must be less than or equal to " + max); - } - - if (!inclusiveMax && value >= max) { - throw new Exception(name + " must be less than " + max); - } - } - - public static void validateInteger(String name, int value, int min, int max, boolean inclusiveMin, - boolean inclusiveMax) throws Exception { - if (inclusiveMin && value < min) { - throw new Exception(name + " must be greater than or equal to " + min); - } - - if (!inclusiveMin && value <= min) { - throw new Exception(name + " must be greater than " + min); - } - - if (inclusiveMax && value > max) { - throw new Exception(name + " must be less than or equal to " + max); - } - - if (!inclusiveMax && value >= max) { - throw new Exception(name + " must be less than " + max); - } - } -} diff --git a/FishUtil/src/main/java/io/deephaven/util/formatters/ISO8601.java b/FishUtil/src/main/java/io/deephaven/util/formatters/ISO8601.java deleted file mode 100644 index b5f6a2da220..00000000000 --- a/FishUtil/src/main/java/io/deephaven/util/formatters/ISO8601.java +++ /dev/null @@ -1,120 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.util.formatters; - -import io.deephaven.configuration.Configuration; - -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.TimeZone; - -public class ISO8601 { - private static final ThreadLocal toISO8601Cache = new ThreadLocal(); - private static final ThreadLocal timeISO8601Cache = new ThreadLocal(); - private static final ThreadLocal dateISO8601Cache = new ThreadLocal(); - private static TimeZone TZ_SERVER = null; - - public static synchronized TimeZone serverTimeZone() { - if (TZ_SERVER == null) { - TZ_SERVER = Configuration.getInstance().getServerTimezone(); - } - return TZ_SERVER; - } - - public static DateFormat ISO8601DateFormat(TimeZone tz) { - SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); - df.setTimeZone(tz); - return df; - } - - public static DateFormat ISE8601TimeFormat() { - return ISO8601TimeFormat(serverTimeZone()); - } - - public static DateFormat ISO8601TimeFormat(TimeZone tz) { - SimpleDateFormat df = new SimpleDateFormat("HH:mm:ss.SSSZ"); - df.setTimeZone(tz); - return df; - } - - public static DateFormat ISE8601DateTimeFormat() { - return ISO8601DateTimeFormat(serverTimeZone()); - } - - public static DateFormat ISO8601DateTimeFormat(TimeZone tz) { - SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - df.setTimeZone(tz); - return df; - } - - public static String toISO8601(long millis) { - return toISO8601(new Date(millis), serverTimeZone()); - } - - public static String toISO8601(Date d) { - return toISO8601(d, serverTimeZone()); - } - - public static String toISO8601(long millis, TimeZone tz) { - return toISO8601(new Date(millis), tz); - } - - public static String toISO8601(Date d, TimeZone tz) { - DateFormat df = toISO8601Cache.get(); - if (df == null) { - df = ISO8601DateTimeFormat(tz); - toISO8601Cache.set(df); - } else { - df.setTimeZone(tz); - } - return df.format(d); - } - - public static String timeISO8601(long millis) { - return timeISO8601(new Date(millis), serverTimeZone()); - } - - public static String timeISO8601(Date d) { - return timeISO8601(d, serverTimeZone()); - } - - public static String timeISO8601(long millis, TimeZone tz) { - return timeISO8601(new Date(millis), tz); - } - - public static String timeISO8601(Date d, TimeZone tz) { - DateFormat df = timeISO8601Cache.get(); - if (df == null) { - df = ISO8601TimeFormat(tz); - timeISO8601Cache.set(df); - } else { - df.setTimeZone(tz); - } - return df.format(d); - } - - public static String dateISO8601(long millis) { - return dateISO8601(new Date(millis), serverTimeZone()); - } - - public static String dateISO8601(Date d) { - return dateISO8601(d, serverTimeZone()); - } - - public static String dateISO8601(long millis, TimeZone tz) { - return dateISO8601(new Date(millis), tz); - } - - public static String dateISO8601(Date d, TimeZone tz) { - DateFormat df = dateISO8601Cache.get(); - if (df == null) { - df = ISO8601DateFormat(tz); - dateISO8601Cache.set(df); - } else { - df.setTimeZone(tz); - } - return df.format(d); - } -} diff --git a/FishUtil/src/main/java/io/deephaven/util/signals/SignalSender.java b/FishUtil/src/main/java/io/deephaven/util/signals/SignalSender.java deleted file mode 100644 index c039e5a79ee..00000000000 --- a/FishUtil/src/main/java/io/deephaven/util/signals/SignalSender.java +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.util.signals; - -import io.deephaven.base.verify.Require; -import io.deephaven.io.logger.Logger; -import io.deephaven.io.logger.StreamLoggerImpl; -import org.jetbrains.annotations.NotNull; - -import java.io.IOException; - -public class SignalSender { - - private final Logger log; - private final boolean useNative; - - public SignalSender(@NotNull final Logger log, final boolean useNative) { - this.log = log; - this.useNative = useNative; - if (useNative) { - SignalUtils.loadNative(); - } - } - - /** - * Helper method - sends SIQQUIT to a process. If this process is a JVM, it will send a stack dump to stdout. - * - * @param processId The process ID to send the signal to - * @return true on success, false on error - */ - public boolean sendQuit(final int processId) { - return sendSignal(processId, SignalUtils.Signal.SIGQUIT); - } - - /** - * Helper method - sends SIQKILL to a process. - * - * @param processId The process ID to send the signal to - * @return true on success, false on error - */ - public boolean kill(final int processId) { - return sendSignal(processId, SignalUtils.Signal.SIGKILL); - } - - /** - * Helper method - sends SIGCONT to a process. - * - * @param processId The process ID to send the signal to - * @return true on success, false on error - */ - public boolean resume(final int processId) { - return sendSignal(processId, SignalUtils.Signal.SIGCONT); - } - - /** - * Helper method - sends SIGSTOP to a process. - * - * @param processId The process ID to send the signal to - * @return true on success, false on error - */ - public boolean suspend(final int processId) { - return sendSignal(processId, SignalUtils.Signal.SIGSTOP); - } - - /** - * Send the specified signal to the target process. - * - * @param processId The process ID to send the signal to - * @param signal The signal to send - * @return true on success, false on error - */ - private boolean sendSignal(final int processId, final SignalUtils.Signal signal) { - Require.gtZero(processId, "processId"); // Don't want to allow fancier usages for now. See 'man -s 2 kill'. - Require.neqNull(signal, "signal"); - - final int rc; - if (useNative) { - rc = SignalUtils.sendSignalNative(processId, signal.getSignalNumber()); - } else { - try { - rc = SignalUtils.sendSignalWithBinKill(processId, signal.getSignalName()); - } catch (IOException e) { - log.error().append("sendSignal: Exception while using /bin/kill to send ").append(signal.toString()) - .append(" to processId ").append(processId).append(": ").append(e).endl(); - return false; - } - } - - if (rc == 0) { - return true; - } - log.error().append("sendSignal: Error while using ").append(useNative ? "native code" : "/bin/kill") - .append(" to send ").append(signal.toString()) - .append(" to processId ").append(processId) - .append(": kill returned ").append(rc).endl(); - return false; - } - - /** - * Simple program for functionality testing. - * - * @param args [ <pid> <signal> <use native?> ] - */ - public static void main(final String... args) { - final int pid = Integer.parseInt(args[0]); - final SignalUtils.Signal signal = SignalUtils.Signal.valueOf(args[1]); - final boolean useNative = Boolean.valueOf(args[2]); - new SignalSender(new StreamLoggerImpl(), useNative).sendSignal(pid, signal); - } -} diff --git a/FishUtil/src/main/java/io/deephaven/util/signals/SignalUtils.java b/FishUtil/src/main/java/io/deephaven/util/signals/SignalUtils.java deleted file mode 100644 index 10aaf8bcd06..00000000000 --- a/FishUtil/src/main/java/io/deephaven/util/signals/SignalUtils.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.util.signals; - -import io.deephaven.util.OSUtil; - -import java.io.IOException; - -public class SignalUtils { - - /** - * What operating system does the JVM think we're on? - */ - private static final OSUtil.OSFamily OPERATING_SYSTEM = OSUtil.getOSFamily(); - - /** - * Placeholder value when we don't know a signal's number on the current OS. - */ - private static final int UNDEFINED_SIGNAL_NUMBER = Integer.MIN_VALUE; - - /** - * Supported signals. Be careful when adding new entries - as you can see, signal numbers don't always line up - * across operating systems. - */ - public enum Signal { - SIGINT("int", 2, 2, 2), SIGTERM("term", 15, 15, 15), SIGQUIT("quit", 3, 3, 3), SIGKILL("kill", 9, 9, - 9), SIGSTOP("stop", 19, 23, 17), SIGCONT("cont", 18, 25, 19); - - private final String signalName; - private final int signalNumber; - - Signal(final String signalName, final int linuxSignalNumber, final int solarisSignalNumber, - final int macOsSignalNumber) { - this.signalName = signalName; - switch (OPERATING_SYSTEM) { - case LINUX: - signalNumber = linuxSignalNumber; - break; - case MAC_OS: - signalNumber = macOsSignalNumber; - break; - case SOLARIS: - signalNumber = solarisSignalNumber; - break; - case WINDOWS: - default: - signalNumber = UNDEFINED_SIGNAL_NUMBER; - break; - } - } - - public String getSignalName() { - return signalName; - } - - public int getSignalNumber() { - if (signalNumber == UNDEFINED_SIGNAL_NUMBER) { - throw new UnsupportedOperationException(this + " is undefined on " + OPERATING_SYSTEM); - } - return signalNumber; - } - } - - /** - * Use /bin/kill to send a signal by name. - * - * @param processId The process ID to send the signal to - * @param signalName The name of the signal to send - * @return The exit value of the child process. - */ - @SuppressWarnings("WeakerAccess") - public static int sendSignalWithBinKill(final int processId, final String signalName) throws IOException { - final ProcessBuilder pb = new ProcessBuilder("/bin/kill", "-s", signalName, Integer.toString(processId)); - final Process p = pb.start(); - - try { - p.getErrorStream().close(); - p.getInputStream().close(); - p.getOutputStream().close(); - } catch (IOException e) { - throw new AssertionError("sendSignalWithBinKill: unexpected exception while closing child process streams: " - + e.getMessage(), e); - } - - while (true) { - try { - return p.waitFor(); - } catch (InterruptedException ignored) { - } - } - } - - /** - * Ensure that libraries have been loaded, before using sendSignalNative(...). - */ - @SuppressWarnings("WeakerAccess") - public static void loadNative() { - System.loadLibrary("FishCommon"); - } - - /** - * Use native code to send a signal by number. - * - * @param processId The process ID to send the signal to - * @param signalNumber The signal number to send - * @return The return value of kill(2). - */ - public static native int sendSignalNative(final int processId, final int signalNumber); -} diff --git a/FishUtil/src/main/java/io/deephaven/util/threads/ThreadDump.java b/FishUtil/src/main/java/io/deephaven/util/threads/ThreadDump.java deleted file mode 100644 index 344cc6e517f..00000000000 --- a/FishUtil/src/main/java/io/deephaven/util/threads/ThreadDump.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.util.threads; - -import io.deephaven.io.logger.Logger; - -import java.io.PrintStream; -import java.lang.management.ManagementFactory; -import java.lang.management.ThreadInfo; -import java.lang.management.ThreadMXBean; -import java.util.function.Consumer; - -/** - * A simple method for generating a Thread dump for this JVM; it doesn't do all the stuff that the kill -3 does; but you - * can easily run it from inside the JVM without having to send yourself a signal. - */ -public class ThreadDump { - @SuppressWarnings("WeakerAccess") - public static void threadDump(final PrintStream out) { - doDump(out::print); - } - - public static void threadDump(final Logger logger) { - doDump(arg -> logger.info().append(arg).endl()); - } - - @SuppressWarnings("WeakerAccess") - public static String threadDump() { - final StringBuilder builder = new StringBuilder(); - doDump(builder::append); - return builder.toString(); - } - - private static void doDump(Consumer output) { - ThreadMXBean threadMXBean = ManagementFactory.getPlatformMXBean(ThreadMXBean.class); - - ThreadInfo[] threadInfos = threadMXBean.dumpAllThreads(true, true); - - for (ThreadInfo threadInfo : threadInfos) { - output.accept(threadInfo.toString()); - } - } - - public static void main(String[] args) { - threadDump(System.out); - } -} diff --git a/IO/src/main/java/io/deephaven/io/NioUtil.java b/IO/src/main/java/io/deephaven/io/NioUtil.java deleted file mode 100644 index 4e24b19ac38..00000000000 --- a/IO/src/main/java/io/deephaven/io/NioUtil.java +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.io; - -import io.deephaven.base.LowGarbageArrayIntegerMap; -import io.deephaven.base.LowGarbageArrayList; -import io.deephaven.base.LowGarbageArraySet; -import io.deephaven.base.verify.Assert; -import io.deephaven.base.verify.Require; - -import java.lang.reflect.Field; -import java.nio.channels.Selector; -import java.nio.channels.spi.AbstractSelector; -import java.util.List; -import java.util.Set; - -// -------------------------------------------------------------------- -/** - * General utilities for NIO - */ -public class NioUtil { - - private static final String JAVA_8_SPEC_VERSION = "1.8"; - - // ---------------------------------------------------------------- - /** - * Use reflection to change the collection implementations so iteration operations used in the selector - * implementation will not produce garbage. - * - *

- * This is only applied when the system property {@code java.specification.version} is equal to "1.8". - * - *

- * We can do this because, by looking at the source code, we can tell that there are no simultaneous iterations so - * reusing one iterator is OK. Because of concurrent modification issues and thread safety issues, this is generally - * likely to be the case anyway. The implementation of selector is not likely to change between minor JDK revisions. - * A major JDK release might produce a rewrite, but in that case we can check the JDK version and apply the - * appropriate set of patches. - */ - public static Selector reduceSelectorGarbage(Selector selector) { - final String javaSpecificationVersion = System.getProperty("java.specification.version"); - if (JAVA_8_SPEC_VERSION.equals(javaSpecificationVersion)) { - return reduceSelectorGarbageImpl(selector); - } - return selector; - } - - private static Selector reduceSelectorGarbageImpl(Selector selector) { - try { - Class selectorImplClass = Class.forName("sun.nio.ch.SelectorImpl"); - Require.instanceOf(selector, "selector", selectorImplClass); - - Field cancelledKeysField = AbstractSelector.class.getDeclaredField("cancelledKeys"); - cancelledKeysField.setAccessible(true); - Set newCancelledKeys = new LowGarbageArraySet(); - cancelledKeysField.set(selector, newCancelledKeys); - - Field keysField = selectorImplClass.getDeclaredField("keys"); - keysField.setAccessible(true); - Field publicKeysField = selectorImplClass.getDeclaredField("publicKeys"); - publicKeysField.setAccessible(true); - Set newKeys = new LowGarbageArraySet(); - keysField.set(selector, newKeys); - publicKeysField.set(selector, newKeys); - - Field selectedKeysField = selectorImplClass.getDeclaredField("selectedKeys"); - selectedKeysField.setAccessible(true); - Field publicSelectedKeysField = selectorImplClass.getDeclaredField("publicSelectedKeys"); - publicSelectedKeysField.setAccessible(true); - Set newSelectedKeys = new LowGarbageArraySet(); - selectedKeysField.set(selector, newSelectedKeys); - publicSelectedKeysField.set(selector, newSelectedKeys); - - if (System.getProperty("os.name").startsWith("Windows") - && System.getProperty("java.vendor").startsWith("Oracle")) { - Class windowsSelectorImplClass = Class.forName("sun.nio.ch.WindowsSelectorImpl"); - Require.instanceOf(selector, "selector", windowsSelectorImplClass); - - Field threadsField = windowsSelectorImplClass.getDeclaredField("threads"); - threadsField.setAccessible(true); - List newThreads = new LowGarbageArrayList(); - threadsField.set(selector, newThreads); - - } else if (System.getProperty("os.name").startsWith("Linux")) { - Class ePollSelectorImplClass = Class.forName("sun.nio.ch.EPollSelectorImpl"); - Require.instanceOf(selector, "selector", ePollSelectorImplClass); - - Field fdToKeyField = ePollSelectorImplClass.getDeclaredField("fdToKey"); - fdToKeyField.setAccessible(true); - LowGarbageArrayIntegerMap newFdToKey = new LowGarbageArrayIntegerMap(); - fdToKeyField.set(selector, newFdToKey); - - } else if (System.getProperty("os.name").startsWith("SunOS")) { - Class devPollSelectorImplClass = Class.forName("sun.nio.ch.DevPollSelectorImpl"); - Require.instanceOf(selector, "selector", devPollSelectorImplClass); - - Field fdToKeyField = devPollSelectorImplClass.getDeclaredField("fdToKey"); - fdToKeyField.setAccessible(true); - LowGarbageArrayIntegerMap newFdToKey = new LowGarbageArrayIntegerMap(); - fdToKeyField.set(selector, newFdToKey); - } - - return selector; - } catch (final NoSuchFieldException | IllegalAccessException | ClassNotFoundException e) { - throw Assert.exceptionNeverCaught(e); - } - } -} diff --git a/IO/src/main/java/io/deephaven/io/sched/Job.java b/IO/src/main/java/io/deephaven/io/sched/Job.java deleted file mode 100644 index ead141f4164..00000000000 --- a/IO/src/main/java/io/deephaven/io/sched/Job.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.io.sched; - -import io.deephaven.base.log.LogOutput; -import io.deephaven.base.log.LogOutputAppendable; - -import java.nio.channels.SelectableChannel; -import java.io.IOException; - -/** - * This is the base class for jobs that can be invoked by the scheduler. - */ -public abstract class Job implements LogOutputAppendable { - - // -------------------------------------------------------------------------- - // public interface - // -------------------------------------------------------------------------- - - /** - * This method is invoked by the scheduler when the job's channel becomes ready. - * - * @param channel the channel which has become ready - * @param readyOps the operations which can be performed on this channel without blocking - * @returns the modified readyOps after the invocation; if non-zero, the job will be invoked again with these - * @throws IOException - if something bad happens - */ - public abstract int invoke(SelectableChannel channel, int readyOps, Runnable handoff) throws IOException; - - /** - * This method is invoked if the job times out. - */ - public abstract void timedOut(); - - /** - * This method is called if the job is explicitly cancelled before it becomes ready or times out. - */ - public abstract void cancelled(); - - // -------------------------------------------------------------------------- - // scheduler state management - // -------------------------------------------------------------------------- - - // TODO: currently, we assume that the scheduler is a singleton, or at the least - // TODO: that no job will be used with more than one scheduler throughout its lifetime. - // TODO: If this changes, we will have to change the state pointer to a set. - - /** the link to the scheduler's state for this job */ - JobState state; - - /** return the state for the given scheduler, or null */ - final JobState getStateFor(Scheduler sched) { - return state; - } - - /** return or create the state for the given scheduler */ - final JobState makeStateFor(Scheduler sched) { - return state == null ? (state = new JobState(this)) : state; - } - - @Override - public LogOutput append(LogOutput logOutput) { - return logOutput.append(LogOutput.BASIC_FORMATTER, this); - } -} diff --git a/IO/src/main/java/io/deephaven/io/sched/JobState.java b/IO/src/main/java/io/deephaven/io/sched/JobState.java deleted file mode 100644 index 00be8941aee..00000000000 --- a/IO/src/main/java/io/deephaven/io/sched/JobState.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.io.sched; - -import java.nio.channels.SelectableChannel; - -/** - * The per-scheduler state for a job. Note that this class is package-private. - */ -class JobState implements Cloneable { - /** the job */ - final Job job; - - /** the update count for this job state */ - long updateClock = 0; - - /** the current deadline for this job */ - long deadline = Long.MAX_VALUE; - - /** the job's current position in the scheduler's timeout queue */ - int tqPos = 0; - - /** true, if this job has been invoked or has timed out */ - boolean gathered = false; - - /** true if the job has been forgotten after being dispatched and not reinstalled */ - boolean forgotten = false; - - /** true if this job has been explicitly cancelled */ - boolean cancelled = false; - - /** this is the channel we are waiting on in the selector */ - SelectableChannel waitChannel = null; - - /** the channel on which the job is ready to be dispatched, or null */ - SelectableChannel readyChannel = null; - - /** the operation set on which the job is ready to be dispatched, or zero */ - int readyOps = 0; - - /** the channel on which this job will select in the next scheduler loop */ - SelectableChannel nextChannel = null; - - /** the interest set on which this job will select in the next scheduler loop */ - int nextOps = 0; - - /** the timeout deadline of this job in the next scheduler loop */ - long nextDeadline = Long.MAX_VALUE; - - /** the nano-time when this job was last enqueued */ - long gatheredNanos = 0; - - /** constructor stores the back-link to the job */ - JobState(Job job) { - this.job = job; - } - - /** - * Clone this object - */ - public JobState clone() throws CloneNotSupportedException { - return (JobState) super.clone(); - } -} diff --git a/IO/src/main/java/io/deephaven/io/sched/JobStateTimeoutQueue.java b/IO/src/main/java/io/deephaven/io/sched/JobStateTimeoutQueue.java deleted file mode 100644 index 9c57851c20e..00000000000 --- a/IO/src/main/java/io/deephaven/io/sched/JobStateTimeoutQueue.java +++ /dev/null @@ -1,211 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.io.sched; - -import io.deephaven.io.logger.Logger; - -import java.util.Set; - -/** - * A priority queue (heap) for JobState instances, ordered by their deadlines. Note that this class is package-private. - */ -class JobStateTimeoutQueue implements Cloneable { - private final Logger log; - - /** the queue storage */ - private JobState[] queue; - - /** the size of the queue (invariant: size < queue.length - 1) */ - private int size = 0; - - public JobStateTimeoutQueue(Logger log, int initialSize) { - this.log = log; - this.queue = new JobState[initialSize]; - } - - /** clone the queue (for testing) */ - public Object clone() throws CloneNotSupportedException { - JobStateTimeoutQueue q = (JobStateTimeoutQueue) super.clone(); - q.queue = new JobState[queue.length]; - for (int i = 1; i <= size; ++i) { - q.queue[i] = queue[i].clone(); - } - q.size = size; - return q; - } - - /** return the priority queue's size */ - int size() { - return size; - } - - /** Returns true if the priority queue contains no elements. */ - boolean isEmpty() { - return size == 0; - } - - /** Adds a job to to the timeout queue */ - void enter(JobState state, long deadline) { - state.deadline = deadline; - if (state.tqPos == 0) { - if (++size == queue.length) { - JobState[] newQueue = new JobState[2 * queue.length]; - System.arraycopy(queue, 0, newQueue, 0, size); - queue = newQueue; - } - queue[size] = state; - state.tqPos = size; - fixUp(size); - assert testInvariant("after fixUp in enter-add"); - } else { - assert queue[state.tqPos] == state; - int k = state.tqPos; - fixDown(k); - fixUp(k); - assert testInvariant("after fixDown/fixUp in enter-change"); - } - } - - /** Return the top of the timeout queue - the next timeout */ - JobState top() { - return queue[1]; - } - - /** Remove the top element from the timeout queue. */ - void removeTop() { - queue[1].tqPos = 0; - if (--size == 0) { - queue[1] = null; - } else { - queue[1] = queue[size + 1]; - queue[size + 1] = null; // Drop extra reference to prevent memory leak - queue[1].tqPos = 1; - fixDown(1); - } - assert testInvariant("after removeTop()"); - } - - /** remove an arbitrary element from the timeout queue */ - void remove(JobState state) { - int k = state.tqPos; - if (k != 0) { - assert queue[k] == state; - state.tqPos = 0; - if (k == size) { - queue[size--] = null; - } else { - queue[k] = queue[size]; - queue[k].tqPos = k; - queue[size--] = null; - fixDown(k); - fixUp(k); - assert testInvariant("after fixDown/fixUp in remove()"); - } - } - assert testInvariant("at end of remove()"); - } - - /** move queue[k] up the heap until it's deadline is >= that of its parent. */ - private void fixUp(int k) { - if (k > 1) { - JobState state = queue[k]; - int j = k >> 1; - JobState parent = queue[j]; - if (parent.deadline > state.deadline) { - queue[k] = parent; - parent.tqPos = k; - k = j; - j = k >> 1; - while (k > 1 && (parent = queue[j]).deadline > state.deadline) { - queue[k] = parent; - parent.tqPos = k; - k = j; - j = k >> 1; - } - queue[k] = state; - state.tqPos = k; - } - } - } - - /** move queue[k] down the heap until it's deadline is <= those of its children. */ - private void fixDown(int k) { - int j = k << 1; - if (j <= size) { - JobState state = queue[k], child = queue[j], child2; - if (j < size && (child2 = queue[j + 1]).deadline < child.deadline) { - child = child2; - j++; - } - if (child.deadline < state.deadline) { - queue[k] = child; - child.tqPos = k; - k = j; - j = k << 1; - while (j <= size) { - child = queue[j]; - if (j < size && (child2 = queue[j + 1]).deadline < child.deadline) { - child = child2; - j++; - } - if (child.deadline >= state.deadline) { - break; - } - queue[k] = child; - child.tqPos = k; - k = j; - j = k << 1; - } - queue[k] = state; - state.tqPos = k; - } - } - } - - boolean testInvariantAux(int i, String what) { - if (i <= size) { - if (queue[i].tqPos != i) { - log.error().append(what).append(": queue[").append(i).append("].tqPos=").append(queue[i].tqPos) - .append(" != ").append(i).endl(); - } - if (!testInvariantAux(i * 2, what)) { - return false; - } - if (!testInvariantAux(i * 2 + 1, what)) { - return false; - } - if (i > 1) { - if (queue[i].deadline < queue[i / 2].deadline) { - log.error().append(what).append(": child[").append(i).append("]=").append(queue[i].deadline) - .append(" < parent[").append((i / 2)).append("]=").append(queue[i / 2].deadline).endl(); - return false; - } - } - } - return true; - } - - boolean testInvariant(String what) { - boolean result = testInvariantAux(1, what); - if (result) { - for (int i = size + 1; i < queue.length; ++i) { - if (queue[i] != null) { - log.error().append(what).append(": size = ").append(size).append(", child[").append(i).append("]=") - .append(queue[i].deadline).append(" != null").endl(); - result = false; - } - } - } - if (result) { - // log.info("timeoutQueue.testInvariant: OK "+what); - } - return result; - } - - void junitGetAllJobs(Set jobs) { - for (int i = 1; i <= size; ++i) { - jobs.add(queue[i].job); - } - } -} diff --git a/IO/src/main/java/io/deephaven/io/sched/Scheduler.java b/IO/src/main/java/io/deephaven/io/sched/Scheduler.java deleted file mode 100644 index f38cde3c881..00000000000 --- a/IO/src/main/java/io/deephaven/io/sched/Scheduler.java +++ /dev/null @@ -1,199 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.io.sched; - -import java.nio.channels.*; -import java.util.*; -import java.util.concurrent.Executor; - -/** - * This class provides a singleton wrapper for scheduling invocations of multiple Job instances from a single thread. - * Job are scheduled in accordance with an interest set on a java.nio.Channel, deadline based time scheduling, and/or - * custom criteria defined by the Jobs' implementation of the ready() method. - * - * Jobs are instantiated by the application and made known to the scheduler by one of the install() methods. Once the - * job is installed, the scheduler will call exactly one of its invoke(), timedOut() or cancelled() methods exactly - * once. After this, the scheduler forgets about the job completely, unless the application installs it again. - */ -public interface Scheduler { - - // -------------------------------------------------------------------------- - // public interface - // -------------------------------------------------------------------------- - - /** - * Return the scheduler's idea of the current time. - */ - public long currentTimeMillis(); - - /** - * Install a job in association with a channel and an interest set. - */ - public void installJob(Job job, long deadline, SelectableChannel channel, int interest); - - /** - * Install a job with only an associated deadline (removing any channel association) - */ - public void installJob(Job job, long deadline); - - /** - * Cancel a job, making the scheduler forget it completely.. - */ - public void cancelJob(Job job); - - /** - * Wait for jobs to become ready, then invoke() them all. This method will form the core of the main loop of a - * scheduler-driven application. The method first waits until: - * - * -- the given timeout expires, -- the earliest job-specific timeout expires, or -- one or more jobs becomes ready - * - * If jobs have become ready, then the entire ready set will be invoked. If any job throws an uncaught exception, - * the job's terminated() method will be called and the job deregistered. This does not abort the invocation of the - * remaining jobs. The return value is then the number of jobs that were invoked. - * - * If no jobs are ready and any job-specific timeouts expire, the associated jobs' timedOut() methods are called. - * The return value is the negative of the number of expired timeouts. - * - * If the time given by the timeout argument expires, then zero is returned. - * - * Note that this method is not synchronized. The application must ensure that it is never called concurrently by - * more than one thread. - * - * @return true, if some job was dispatched - */ - public boolean work(long timeout, Runnable handoff); - - /** - * Shut down the scheduler, calling close() on the underlying Selector. - */ - public void close(); - - /** - * Return true if the scheduler is closing or closed. - */ - public boolean isClosed(); - - // -------------------------------------------------------------------------- - // test support methods - // -------------------------------------------------------------------------- - - /** - * Return a reference to the selector - */ - public Selector junitGetSelector(); - - /** - * Return all jobs known to the scheduler, in whatever state. - */ - public Set junitGetAllJobs(); - - /** - * Return the contents of the timeout queue, in deadline order - * - * @return the jobs in the timeout queue - */ - public ArrayList junitGetTimeoutQueue(); - - /** - * Return the selection keys currently known to the scheduler. - */ - public ArrayList junitGetAllKeys(); - - /** - * Return the selection keys currently known to the scheduler. - */ - public ArrayList junitGetReadyKeys(); - - /** - * Return a map containing all channels and the jobs to which they are associated. - */ - public Map junitGetChannelsAndJobs(); - - /** - * Return true if the timeout queue invariant holds. - */ - public boolean junitTestTimeoutQueueInvariant(); - - public class Null implements Scheduler { - @Override - public long currentTimeMillis() { - return 0; - } - - @Override - public void installJob(Job job, long deadline, SelectableChannel channel, int interest) {} - - @Override - public void installJob(Job job, long deadline) {} - - @Override - public void cancelJob(Job job) {} - - @Override - public boolean work(long timeout, Runnable handoff) { - return false; - } - - @Override - public void close() {} - - @Override - public boolean isClosed() { - return false; - } - - @Override - public Selector junitGetSelector() { - return null; - } - - @Override - public Set junitGetAllJobs() { - return null; - } - - @Override - public ArrayList junitGetTimeoutQueue() { - return null; - } - - @Override - public ArrayList junitGetAllKeys() { - return null; - } - - @Override - public ArrayList junitGetReadyKeys() { - return null; - } - - @Override - public Map junitGetChannelsAndJobs() { - return null; - } - - @Override - public boolean junitTestTimeoutQueueInvariant() { - return false; - } - } - - public final class ExecutorAdaptor implements Executor { - final Scheduler scheduler; - - public ExecutorAdaptor(final Scheduler scheduler) { - this.scheduler = scheduler; - } - - @Override - public void execute(final Runnable runnable) { - scheduler.installJob(new TimedJob() { - @Override - public final void timedOut() { - runnable.run(); - } - }, 0); - } - } -} diff --git a/IO/src/main/java/io/deephaven/io/sched/TimedJob.java b/IO/src/main/java/io/deephaven/io/sched/TimedJob.java deleted file mode 100644 index 9fb8c2899fb..00000000000 --- a/IO/src/main/java/io/deephaven/io/sched/TimedJob.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.io.sched; - -import io.deephaven.base.log.LogOutput; - -import java.nio.channels.SelectableChannel; - -/** - * This is the base class for jobs which are only interested in timing events. It provides default invoke() and - * cancelled() method which do nothing. - */ -public abstract class TimedJob extends Job { - public int invoke(SelectableChannel channel, int readyOps, Runnable handoff) { - if (handoff != null) { - handoff.run(); - } - return 0; - } - - public void cancelled() { - // do nothing - } - - @Override - public LogOutput append(LogOutput logOutput) { - return logOutput.append(LogOutput.BASIC_FORMATTER, this); - } -} diff --git a/IO/src/main/java/io/deephaven/io/sched/YASchedulerImpl.java b/IO/src/main/java/io/deephaven/io/sched/YASchedulerImpl.java deleted file mode 100644 index f10c7fc70b8..00000000000 --- a/IO/src/main/java/io/deephaven/io/sched/YASchedulerImpl.java +++ /dev/null @@ -1,979 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.io.sched; - -import io.deephaven.base.RingBuffer; -import io.deephaven.base.stats.*; -import io.deephaven.io.logger.Logger; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.channels.*; -import java.util.*; - -/** - * Yet Another implementation of the Scheduler interface -- the best one yet. - * - * This class provides a singleton wrapper for scheduling invocations of multiple Job instances from a single thread. - * Job are scheduled in accordance with an interest set on a java.nio.Channel, deadline based time scheduling, and/or - * custom criteria defined by the Jobs' implementation of the ready() method. - * - * Jobs are instantiated by the application and made known to the scheduler by one of the installJob() methods. A - * previously installed job can be removed from the scheduler with the cancelJob() method. The installJob() and - * cancelJob() methods are thread-safe. It is allowed to call installJob() on a job that is already installed, or - * cancelJob() on a job that is not current in the scheduler. In the former case, the channel and/or deadline will be - * updated accordingly; in the latter, the call will be ignored. - * - * Once the job is installed, the scheduler promises to call exactly one of its invoke(), timedOut() or cancelled() - * methods exactly once. The invoke() method will be called only if the job was (last) installed with a channel and - * non-zero interest set. The timedOut() method can be called for any job, since all jobs have an associated deadline - * (although the timeout value can be set to Integer.MAX_VALUE to make if effectively infinite). The cancelled() method - * is called only if the job is removed by a cancelJob() call before either the channe is ready or the deadline expires. - * - * After the job is called back, the scheduler forgets about the job completely, unless the application installs it - * again. That is, from the scheduler's point of view *all* jobs are one-shots. This design is based on the observation - * that it is easier to reschedule jobs on every invocation in the style of a tail-recursive loop, as opposed to - * maintaining persistent state in the scheduler. - * - * The application must drive the scheduler by calling the work() method in a loop. The work() method is *not* - * thread-safe; the application must either call it from a single thread or synchronize calls accordingly. - */ -public class YASchedulerImpl implements Scheduler { - - /** the scheduler name, for debug and stats output */ - protected final String name; - - /** the java.nio.Selector instance */ - private final Selector selector; - - /** the logger */ - protected final Logger log; - - /** lock for internal state */ - private final Object stateLock = new Object(); - - /** if non-zero, there is a select() in progress that will terminate at the specified deadline */ - private long selectingTill = 0; - - private volatile boolean spinWakeSelector = false; - - /** the update clock for this scheduler */ - private long updateClock = 1; - - /** the waiting jobs, ordered by deadline */ - private final JobStateTimeoutQueue timeoutQueue; - - /** invokable/timed-out jobs are stored here */ - private RingBuffer dispatchQueue = new RingBuffer(128); - - /** the list of jobs which might have changed since the last update() call */ - private ArrayList changedStates = new ArrayList(128); - - /** add a state to the changedStates list */ - private boolean changedState(JobState state) { - if (state.updateClock < updateClock) { - state.updateClock = updateClock; - changedStates.add(state); - return true; - } - - // Assert.eqTrue(isInChangedStates(state), "isInChangedStates(state)"); // temporary - - return false; - } - - private boolean isInChangedStates(JobState state) { - final int L = changedStates.size(); - for (int i = 0; i < L; ++i) { - if (state == changedStates.get(i)) { - return true; - } - } - return false; - } - - /** if there are lots of tiny jobs, taking timing measurements may be time consuming. */ - private final boolean doTimingStats; - - private final boolean doSpinSelect; - - /** time base for loop duration measurements */ - private long lastNanos = 0; - - private void mark(Value v) { - if (doTimingStats) { - long t = System.nanoTime(); - if (lastNanos != 0) { - v.sample((t - lastNanos + 500) / 1000); - } - lastNanos = t; - } - } - - /** have we been closed? */ - private volatile boolean isClosed = false; - - // statistics - private Value invokeCount; - private Value timeoutCount; - private Value selectDuration; - private Value workDuration; - private Value gatheredDuration; - private Value channelInstalls; - private Value timedInstalls; - private Value jobCancels; - private Value jobUpdates; - private Value keyUpdates; - private Value keyOrphans; - private Value selectorWakeups; - private Value channelInterestWakeups; - private Value channelTimeoutWakeups; - private Value plainTimeoutWakeups; - private Value cancelWakeups; - - /** - * The constructor. - */ - public YASchedulerImpl(Selector selector, Logger log) throws IOException { - this("Scheduler", selector, log); - } - - public YASchedulerImpl(String name, Selector selector, Logger log) throws IOException { - this(name, selector, log, true, false); - } - - public YASchedulerImpl(String name, Selector selector, Logger log, boolean doTimingStats, boolean doSpinSelect) { - this.name = name; - this.selector = selector; - this.log = log; - this.doTimingStats = doTimingStats; - this.doSpinSelect = doSpinSelect; - - this.timeoutQueue = new JobStateTimeoutQueue(log, 1024); - - this.invokeCount = Stats.makeItem(name, "invokeCount", Counter.FACTORY, - "The number of jobs invoked for I/O").getValue(); - this.timeoutCount = Stats.makeItem(name, "timeoutCount", Counter.FACTORY, - "The number of jobs that have timed out").getValue(); - this.selectDuration = Stats.makeItem(name, "SelectDuration", State.FACTORY, - "The number of microseconds spent in select()").getValue(); - this.workDuration = Stats.makeItem(name, "WorkDuration", State.FACTORY, - "The number of microseconds between successive select() calls").getValue(); - this.gatheredDuration = Stats.makeItem(name, "GatheredDuration", State.FACTORY, - "The number of microseconds jobs spend waiting after being gathered").getValue(); - this.channelInstalls = Stats.makeItem(name, "channelInstalls", Counter.FACTORY, - "The number of installJob() calls with a channel").getValue(); - this.timedInstalls = Stats.makeItem(name, "timedInstalls", Counter.FACTORY, - "The number of installJob() calls with just a timeout").getValue(); - this.jobCancels = Stats.makeItem(name, "jobCancels", Counter.FACTORY, - "The number of cancelJob() calls").getValue(); - this.jobUpdates = Stats.makeItem(name, "jobUpdates", Counter.FACTORY, - "The number of updates applied to the job state pre- and post-select").getValue(); - this.keyUpdates = Stats.makeItem(name, "keyUpdates", Counter.FACTORY, - "The number of times an NIO SelectionKey was updated with non-zero interest").getValue(); - this.keyOrphans = Stats.makeItem(name, "keyOrphans", Counter.FACTORY, - "The number of times an NIO SelectionKey's interest was cleared").getValue(); - this.selectorWakeups = Stats.makeItem(name, "selectorWakeups", Counter.FACTORY, - "The number of times the selector had to be woken up").getValue(); - - this.channelInterestWakeups = Stats.makeItem(name, "channelInterestWakeups", Counter.FACTORY, - "The number of selector wakeups due to a change in a channel's interest set").getValue(); - this.channelTimeoutWakeups = Stats.makeItem(name, "channelTimeoutWakeups", Counter.FACTORY, - "The number of selector wakeups due to a channel's timeout becoming the earliest").getValue(); - this.plainTimeoutWakeups = Stats.makeItem(name, "plainTimeoutWakeups", Counter.FACTORY, - "The number of selector wakeups due to a plain timeout becoming the earliest").getValue(); - this.cancelWakeups = Stats.makeItem(name, "cancelWakeups", Counter.FACTORY, - "The number of selector wakeups due to a job cancellation").getValue(); - } - - /** - * Return the scheduler's idea of the current time. - */ - public long currentTimeMillis() { - return System.currentTimeMillis(); - } - - /** - * Install a job in association with a channel and an interest set. - */ - public void installJob(Job job, long deadline, SelectableChannel channel, int interest) { - synchronized (stateLock) { - JobState state = job.makeStateFor(this); - SelectionKey key = channel.keyFor(selector); - - // see if we will need to wake up the selector - boolean wakeup = false; - if (key == null || !key.isValid()) { - wakeup = true; - } else if (deadline < selectingTill) { - wakeup = true; - channelTimeoutWakeups.sample(1); - } else if (key.interestOps() != interest && (channel != state.nextChannel || interest != state.nextOps)) { - wakeup = true; - channelInterestWakeups.sample(1); - } - - state.nextChannel = channel; - state.nextOps = interest; - state.nextDeadline = deadline; - state.cancelled = false; - state.forgotten = false; - changedState(state); - - if (log.isDebugEnabled()) { - log.debug().append(name).append(" installed job ").append(job) - .append(", d=").append(deadline) - .append(", ni=").append(state.nextOps) - // .append(", k=").append(key) - .append(", ki=").append((key == null || !key.isValid() ? 0 : key.interestOps())) - .append(", w=").append(wakeup) - .endl(); - } - - if (wakeup) { - maybeWakeSelector(); - } - - // must always wake if doing spin select since we aren't setting selectingTill - else if (doSpinSelect) { - spinWakeSelector = true; - } - - channelInstalls.sample(1); - } - } - - /** - * Install a job with only an associated deadline (removing any channel association) - */ - public void installJob(Job job, long deadline) { - synchronized (stateLock) { - JobState state = job.makeStateFor(this); - state.nextChannel = null; - state.nextOps = 0; - state.nextDeadline = deadline; - state.cancelled = false; - state.forgotten = false; - final boolean changed = changedState(state); - - // Note: We don't need to be concerned with waking up due to channelInterest changes, since - // we would have to be reducing the interest set which can only lead to a later wakeup time. - - // if the new deadline is earlier than the current top, wake up the selector - boolean wakeup = false; - if (deadline < selectingTill) { - plainTimeoutWakeups.sample(1); - maybeWakeSelector(); - } - - // must always wake if doing spin select since we aren't setting selectingTill - else if (doSpinSelect) { - spinWakeSelector = true; - } - - if (log.isDebugEnabled()) { - log.debug().append(name).append(" installed job ").append(job) - .append(", d=").append(deadline) - .append(", w=").append(wakeup) - .append(", c=").append(changed) - .endl(); - } - - timedInstalls.sample(1); - } - } - - /** - * Cancel a job's selection key with the scheduler. - * - * @param job the job to be cancelled. - */ - public void cancelJob(Job job) { - synchronized (stateLock) { - if (log.isDebugEnabled()) { - log.debug().append(name).append(" explicitly cancelling ").append(job) - .append(" in YAScheduler.cancelJob").endl(); - } - JobState state = job.getStateFor(this); - if (state != null) { - state.nextChannel = null; - state.nextOps = 0; - state.nextDeadline = 0; - state.cancelled = true; - state.forgotten = false; - changedState(state); - - if (state.waitChannel != null) { - cancelWakeups.sample(1); - maybeWakeSelector(); - } - jobCancels.sample(1); - } - } - } - - /** - * drop the association of a state with a channel - */ - private void dropChannel(JobState state) { - if (state.waitChannel != null) { - SelectionKey key = state.waitChannel.keyFor(selector); - try { - if (key != null && key.isValid() && key.attachment() == state) { - key.attach(null); - if (key.interestOps() != 0) { - key.interestOps(0); - if (log.isDebugEnabled()) { - log.debug().append(name).append(" setting interest on orphaned key ").append(key.toString()) - .append(" to 0").endl(); - } - keyUpdates.sample(1); - } - } - } catch (CancelledKeyException x) { - // ignore it - if (log.isDebugEnabled()) { - log.info().append(name).append(" got CancelledKeyException while dropping channel ") - .append(state.waitChannel.toString()).endl(); - } - } - state.waitChannel = null; - } - } - - /** - * associate a channel with a state - */ - private boolean grabChannel(JobState state) { - try { - SelectionKey key = state.nextChannel.keyFor(selector); - if (key == null) { - key = state.nextChannel.register(selector, state.nextOps, state); - log.debug().append(name).append(" update ").append(state.job) - .append(": registered channel ").append(state.nextChannel.toString()) - .append(", ni=").append(state.nextOps) - .append(", k=").append(key.toString()) - .endl(); - } else { - key.attach(state); - if (key.interestOps() != state.nextOps) { - if (log.isDebugEnabled()) { - log.debug().append(name).append(" update ").append(state.job) - .append(": setting interest on key ").append(key.toString()).append(" to ") - .append(state.nextOps) - .endl(); - } - key.interestOps(state.nextOps); - keyUpdates.sample(1); - } else { - if (log.isDebugEnabled()) { - log.debug().append(name).append(" update ").append(state.job) - .append(": interest on key ").append(key.toString()).append(" already at ") - .append(state.nextOps) - .endl(); - } - } - } - if (state.waitChannel != null && state.waitChannel != state.nextChannel) { - SelectionKey waitKey = state.waitChannel.keyFor(selector); - if (waitKey != null && waitKey.attachment() == state) { - try { - waitKey.interestOps(0); - } catch (CancelledKeyException x) { - // ignore this - } - } - } - state.waitChannel = state.nextChannel; - return true; - } catch (ClosedChannelException x) { - // fall through - } catch (CancelledKeyException x) { - // fall through - } - state.waitChannel = null; - log.error().append(name).append(" tried to register ").append(state.job).append(" on closed channel ") - .append(state.nextChannel.toString()).endl(); - return false; - } - - /** - * Apply changes to the job states. - * - * NOTE: assumes that stateLock is held - */ - private void update() { - // DO NOT USE FOREACH HERE AS IT CREATES AN INTERATOR -> No Allocation changes - int size = changedStates.size(); - for (int i = 0; i < size; i++) { - JobState state = changedStates.get(i); - jobUpdates.sample(1); - - if (log.isDebugEnabled()) { - SelectionKey key = null; - if (state.nextChannel != null) { - key = state.nextChannel.keyFor(selector); - } - log.debug().append(name).append(" updating job ").append(state.job) - .append(", d=").append(state.nextDeadline) - .append(", ni=").append(state.nextOps) - .append(", k=").append(key == null ? "null" : key.toString()) - .append(", ki=").append(key == null || !key.isValid() ? 0 : key.interestOps()) - .endl(); - } - - if (state.gathered) { - // job is waiting to be invoked; leave it alone - } else if (state.nextChannel != null && state.nextOps != 0) { - if (!grabChannel(state)) { - log.error().append(name).append(" cancelling ").append(state.job) - .append(" after failed I/O registration").endl(); - timeoutQueue.remove(state); - state.cancelled = true; - dispatchQueue.add(state); - } else { - timeoutQueue.enter(state, state.nextDeadline); - } - } else if (state.forgotten) { - dropChannel(state); - timeoutQueue.remove(state); - } else if (state.cancelled) { - dropChannel(state); - timeoutQueue.remove(state); - if (log.isDebugEnabled()) { - log.debug().append(name).append(" cancelling ").append(state.job).append(" from update()").endl(); - } - state.cancelled = true; - dispatchQueue.add(state); - } else { - dropChannel(state); - timeoutQueue.enter(state, state.nextDeadline); - } - - state.forgotten = true; - state.nextChannel = null; - state.nextOps = 0; - state.nextDeadline = 0; - - assert state.waitChannel == null || state.waitChannel.keyFor(selector).attachment() == state; - } - if (log.isDebugEnabled()) { - log.debug().append(name).append(" updated ").append(changedStates.size()).append(" jobs").endl(); - } - changedStates.clear(); - updateClock++; - } - - /** - * compute the timeout value for the next select() call - * - * NOTE: assumes that stateLock is held - */ - private long computeTimeout(long now, long timeout) { - if (!dispatchQueue.isEmpty()) { - if (log.isDebugEnabled()) { - log.debug().append(name).append(" update: dispatch queue is not empty, setting timeout to zero").endl(); - } - timeout = 0; - } else if (!timeoutQueue.isEmpty()) { - JobState next = timeoutQueue.top(); - long remain = next.deadline - now; - if (log.isDebugEnabled()) { - log.debug().append(name).append(" update: next timeout due in ").append(remain).append(" millis: ") - .append(next.job).endl(); - } - timeout = Math.max(0, Math.min(timeout, remain)); - } - return timeout; - } - - /** - * Wait for something to happen - */ - private void select(long timeout) { - try { - if (log.isDebugEnabled()) { - log.debug().append(name).append(" calling select(").append(timeout).append(")").endl(); - } - - mark(workDuration); - - if (timeout > 0) { - selector.select(timeout); - } else { - selector.selectNow(); - } - - mark(selectDuration); - } catch (IOException x) { - if (java.util.regex.Pattern.matches(".*Operation not permitted.*", x.toString())) { - // There is a documented bug (http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6481709) in some - // versions of the epoll selector which causes occasional "Operation not permitted" errors to be - // thrown. - log.warn().append(name).append( - " Ignoring 'Operation not permitted' exception, see http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6481709") - .endl(); - } else { - if (!isClosed()) { - log.fatal(x).append(name).append(" Unexpected IOException in select(): ").append(x.getMessage()) - .endl(); - System.exit(1); - } - } - } catch (ClosedSelectorException x) { - if (!isClosed()) { - log.fatal(x).append(name).append(" ClosedSelectorException in select(): ").append(x.getMessage()) - .endl(); - System.exit(1); - } - } - } - - private void spinSelect(long times) { - try { - if (log.isDebugEnabled()) { - log.debug().append(name).append(" calling spinSelect(").append(times).append(")").endl(); - } - - mark(workDuration); - - while (selector.selectNow() == 0 && !spinWakeSelector && (times-- > 0)) { - } - - mark(selectDuration); - } catch (IOException x) { - if (java.util.regex.Pattern.matches(".*Operation not permitted.*", x.toString())) { - // There is a documented bug (http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6481709) in some - // versions of the epoll selector which causes occasional "Operation not permitted" errors to be - // thrown. - log.warn().append(name).append( - " Ignoring 'Operation not permitted' exception, see http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6481709") - .endl(); - } else { - if (!isClosed()) { - log.fatal(x).append(name).append(" Unexpected IOException in spinSelect(): ").append(x.getMessage()) - .endl(); - System.exit(1); - } - } - } catch (ClosedSelectorException x) { - if (!isClosed()) { - log.fatal(x).append(name).append(" ClosedSelectorException in spinSelect(): ").append(x.getMessage()) - .endl(); - System.exit(1); - } - } - } - - /** - * Gather up selected and timed-out jobs - * - * NOTE: assumes that stateLock is held - */ - private void gather(long now) { - JobState state; - int numInvokes = 0; - // first gather all of the invokable jobs - for (SelectionKey key : selector.selectedKeys()) { - ++numInvokes; - try { - if ((state = (JobState) key.attachment()) == null) { - // clear interest ops, so we don't select in a tight loop - if (key.isValid() && key.interestOps() != 0) { - if (log.isDebugEnabled()) { - log.debug().append(name).append(" clearing interest in orphaned key ") - .append(key.toString()).append(" in YASchedulerImpl.gather").endl(); - } - if (key.isValid()) { - key.interestOps(0); - } - keyOrphans.sample(1); - } - } else { - key.attach(null); - state.readyChannel = key.channel(); - state.readyOps = key.readyOps(); - state.gathered = true; - state.gatheredNanos = lastNanos; - dispatchQueue.add(state); - timeoutQueue.remove(state); - if (log.isDebugEnabled()) { - log.debug().append(name).append(" gather ").append(key.toString()).append(" -> ") - .append(state.job) - .append(", ops=").append(key.readyOps()) - .append(", ki=").append(key.interestOps()) - .append(", dq=").append(dispatchQueue.size()) - .endl(); - } - } - } catch (CancelledKeyException x) { - // We can't guarantee that some thread won't try to write to the channel and - // cause it to cancel the key -- if that happens, then we'll get the exception - // here. But that's okay, because it's either an orphan channel which we just - // want to get rid of, or the IOJob will get the exception later and handle it. - } - } - selector.selectedKeys().clear(); - invokeCount.sample(numInvokes); - - // now get all of the expired timeouts - int numTimeouts = 0; - while (!timeoutQueue.isEmpty() && now >= (state = timeoutQueue.top()).deadline) { - ++numTimeouts; - timeoutQueue.removeTop(); - state.gathered = true; - state.gatheredNanos = lastNanos; - dispatchQueue.add(state); - } - timeoutCount.sample(numTimeouts); - - if (log.isDebugEnabled()) { - log.debug().append(name).append(" gathered ").append(numInvokes).append(" for I/O and ").append(numTimeouts) - .append(" timeouts").endl(); - } - } - - /** - * dispatch a gathered job, if there are any - */ - private boolean dispatch(Runnable handoff) { - JobState state; - SelectableChannel readyChannel; - int readyOps; - boolean cancelled; - synchronized (stateLock) { - if ((state = dispatchQueue.poll()) == null) { - return false; - } - - readyChannel = state.readyChannel; - readyOps = state.readyOps; - cancelled = state.cancelled; - state.readyChannel = null; - state.readyOps = 0; - state.gathered = false; - // NOTE: we only need to record the state as changed if it has a channel; - // cancelled and timed-out states will just be forgotten. - if (!cancelled && readyChannel != null) { - changedState(state); - } - if (log.isDebugEnabled()) { - log.debug().append(name).append(" dispatch ").append(state.job) - .append(", ops=").append(readyOps) - .append(", dq=").append(dispatchQueue.size()) - .endl(); - } - assert readyChannel == null || readyOps != 0; - } - - // dispatch the job outside of the state lock - try { - if (cancelled) { - if (log.isDebugEnabled()) { - log.debug().append(name).append(" cancelled ").append(state.job).endl(); - } - state.job.cancelled(); - } else { - if (doTimingStats) - gatheredDuration.sample((System.nanoTime() - state.gatheredNanos + 500) / 1000); - if (readyOps != 0) { - if (log.isDebugEnabled()) { - log.debug().append(name).append(" invoke ").append(state.job).endl(); - } - state.job.invoke(readyChannel, readyOps, handoff); - } else { - if (log.isDebugEnabled()) { - log.debug().append(name).append(" timedOut ").append(state.job).endl(); - } - if (handoff != null) { - handoff.run(); - } - state.job.timedOut(); - } - } - } catch (Throwable x) { - log.fatal(x).append(": unhandled Throwable in dispatch on job [").append(state.job).append("]: ") - .append(x.getMessage()).endl(); - throw new RuntimeException(x); - } - - return true; - } - - /** - * Wake up the selector, if necessary. - * - * NOTE: assumes that stateLock is held! - */ - private void maybeWakeSelector() { - if (selectingTill > 0) { - if (log.isDebugEnabled()) { - log.debug().append(name).append(" waking up the scheduler").endl(); - } - selector.wakeup(); - selectorWakeups.sample(1); - } - - if (doSpinSelect) { - spinWakeSelector = true; - } - } - - /** - * Wait for jobs to become ready, then invoke() them all. This method will form the core of the main loop of a - * scheduler-driven application. The method first waits until: - * - * -- the given timeout expires, -- the earliest job-specific timeout expires, or -- one or more jobs becomes ready - * - * Note that this method is not synchronized. The application must ensure that it is never called concurrently by - * more than one thread. - * - * @return true, if some work was done. - */ - public boolean work(long timeout, Runnable handoff) { - if (doSpinSelect) { - // just use the millis timeout as the number of times to spin - long times = timeout; - return spinWork(times, handoff); - } - - boolean didOne = dispatch(handoff); - if (!didOne) { - // apply any changes to the states - synchronized (stateLock) { - update(); - long now = currentTimeMillis(); - timeout = computeTimeout(now, timeout); - assert selectingTill == 0 : "no more than one thread should ever call work!"; - if (timeout > 0) { - selectingTill = now + timeout; - } - } - - // wait for something to happen - select(timeout); - - // apply changes while we were waiting, then gather up all of the jobs that can be dispatched - synchronized (stateLock) { - selectingTill = 0; - update(); - long now = currentTimeMillis(); - gather(now); - } - - // and try again - didOne = dispatch(handoff); - } - return didOne; - } - - private boolean spinWork(long times, Runnable handoff) { - boolean didOne = dispatch(handoff); - if (!didOne) { - // apply any changes to the states - synchronized (stateLock) { - update(); - if (!dispatchQueue.isEmpty() || spinWakeSelector) { - times = 1; // only want to spin on select once since we have stuff to dispatch - spinWakeSelector = false; - } - assert selectingTill == 0 : "no more than one thread should ever call work!"; - } - - // spin for something to happen - spinSelect(times); - - // apply changes while we were waiting, then gather up all of the jobs that can be dispatched - synchronized (stateLock) { - selectingTill = 0; - update(); - long now = currentTimeMillis(); - gather(now); - } - - // and try again - didOne = dispatch(handoff); - } - return didOne; - } - - /** - * Shuts down the scheduler, calling close() on the underlying Selector instance. - */ - public void close() { - isClosed = true; - clear(); - try { - selector.close(); - } catch (IOException x) { - log.warn(x).append(name).append(" Scheduler.close: ignoring exception from selector.close(): ") - .append(x.getMessage()).endl(); - } - } - - /** - * Return true if the scheduler is closed, or in the process of closing. - */ - public boolean isClosed() { - return isClosed; - } - - /** - * Clear out the scheduler state - */ - private void clear() { - Set allJobs = getAllJobs(); - for (Job j : allJobs) { - cancelJob(j); - } - log.info().append(name).append(" Scheduler.clear: starting with ").append(allJobs.size()).append(" jobs") - .endl(); - synchronized (stateLock) { - update(); - } - ArrayList allKeys = getAllKeys(); - for (SelectionKey k : allKeys) { - k.cancel(); - } - synchronized (stateLock) { - update(); - } - try { - selector.selectNow(); - } catch (IOException x) { - throw new UncheckedIOException(x); - } - while (true) { - try { - if (!dispatch(null)) { - break; - } - } catch (Exception x) { - log.warn().append(name).append(" Scheduler.clear: ignoring shutdown exception: ").append(x).endl(); - } - } - log.info().append(name).append(" Scheduler.clear: finished").endl(); - } - - /** - * return the set of all jobs known to the scheduler, in whatever state - */ - private Set getAllJobs() { - synchronized (stateLock) { - update(); - Set result = new HashSet(); - timeoutQueue.junitGetAllJobs(result); - for (JobState state : changedStates) { - assert state != null; - if (state.job != null) { - result.add(state.job); - } - } - for (SelectionKey key : junitGetAllKeys()) { - Object attachment; - if (key != null && (attachment = key.attachment()) != null && attachment instanceof JobState) { - JobState state = (JobState) attachment; - if (state.job != null) { - result.add(state.job); - } - } - } - return result; - } - } - - /** - * Return the selection keys currently known to the scheduler. - */ - private ArrayList getAllKeys() { - synchronized (stateLock) { - update(); - Set keys = selector.keys(); - selector.wakeup(); - synchronized (keys) { - return new ArrayList(keys); - } - } - } - - // -------------------------------------------------------------------------- - // test support methods (white-box) - // -------------------------------------------------------------------------- - - public Selector junitGetSelector() { - return selector; - } - - /** - * return the set of all jobs known to the scheduler, in whatever state - */ - public Set junitGetAllJobs() { - return getAllJobs(); - } - - /** - * Return the contents of the timeout queue, in deadline order - * - * @return the jobs in the timeout queue - */ - public ArrayList junitGetTimeoutQueue() { - synchronized (stateLock) { - update(); - ArrayList result = new ArrayList(timeoutQueue.size()); - try { - JobStateTimeoutQueue q = (JobStateTimeoutQueue) timeoutQueue.clone(); - while (!q.isEmpty()) { - result.add(q.top().job); - q.removeTop(); - } - } catch (CloneNotSupportedException x) { - // ignore - } - return result; - } - } - - /** - * Return the selection keys currently known to the scheduler. - */ - public ArrayList junitGetAllKeys() { - return getAllKeys(); - } - - /** - * Return the selection keys currently known to the scheduler. - */ - public ArrayList junitGetReadyKeys() { - return new ArrayList(selector.selectedKeys()); - } - - /** - * Return a map containing all channels and the jobs to which they are associated. - */ - public Map junitGetChannelsAndJobs() { - synchronized (stateLock) { - update(); - Map result = new HashMap(); - for (SelectionKey key : junitGetAllKeys()) { - Object attachment; - if (key != null && (attachment = key.attachment()) != null && attachment instanceof JobState) { - JobState state = (JobState) attachment; - if (state.job != null) { - result.put(key.channel(), ((JobState) attachment).job); - } - } - } - return result; - } - } - - /** - * Return true if the timeout queue invariant holds. - */ - public boolean junitTestTimeoutQueueInvariant() { - synchronized (stateLock) { - return timeoutQueue.testInvariant("in call from junit"); - } - } -} diff --git a/IO/src/test/java/io/deephaven/io/sched/TestJobStateTimeoutQueue.java b/IO/src/test/java/io/deephaven/io/sched/TestJobStateTimeoutQueue.java deleted file mode 100644 index a572d425d89..00000000000 --- a/IO/src/test/java/io/deephaven/io/sched/TestJobStateTimeoutQueue.java +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.io.sched; - -import io.deephaven.io.logger.Logger; -import junit.framework.TestCase; - -import java.nio.channels.SelectableChannel; -import java.io.IOException; - -public class TestJobStateTimeoutQueue extends TestCase { - - public void setUp() throws Exception { - super.setUp(); - } - - public void tearDown() throws Exception { - super.tearDown(); - } - - /** - * A null Job implementation - */ - private static class NullJob extends Job { - public int invoke(SelectableChannel channel, int readyOps, Runnable handoff) throws IOException { - return 0; - } - - public void timedOut() {} - - public void cancelled() {} - } - - /** - * Macro test - */ - public void testTimeoutQueue() { - JobState[] ja = new JobState[10]; - for (int i = 0; i < ja.length; ++i) { - ja[i] = new JobState(new NullJob()); - } - JobStateTimeoutQueue q = new JobStateTimeoutQueue(Logger.NULL, 10); - - q.enter(ja[0], 1); - assertTrue(q.testInvariant("insert 1")); - q.enter(ja[1], 9); - assertTrue(q.testInvariant("insert 9")); - q.enter(ja[2], 8); - assertTrue(q.testInvariant("insert 8")); - q.enter(ja[3], 5); - assertTrue(q.testInvariant("insert 5")); - q.enter(ja[4], 2); - assertTrue(q.testInvariant("insert 2")); - q.enter(ja[5], 3); - assertTrue(q.testInvariant("insert 3")); - q.enter(ja[6], 6); - assertTrue(q.testInvariant("insert 6")); - q.enter(ja[7], 4); - assertTrue(q.testInvariant("insert 4")); - q.enter(ja[8], 7); - assertTrue(q.testInvariant("insert 7")); - q.enter(ja[9], 10); - assertTrue(q.testInvariant("insert 10")); - - assertEquals(ja[0], q.top()); - q.removeTop(); - q.testInvariant("remove 1"); - assertEquals(ja[4], q.top()); - q.removeTop(); - q.testInvariant("remove 2"); - assertEquals(ja[5], q.top()); - q.removeTop(); - q.testInvariant("remove 3"); - assertEquals(ja[7], q.top()); - q.removeTop(); - q.testInvariant("remove 4"); - assertEquals(ja[3], q.top()); - q.removeTop(); - q.testInvariant("remove 5"); - assertEquals(ja[6], q.top()); - q.removeTop(); - q.testInvariant("remove 6"); - assertEquals(ja[8], q.top()); - q.removeTop(); - q.testInvariant("remove 7"); - assertEquals(ja[2], q.top()); - q.removeTop(); - q.testInvariant("remove 8"); - assertEquals(ja[1], q.top()); - q.removeTop(); - q.testInvariant("remove 9"); - assertEquals(ja[9], q.top()); - q.removeTop(); - q.testInvariant("remove 10"); - - assertTrue(q.testInvariant("after clone")); - } - - /** - * Test change of deadline within queue - */ - public void testDeadlineChange() { - JobState j1 = new JobState(new NullJob()); - JobState j2 = new JobState(new NullJob()); - JobState j3 = new JobState(new NullJob()); - JobStateTimeoutQueue q = new JobStateTimeoutQueue(Logger.NULL, 10); - - q.enter(j1, 1000); - q.enter(j2, 2000); - q.enter(j3, 3000); - - assertEquals(j1, q.top()); - - q.enter(j2, 200); - assertEquals(j2, q.top()); - - q.enter(j2, 20000); - assertEquals(j1, q.top()); - - q.enter(j1, 100000); - assertEquals(j3, q.top()); - } -} diff --git a/Net/build.gradle b/Net/build.gradle deleted file mode 100644 index 31bf972685b..00000000000 --- a/Net/build.gradle +++ /dev/null @@ -1,30 +0,0 @@ -plugins { - id 'io.deephaven.project.register' -} - -dependencies { - implementation project(':Base') - implementation project(':DataStructures') - implementation project(':IO') - implementation project(':Configuration') - implementation project(':FishUtil') - implementation project(':log-factory') - - testImplementation project(path: ':Base', configuration: 'tests') - - testRuntimeOnly project(':log-to-slf4j') - Classpaths.inheritSlf4j(project, 'slf4j-simple', 'testRuntimeOnly') -} - -test { - useJUnit() - - enableAssertions = true - maxHeapSize = '3g' - - systemProperty 'Configuration.rootFile', 'lib-tests.prop' - systemProperty 'deephaven.dataDir', "$rootDir/tmp/workspace" - systemProperty 'configuration.quiet', 'true' - - exclude '**/NoTest*' -} \ No newline at end of file diff --git a/Net/gradle.properties b/Net/gradle.properties deleted file mode 100644 index c186bbfdde1..00000000000 --- a/Net/gradle.properties +++ /dev/null @@ -1 +0,0 @@ -io.deephaven.project.ProjectType=JAVA_PUBLIC diff --git a/Net/src/main/java/io/deephaven/net/CommBase.java b/Net/src/main/java/io/deephaven/net/CommBase.java deleted file mode 100644 index cdbdb9b42f3..00000000000 --- a/Net/src/main/java/io/deephaven/net/CommBase.java +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.net; - -import io.deephaven.base.FatalErrorHandler; -import io.deephaven.base.FatalErrorHandlerFactory; -import io.deephaven.configuration.Configuration; -import io.deephaven.net.impl.nio.NIODriver; -import io.deephaven.io.NioUtil; -import io.deephaven.io.logger.Logger; -import io.deephaven.io.sched.*; - -import java.io.IOException; -import java.nio.channels.Selector; - -public class CommBase { - - private static volatile FatalErrorHandler defaultFatalErrorHandler; - - public static FatalErrorHandler getDefaultFatalHandler() { - if (defaultFatalErrorHandler == null) { - synchronized (CommBase.class) { - if (defaultFatalErrorHandler == null) { - final String defaultFatalErrorHandlerClassName = - Configuration.getInstance().getProperty("Comm.fatalErrorHandlerFactoryClass"); - final Class defaultFatalErrorHandlerClass; - try { - defaultFatalErrorHandlerClass = Class.forName(defaultFatalErrorHandlerClassName); - } catch (ClassNotFoundException e) { - throw new IllegalArgumentException( - "Could not find envelopeHandlerFactoryClass " + defaultFatalErrorHandlerClassName, e); - } - final FatalErrorHandlerFactory defaultFatalErrorHandlerFactory; - try { - defaultFatalErrorHandlerFactory = - (FatalErrorHandlerFactory) defaultFatalErrorHandlerClass.newInstance(); - } catch (InstantiationException | IllegalAccessException | ClassCastException e) { - throw new IllegalArgumentException( - "Could not instantiate envelopeHandlerFactoryClass " + defaultFatalErrorHandlerClass, - e); - } - defaultFatalErrorHandler = defaultFatalErrorHandlerFactory.get(); - } - } - } - return defaultFatalErrorHandler; - } - - public static void signalFatalError(final String message, Throwable x) { - try { - FatalErrorHandler feh = getDefaultFatalHandler(); - feh.signalFatalError(message, x); - } catch (Throwable fehx) { - // dump this to stderr, it's not great, but we had an error raising an error and really do want both of - // these in the log - fehx.printStackTrace(System.err); - x.printStackTrace(System.err); - throw new RuntimeException("Could not raise fatal error: " + message, x); - } - } - - /** - * Return the scheduler used by the NIO implementation - */ - public static Scheduler getScheduler() { - NIODriver.init(); - return NIODriver.getScheduler(); - } - - /** - * Create a private, single-threaded scheduler and driver thread - */ - public static class SingleThreadedScheduler extends YASchedulerImpl { - private final Thread driver; - private volatile boolean done = false; - - public SingleThreadedScheduler(final String name, Logger log) throws IOException { - super(name, NioUtil.reduceSelectorGarbage(Selector.open()), log); - this.driver = new Thread(() -> { - try { - while (!SingleThreadedScheduler.this.done) { - work(10, null); - } - } catch (Throwable x) { - signalFatalError(name + " exception", x); - } - }); - driver.setName(name + "-Driver"); - driver.setDaemon(true); - } - - public SingleThreadedScheduler start() { - driver.start(); - return this; - } - - public void stop() { - done = true; - } - } - - public static SingleThreadedScheduler singleThreadedScheduler(final String name, Logger log) { - try { - return new SingleThreadedScheduler(name, log); - } catch (IOException x) { - signalFatalError(name + " exception", x); - return null; - } - } -} diff --git a/Net/src/main/java/io/deephaven/net/impl/nio/FastNIODriver.java b/Net/src/main/java/io/deephaven/net/impl/nio/FastNIODriver.java deleted file mode 100644 index 492d5718f0b..00000000000 --- a/Net/src/main/java/io/deephaven/net/impl/nio/FastNIODriver.java +++ /dev/null @@ -1,285 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.net.impl.nio; - -import io.deephaven.base.UnfairMutex; -import io.deephaven.configuration.Configuration; -import io.deephaven.net.CommBase; -import io.deephaven.io.NioUtil; -import io.deephaven.io.logger.LogCrashDump; -import io.deephaven.io.logger.Logger; -import io.deephaven.io.sched.Scheduler; -import io.deephaven.io.sched.TimedJob; -import io.deephaven.io.sched.YASchedulerImpl; - -import java.io.IOException; -import java.nio.channels.Selector; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -public final class FastNIODriver implements Runnable { - private static Logger log; - - public static int numTotalThreads(String property) { - final String[] values = Configuration.getInstance().getProperty(property).split(","); - return Integer.parseInt(values[0]) * Integer.parseInt(values[1]); - } - - public static int threadsPerScheduler(String property) { - final String[] values = Configuration.getInstance().getProperty(property).split(","); - if (values.length != 6) - return 0; - return Integer.parseInt(values[1]); - } - - public static Scheduler[] createSchedulers(String name, String property, Logger log) { - return createSchedulers(name, property, log, Configuration.getInstance()); - } - - public static Scheduler[] createSchedulers(String name, String property, Logger log, Configuration config) { - final String[] values = config.getProperty(property).split(","); - if (values.length != 6) - return null; - - final int numSchedulers = Integer.parseInt(values[0]); - final int threadsPerScheduler = Integer.parseInt(values[1]); - final long timeoutsOrSpins = Long.parseLong(values[2]); - final int spinsUntilPark = Integer.parseInt(values[3]); - final boolean doTimingStats = Boolean.parseBoolean(values[4]); - final boolean doSpinSelect = Boolean.parseBoolean(values[5]); - final Scheduler[] schedulers = new Scheduler[numSchedulers]; - for (int i = 0; i < numSchedulers; ++i) { - schedulers[i] = createDrivers(name + "-" + i, log, threadsPerScheduler, threadsPerScheduler, - timeoutsOrSpins, spinsUntilPark, false, doTimingStats, doSpinSelect).getScheduler(); - } - return schedulers; - } - - public static FastNIODriver createDrivers(String name, Logger log, int initialThreads, int maxThreads, - long workTimeout, int spinsUntilPark, boolean crashOnMax) { - return createDrivers(name, log, initialThreads, maxThreads, workTimeout, spinsUntilPark, crashOnMax, true, - false); - } - - public static FastNIODriver createDrivers(String name, Logger log, int initialThreads, int maxThreads, - long workTimeout, int spinsUntilPark, boolean crashOnMax, boolean doTimingStats, boolean doSpinSelect) { - FastNIODriver.log = log; - log.info().append(name).append(": Starting FastNIODriver Scheduler: threads: ").append(initialThreads) - .append(", maxThreads: ").append(maxThreads) - .append(", workTimeout/spinsOnSelect: ").append(workTimeout) - .append(", spinsUntilPark: ").append(spinsUntilPark) - .append(", doSpinSelect: ").append(doSpinSelect) - .endl(); - try { - final Scheduler scheduler = new YASchedulerImpl(name, NioUtil.reduceSelectorGarbage(Selector.open()), log, - doTimingStats, doSpinSelect); - - final UnfairMutex mutex = new UnfairMutex(spinsUntilPark, maxThreads); - final AtomicBoolean shutdown = new AtomicBoolean(false); - final AtomicInteger created = new AtomicInteger(0); - final AtomicInteger destroyed = new AtomicInteger(0); - final AtomicInteger available = new AtomicInteger(0); - final InternalThread[] threads = new InternalThread[initialThreads]; - // separate the creation and start so the created / available values are setup - for (int i = 0; i < initialThreads; ++i) { - threads[i] = createNewThread(name, scheduler, mutex, shutdown, workTimeout, created, destroyed, - available, maxThreads, crashOnMax); - } - for (int i = 0; i < initialThreads; ++i) { - threads[i].start(); - } - - return threads[0].driver; - } catch (IOException x) { - CommBase.signalFatalError(name + ": FastNIODriver can't create scheduler", x); - return null; - } - } - - private static class InternalThread extends Thread { - private final FastNIODriver driver; - - private InternalThread(final FastNIODriver driver) { - super(driver); - this.driver = driver; - } - } - - private static InternalThread createNewThread(final String name, final Scheduler scheduler, final UnfairMutex mutex, - final AtomicBoolean shutdown, final long workTimeout, final AtomicInteger created, - final AtomicInteger destroyed, final AtomicInteger available, final int maxThreads, - final boolean crashOnMax) { - InternalThread t = new InternalThread(new FastNIODriver(name, scheduler, mutex, shutdown, workTimeout, created, - destroyed, available, maxThreads, crashOnMax)); - t.setDaemon(true); - t.setName(name + "-FastNIODriver-" + created.getAndIncrement()); - int a = available.incrementAndGet(); - log.info().append("Creating thread ").append(t.getName()).append(". available: ").append(a).endl(); - return t; - } - - private final Scheduler scheduler; - private final UnfairMutex mutex; - private final AtomicBoolean shutdown; - private final long workTimeout; - private final Runnable mutexUnlockHandoff; - private boolean alreadyHandedOff; - - private final AtomicInteger created; - private final AtomicInteger destroyed; - private final AtomicInteger available; - private final int maxThreads; - private final boolean crashOnMax; - - private FastNIODriver(final String name, final Scheduler scheduler, final UnfairMutex mutex, - final AtomicBoolean shutdown, final long workTimeout, final AtomicInteger created, - final AtomicInteger destroyed, final AtomicInteger available, final int maxThreads, - final boolean crashOnMax) { - this.scheduler = scheduler; - this.mutex = mutex; - this.shutdown = shutdown; - this.workTimeout = workTimeout; - this.created = created; - this.destroyed = destroyed; - this.available = available; - this.maxThreads = maxThreads; - this.crashOnMax = crashOnMax; - alreadyHandedOff = false; - mutexUnlockHandoff = () -> { - if (!alreadyHandedOff) { - if (shouldCreate()) { - // nobody to handoff to! let's create a new driver - createNewThread(name, scheduler, mutex, shutdown, workTimeout, created, destroyed, available, - maxThreads, crashOnMax).start(); - } - mutex.unlock(); - alreadyHandedOff = true; - } - }; - } - - // only called when we have the mutex... - private boolean shouldCreate() { - if (available.get() == 0) { - // don't need to worry about races w/ index b/c we have lock - if (created.get() == maxThreads) { - if (crashOnMax) { - log.fatal().append("FastNIODriver: exceeded maximum thread pool limit: ").append(summary()).endl(); - LogCrashDump.logCrashDump(log); - CommBase.signalFatalError("FastNIODriver: exceeded maximum thread pool limit: " + summary(), - new Throwable()); - } - return false; - } - return true; - } - return false; - } - - public String summary() { - return "(available: " + available.get() + ", created: " + created.get() + ", destroyed: " + destroyed.get() - + ")"; - } - - @Override - public void run() { - final Thread me = Thread.currentThread(); - Throwable throwable = null; - while (true) { - if (shutdown.get()) { - break; - } - mutex.lock(); - alreadyHandedOff = false; - if (shutdown.get()) { - mutexUnlockHandoff.run(); - break; - } - - try { - - available.getAndDecrement(); - do { - scheduler.work(workTimeout, mutexUnlockHandoff); - } while (mutex.getOwner() == me); - available.getAndIncrement(); - - } catch (Throwable x) { - throwable = x; - shutdown.set(true); - scheduler.installJob(new TimedJob() { - public void timedOut() {} - }, 0); // wake us up yo - mutexUnlockHandoff.run(); // we aren't sure whether the scheduler.work has already called the handoff - // or not yet, so go ahead and call it (it won't double release it) - long deadline = System.currentTimeMillis() + 5000; - // b/c we haven't destroyed ourself yet... - // meh spinning :/ - while (created.get() != destroyed.get() + 1) { - if (deadline - System.currentTimeMillis() < 0) { - break; - } - Thread.yield(); // better than spinning? - } - - break; - } - } - - if (destroyed.incrementAndGet() == created.get()) { - scheduler.close(); - } - - if (throwable == null) { - log.error().append("Thread ").append(me.getName()).append(" is terminating: ").append(summary()).endl(); - } else { - log.fatal(throwable).append("Thread ").append(me.getName()).append(" is terminating on a fatal exception: ") - .append(summary()).endl(); - } - - if (throwable != null) - CommBase.signalFatalError("Unhandled throwable from FastNIODriver scheduler", throwable); - } - - public boolean isShutdown() { - return shutdown.get(); - } - - public boolean shutdown(long maxWait) { - shutdown.set(true); - scheduler.installJob(new TimedJob() { - public void timedOut() {} - }, 0); - long deadline = System.currentTimeMillis() + maxWait; - while (created.get() != destroyed.get()) { - if (deadline - System.currentTimeMillis() < 0) { - break; - } - try { - Thread.sleep(1); // better than spinning? - } catch (InterruptedException e) { - // ignore - } - } - - return created.get() == destroyed.get(); - } - - public Scheduler getScheduler() { - return scheduler; - } - - // whitebox test support methods - public int junit_getWaiting() { - return available.get(); - } - - public int junit_getCreated() { - return created.get(); - } - - public int junit_getDestroyed() { - return destroyed.get(); - } -} diff --git a/Net/src/main/java/io/deephaven/net/impl/nio/NIODriver.java b/Net/src/main/java/io/deephaven/net/impl/nio/NIODriver.java deleted file mode 100644 index f1c57602a45..00000000000 --- a/Net/src/main/java/io/deephaven/net/impl/nio/NIODriver.java +++ /dev/null @@ -1,295 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.net.impl.nio; - -import io.deephaven.internal.log.LoggerFactory; -import java.io.IOException; -import java.nio.channels.Selector; -import java.util.concurrent.atomic.AtomicInteger; - -import io.deephaven.net.CommBase; -import io.deephaven.configuration.Configuration; -import io.deephaven.io.NioUtil; -import io.deephaven.io.logger.LogCrashDump; -import io.deephaven.io.logger.Logger; -import io.deephaven.io.sched.Scheduler; -import io.deephaven.io.sched.TimedJob; -import io.deephaven.io.sched.YASchedulerImpl; - -public class NIODriver implements Runnable { - private static Logger log; - - private static boolean initialized = false; - private static volatile boolean stopped = false; - - private static Scheduler sched = null; - private static FastNIODriver driver = null; - - private static final Object lock = new Object(); - private static Thread leader = null; - private static AtomicInteger available = new AtomicInteger(0); - private static int created = 0; - private static int destroyed = 0; - - public static int WORK_TIMEOUT; - public static int NUM_INITIAL_THREADS; - public static int HARD_MAX_THREADS; - - private static final boolean useFastNIODriver = Configuration.getInstance().getBoolean("NIO.driver.useFast"); - - /** - * Let another thread take over the leadership. - */ - private static void handoff() { - Thread me = Thread.currentThread(); - synchronized (lock) { - if (leader != me) { - LogCrashDump.logCrashDump(log); - CommBase.signalFatalError("NIODriver: WTF? in handoff(), but not the leader?", new Throwable()); - } - - if (log.isDebugEnabled()) { - log.debug().append("Thread ").append(me.getName()).append(" is giving up leadership").endl(); - } - - leader = null; - - if (stopped || available.get() != 0) { - lock.notify(); - } else { - // no joy, have to add another thread - log.warn().append("Thread ").append(me.getName()).append(" is handing off with no threads available: ") - .append(summary()).endl(); - addThread(); - } - } - } - - /** - * A procedure which calls handoff(), to give the scheduler when we are running full-bore - */ - private static final Runnable handoffProc = NIODriver::handoff; - - /** - * return a string telling how many threads are doing what - */ - public static String summary() { - if (useFastNIODriver) { - return driver.summary(); - } else { - return "(available: " + available + ", created: " + created + ", destroyed: " + destroyed + ")"; - } - } - - /** - * one-time initialization - */ - public static void init() { - if (!initialized) { - init(LoggerFactory.getLogger(NIODriver.class)); - } - } - - public static void init(Logger log) { - synchronized (lock) { - if (!initialized) { - NIODriver.log = log; - WORK_TIMEOUT = Configuration.getInstance().getInteger("NIO.driver.workTimeout"); - NUM_INITIAL_THREADS = Configuration.getInstance().getInteger("NIO.driver.initialThreadCount"); - HARD_MAX_THREADS = Configuration.getInstance().getInteger("NIO.driver.maxThreadCount"); - if (useFastNIODriver) { - driver = FastNIODriver.createDrivers("Static", log, NUM_INITIAL_THREADS, HARD_MAX_THREADS, - WORK_TIMEOUT, 1000, true); - sched = driver.getScheduler(); - } else { - try { - sched = new YASchedulerImpl(NioUtil.reduceSelectorGarbage(Selector.open()), log); - } catch (IOException x) { - sched = null; - CommBase.signalFatalError("NIODriver.init: can't create scheduler", x); - } - for (int i = 0; i < NUM_INITIAL_THREADS; ++i) { - addThread(); - } - } - initialized = true; - } - } - - } - - /** - * Shut down, and wait for all threads to terminate. This method is really just for testing; it's a bad idea to do - * this in production because waiting for threads to terminate is prone to deadlocks. If desired, though, it can be - * called from an AbstractService shutdown hook installed in init(). - */ - public static boolean shutdown(long maxWait) { - synchronized (lock) { - if (!initialized) - return true; - - if (useFastNIODriver) { - if (driver.shutdown(maxWait)) { - initialized = false; - log.info().append("NIODriver.shutdown: finished").endl(); - return true; - } else { - return false; - } - } else { - long deadline = System.currentTimeMillis() + maxWait, remain = maxWait; - stopped = true; - lock.notifyAll(); - // force the scheduler to wake up - sched.installJob(new TimedJob() { - public void timedOut() {} - }, 0); - while (created != destroyed) { - try { - log.info().append("NIODriver.shutdown: waiting for threads to terminate: ").append(summary()) - .endl(); - lock.wait(Math.max(remain, 0)); - } catch (InterruptedException x) { - // ignore - } - if ((remain = deadline - System.currentTimeMillis()) < 0) { - return false; - } - } - sched.close(); - log.info().append("NIODriver.shutdown: finished").endl(); - leader = null; - sched = null; - initialized = stopped = false; - created = destroyed = 0; - available.set(0); - return true; - } - } - } - - /** - * Return the scheduler used by the NIO driver - */ - public static Scheduler getScheduler() { - return sched; - } - - /** - * Return the scheduler used by the NIO driver - */ - public static Logger getLogger() { - return log; - } - - /** - * add a thread to the pool - * - * NOTE: caller must hold the lock! - * - * NOTE: We increment the "waiting" variable *before* we start the new thread, and then make sure to correct it in - * the first iteration of the thread loop. This prevents a race in which we handoff() method creates too many - * threads, because it keeps getting called before the first thread it creates can get started. - */ - private static void addThread() { - if (created == HARD_MAX_THREADS) { - log.fatal().append("NIODriver: exceeded maximum thread pool limit: ").append(summary()).endl(); - LogCrashDump.logCrashDump(log); - CommBase.signalFatalError("NIODriver: exceeded maximum thread pool limit: " + summary(), new Throwable()); - } - Thread thread = new Thread(new NIODriver()); - thread.setDaemon(true); - thread.setName("NIODriver-" + created); - created++; - available.incrementAndGet(); - log.info().append("Thread ").append(thread.getName()).append(" is starting: ").append(summary()).endl(); - thread.start(); - } - - /** - * the threads' run method just does an endless loop, trying to become the leader whenever it can - */ - public void run() { - Thread me = Thread.currentThread(); - STOP: { - while (true) { - synchronized (lock) { - while (leader != me) { - if (stopped) { - destroyed++; - log.info().append("Thread ").append(me.getName()).append(" is terminating: ") - .append(summary()).endl(); - lock.notifyAll(); - break STOP; - } else if (leader == null) { - if (log.isDebugEnabled()) { - log.debug().append("Thread ").append(me.getName()).append(" is assuming leadership") - .endl(); - } - leader = me; - } else { - try { - if (log.isDebugEnabled()) { - log.debug().append("Thread ").append(me.getName()).append(" is waiting ") - .append(summary()).endl(); - } - lock.wait(); - if (log.isDebugEnabled()) { - log.debug().append("Thread ").append(me.getName()).append(" has awoken ") - .append(summary()).endl(); - } - } catch (InterruptedException x) { - // ignore - } - } - } - } - try { - available.decrementAndGet(); - sched.work(WORK_TIMEOUT, handoffProc); - available.incrementAndGet(); - } catch (Throwable x) { - synchronized (lock) { - destroyed++; - log.fatal(x).append("Thread ").append(me.getName()) - .append(" is terminating on a fatal exception: ").append(summary()).endl(); - lock.notifyAll(); - } - - NIODriver.shutdown(5000); - CommBase.signalFatalError("Unhandled throwable from NIO scheduler", x); - break STOP; - } - } - } - } - - // whitebox test support methods - public static int junit_getWaiting() { - if (useFastNIODriver) { - return driver.junit_getWaiting(); - } else { - return available.get(); - } - } - - public static int junit_getCreated() { - if (useFastNIODriver) { - return driver.junit_getCreated(); - } else { - return created; - } - } - - public static int junit_getDestroyed() { - if (useFastNIODriver) { - return driver.junit_getDestroyed(); - } else { - return destroyed; - } - } - - // ################################################################ - -} diff --git a/Stats/build.gradle b/Stats/build.gradle index e9d46238886..bac1cdce147 100644 --- a/Stats/build.gradle +++ b/Stats/build.gradle @@ -7,8 +7,6 @@ dependencies { implementation project(':DataStructures') implementation project(':IO') implementation project(':Configuration') - implementation project(':FishUtil') - implementation project(':Net') implementation project(':log-factory') implementation project(':engine-context') compileOnly 'com.google.code.java-allocation-instrumenter:java-allocation-instrumenter:3.3.0' diff --git a/Stats/src/main/java/io/deephaven/stats/StatsCPUCollector.java b/Stats/src/main/java/io/deephaven/stats/StatsCPUCollector.java index 1f003b5fb1e..7b1a3e8c8f8 100644 --- a/Stats/src/main/java/io/deephaven/stats/StatsCPUCollector.java +++ b/Stats/src/main/java/io/deephaven/stats/StatsCPUCollector.java @@ -7,7 +7,7 @@ import io.deephaven.configuration.Configuration; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; -import io.deephaven.util.OSUtil; +import io.deephaven.stats.util.OSUtil; import io.deephaven.base.stats.*; import io.deephaven.hash.KeyedLongObjectHash; import io.deephaven.hash.KeyedLongObjectHashMap; diff --git a/Stats/src/main/java/io/deephaven/stats/StatsDriver.java b/Stats/src/main/java/io/deephaven/stats/StatsDriver.java index 29c10096d13..beeda59894b 100644 --- a/Stats/src/main/java/io/deephaven/stats/StatsDriver.java +++ b/Stats/src/main/java/io/deephaven/stats/StatsDriver.java @@ -5,23 +5,27 @@ import io.deephaven.base.clock.Clock; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.net.CommBase; import io.deephaven.util.SafeCloseable; -import io.deephaven.util.formatters.ISO8601; import io.deephaven.base.stats.*; import io.deephaven.base.text.TimestampBuffer; import io.deephaven.configuration.Configuration; import io.deephaven.io.log.*; -import io.deephaven.io.sched.TimedJob; import io.deephaven.io.log.impl.LogEntryPoolImpl; import io.deephaven.io.log.impl.LogSinkImpl; +import io.deephaven.util.annotations.ReferentialIntegrity; +import io.deephaven.util.thread.NamingThreadFactory; import java.util.Properties; +import java.util.TimeZone; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; /** * Drives the collection of statistics on a 1-second timer task. */ -public class StatsDriver extends TimedJob { +public class StatsDriver { public interface StatusAdapter { void sendAlert(String alertText); @@ -39,11 +43,11 @@ public boolean cmsAlertEnabled() { } private final LogEntryPool entryPool; - private final LogSink sink; + private final LogSink sink; private final LogEntry[] entries; private final LogEntryPool entryPoolHisto; - private final LogSink sinkHisto; + private final LogSink sinkHisto; private final LogEntry[] entriesHisto; private final TimestampBuffer systemTimestamp; @@ -52,9 +56,8 @@ public boolean cmsAlertEnabled() { public final static String header = "Stat,IntervalName,NowSec,NowString,AppNowSec,AppNowString,TypeTag,Name,N,Sum,Last,Min,Max,Avg,Sum2,Stdev"; - private long nextInvocation = System.currentTimeMillis(); - private long nextCpuUpdate = nextInvocation + CPU_INTERVAL; - private long nextMemUpdate = nextInvocation + MEM_INTERVAL; + private long nextCpuUpdate; + private long nextMemUpdate; private static final long STEP = 1000; private static final long MEM_INTERVAL = 1000; @@ -71,9 +74,14 @@ public boolean cmsAlertEnabled() { private final StatsIntradayLogger intraday; private final Value clockValue; private final ExecutionContext executionContext; + @ReferentialIntegrity + private final ScheduledExecutorService scheduler; + @ReferentialIntegrity + private final ScheduledFuture updateJobFuture; private final StatsMemoryCollector memStats; private final StatsCPUCollector cpuStats; + @ReferentialIntegrity private ObjectAllocationCollector objectAllocation; public StatsDriver(Clock clock) { @@ -116,8 +124,9 @@ public StatsDriver(Clock clock, StatsIntradayLogger intraday, boolean getFdStats } } - this.systemTimestamp = new TimestampBuffer(ISO8601.serverTimeZone()); - this.appTimestamp = new TimestampBuffer(ISO8601.serverTimeZone()); + final TimeZone serverTimeZone = Configuration.getInstance().getServerTimezone(); + this.systemTimestamp = new TimestampBuffer(serverTimeZone); + this.appTimestamp = new TimestampBuffer(serverTimeZone); if (path == null) { this.entryPool = null; @@ -150,9 +159,11 @@ public StatsDriver(Clock clock, StatsIntradayLogger intraday, boolean getFdStats clockValue = null; } - long now = System.currentTimeMillis(); - long delay = STEP - (now % STEP); - nextInvocation = now + delay; + final long now = System.currentTimeMillis(); + final long delay = STEP - (now % STEP); + nextCpuUpdate = now + delay + CPU_INTERVAL; + nextMemUpdate = now + delay + MEM_INTERVAL; + cpuStats = new StatsCPUCollector(CPU_INTERVAL, getFdStats); memStats = new StatsMemoryCollector(MEM_INTERVAL, statusAdapter::sendAlert, statusAdapter::cmsAlertEnabled); if (Configuration.getInstance().getBoolean("allocation.stats.enabled")) { @@ -160,13 +171,18 @@ public StatsDriver(Clock clock, StatsIntradayLogger intraday, boolean getFdStats } executionContext = ExecutionContext.getContext(); - // now that the StatsDriver is completely constructed, we can schedule the first iteration + // now that the StatsDriver is completely constructed, we can schedule the update job if (Configuration.getInstance().getBoolean("statsdriver.enabled")) { - schedule(); + scheduler = Executors.newSingleThreadScheduledExecutor( + new NamingThreadFactory(StatsDriver.class, "updateScheduler", true)); + updateJobFuture = scheduler.scheduleAtFixedRate(this::update, delay, STEP, TimeUnit.MILLISECONDS); + } else { + scheduler = null; + updateJobFuture = null; } } - public void timedOut() { + private void update() { long t0 = System.nanoTime(); long now = System.currentTimeMillis(); long appNow = clock == null ? now : clock.currentTimeMillis(); @@ -207,20 +223,12 @@ public void timedOut() { } } - schedule(); - statsTiming.sample((System.nanoTime() - t0 + 500) / 1000); } - private void schedule() { - CommBase.getScheduler().installJob(this, nextInvocation); - long steps = Math.max(1L, (((System.currentTimeMillis() - nextInvocation) / STEP) + 1)); - nextInvocation += steps * STEP; - } - private final ItemUpdateListener LISTENER = new ItemUpdateListener() { @Override - public void handleItemUpdated(Item item, long now, long appNow, int intervalIndex, long intervalMillis, + public void handleItemUpdated(Item item, long now, long appNow, int intervalIndex, long intervalMillis, String intervalName) { final Value v = item.getValue(); final History history = v.getHistory(); diff --git a/FishUtil/src/main/java/io/deephaven/util/OSUtil.java b/Stats/src/main/java/io/deephaven/stats/util/OSUtil.java similarity index 85% rename from FishUtil/src/main/java/io/deephaven/util/OSUtil.java rename to Stats/src/main/java/io/deephaven/stats/util/OSUtil.java index 4e136975d2c..ef3566f4696 100644 --- a/FishUtil/src/main/java/io/deephaven/util/OSUtil.java +++ b/Stats/src/main/java/io/deephaven/stats/util/OSUtil.java @@ -1,20 +1,22 @@ /** * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending */ -package io.deephaven.util; +package io.deephaven.stats.util; import org.jetbrains.annotations.NotNull; import java.util.Arrays; import java.util.function.Predicate; -@SuppressWarnings("WeakerAccess") public class OSUtil { public enum OSFamily { - - LINUX(name -> name.startsWith("Linux")), WINDOWS(name -> name.contains("Windows")), MAC_OS( - name -> name.startsWith("Mac OS")), SOLARIS(name -> name.startsWith("SunOs")); + // @formatter:off + LINUX(name -> name.startsWith("Linux")), + WINDOWS(name -> name.contains("Windows")), + MAC_OS(name -> name.startsWith("Mac OS")), + SOLARIS(name -> name.startsWith("SunOs")); + // @formatter:on private final Predicate nameMatcher; diff --git a/TableLogger/TableLogger.gradle b/TableLogger/TableLogger.gradle index 411b1b752d0..a4173ac368d 100644 --- a/TableLogger/TableLogger.gradle +++ b/TableLogger/TableLogger.gradle @@ -2,12 +2,8 @@ plugins { id 'io.deephaven.project.register' } -configurations { - implementation.extendsFrom fishUtil, fishData - testImplementation.extendsFrom fishDataTest -} - dependencies { + implementation project(':Base') implementation project(':Util') testRuntimeOnly project(path: ':configs') testRuntimeOnly project(path: ':test-configs') diff --git a/FishUtil/src/main/java/io/deephaven/util/process/BaseProcessEnvironment.java b/Util/src/main/java/io/deephaven/util/process/BaseProcessEnvironment.java similarity index 100% rename from FishUtil/src/main/java/io/deephaven/util/process/BaseProcessEnvironment.java rename to Util/src/main/java/io/deephaven/util/process/BaseProcessEnvironment.java diff --git a/FishUtil/src/main/java/io/deephaven/util/process/DefaultFatalErrorReporter.java b/Util/src/main/java/io/deephaven/util/process/DefaultFatalErrorReporter.java similarity index 100% rename from FishUtil/src/main/java/io/deephaven/util/process/DefaultFatalErrorReporter.java rename to Util/src/main/java/io/deephaven/util/process/DefaultFatalErrorReporter.java diff --git a/FishUtil/src/main/java/io/deephaven/util/process/DefaultProcessEnvironment.java b/Util/src/main/java/io/deephaven/util/process/DefaultProcessEnvironment.java similarity index 100% rename from FishUtil/src/main/java/io/deephaven/util/process/DefaultProcessEnvironment.java rename to Util/src/main/java/io/deephaven/util/process/DefaultProcessEnvironment.java diff --git a/FishUtil/src/main/java/io/deephaven/util/process/FatalErrorReporter.java b/Util/src/main/java/io/deephaven/util/process/FatalErrorReporter.java similarity index 100% rename from FishUtil/src/main/java/io/deephaven/util/process/FatalErrorReporter.java rename to Util/src/main/java/io/deephaven/util/process/FatalErrorReporter.java diff --git a/FishUtil/src/main/java/io/deephaven/util/process/FatalErrorReporterBase.java b/Util/src/main/java/io/deephaven/util/process/FatalErrorReporterBase.java similarity index 100% rename from FishUtil/src/main/java/io/deephaven/util/process/FatalErrorReporterBase.java rename to Util/src/main/java/io/deephaven/util/process/FatalErrorReporterBase.java diff --git a/FishUtil/src/main/java/io/deephaven/util/process/LoggerShutdownTask.java b/Util/src/main/java/io/deephaven/util/process/LoggerShutdownTask.java similarity index 100% rename from FishUtil/src/main/java/io/deephaven/util/process/LoggerShutdownTask.java rename to Util/src/main/java/io/deephaven/util/process/LoggerShutdownTask.java diff --git a/FishUtil/src/main/java/io/deephaven/util/process/OnetimeShutdownTask.java b/Util/src/main/java/io/deephaven/util/process/OnetimeShutdownTask.java similarity index 100% rename from FishUtil/src/main/java/io/deephaven/util/process/OnetimeShutdownTask.java rename to Util/src/main/java/io/deephaven/util/process/OnetimeShutdownTask.java diff --git a/FishUtil/src/main/java/io/deephaven/util/process/ProcessEnvironment.java b/Util/src/main/java/io/deephaven/util/process/ProcessEnvironment.java similarity index 100% rename from FishUtil/src/main/java/io/deephaven/util/process/ProcessEnvironment.java rename to Util/src/main/java/io/deephaven/util/process/ProcessEnvironment.java diff --git a/FishUtil/src/main/java/io/deephaven/util/process/ShutdownManager.java b/Util/src/main/java/io/deephaven/util/process/ShutdownManager.java similarity index 100% rename from FishUtil/src/main/java/io/deephaven/util/process/ShutdownManager.java rename to Util/src/main/java/io/deephaven/util/process/ShutdownManager.java diff --git a/FishUtil/src/main/java/io/deephaven/util/process/ShutdownManagerImpl.java b/Util/src/main/java/io/deephaven/util/process/ShutdownManagerImpl.java similarity index 99% rename from FishUtil/src/main/java/io/deephaven/util/process/ShutdownManagerImpl.java rename to Util/src/main/java/io/deephaven/util/process/ShutdownManagerImpl.java index 520b86a96bf..5b3e4dc68ae 100644 --- a/FishUtil/src/main/java/io/deephaven/util/process/ShutdownManagerImpl.java +++ b/Util/src/main/java/io/deephaven/util/process/ShutdownManagerImpl.java @@ -9,8 +9,8 @@ import io.deephaven.io.log.LogEntry; import io.deephaven.io.log.LogLevel; import io.deephaven.io.logger.Logger; -import io.deephaven.util.threads.ThreadDump; import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.util.thread.ThreadDump; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; diff --git a/buildSrc/src/main/groovy/io.deephaven.java-classpath-conventions.gradle b/buildSrc/src/main/groovy/io.deephaven.java-classpath-conventions.gradle index baa0413b77f..d008aa19202 100644 --- a/buildSrc/src/main/groovy/io.deephaven.java-classpath-conventions.gradle +++ b/buildSrc/src/main/groovy/io.deephaven.java-classpath-conventions.gradle @@ -29,14 +29,12 @@ configurations { fishDataStructure.extendsFrom fishIo fishConfig.extendsFrom fishDataStructure fishDataGenerator.extendsFrom jdom - fishNet.extendsFrom fishIo fishNumerics.extendsFrom fishBase - fishUtil.extendsFrom fishConfig fishBaseTest.extendsFrom junit fishIoTest.extendsFrom fishBaseTest dhNumerics.extendsFrom fishNumerics, jama - dhUtil.extendsFrom commonsIo, commonsLang3, commonsText, fishUtil, fishNet, fishIo, jdom + dhUtil.extendsFrom commonsIo, commonsLang3, commonsText, fishConfig, fishIo, jdom dhPlot.extendsFrom dhUtil dhBenchmarkSupport.extendsFrom fishData dhIntegrations.extendsFrom math3 @@ -70,10 +68,6 @@ dependencies { fishConfig project(':Configuration') - fishUtil project(':FishUtil') - - fishNet project(':Net') - fishBaseTest project(path: ':Base', configuration: 'tests') fishIoTest project(path: ':IO', configuration: 'tests') diff --git a/engine/table/build.gradle b/engine/table/build.gradle index 4456f02c262..261678a3c06 100644 --- a/engine/table/build.gradle +++ b/engine/table/build.gradle @@ -28,8 +28,6 @@ dependencies { implementation project(':Configuration') implementation project(':log-factory') implementation project(':Stats') - implementation project(':Net') - implementation project(':FishUtil') implementation 'com.github.f4b6a3:uuid-creator:5.2.0' // TODO(deephaven-core#3204): t-digest 3.3 appears to have higher errors than 3.2 diff --git a/engine/table/src/main/java/io/deephaven/engine/updategraph/impl/PeriodicUpdateGraph.java b/engine/table/src/main/java/io/deephaven/engine/updategraph/impl/PeriodicUpdateGraph.java index a2fbaee6581..34e0462dde7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/updategraph/impl/PeriodicUpdateGraph.java +++ b/engine/table/src/main/java/io/deephaven/engine/updategraph/impl/PeriodicUpdateGraph.java @@ -27,9 +27,6 @@ import io.deephaven.io.log.LogEntry; import io.deephaven.io.log.impl.LogOutputStringImpl; import io.deephaven.io.logger.Logger; -import io.deephaven.io.sched.Scheduler; -import io.deephaven.io.sched.TimedJob; -import io.deephaven.net.CommBase; import io.deephaven.util.SafeCloseable; import io.deephaven.util.annotations.TestUseOnly; import io.deephaven.util.datastructures.SimpleReferenceManager; @@ -51,6 +48,8 @@ import java.util.function.BooleanSupplier; import java.util.function.LongConsumer; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + /** *

* This class uses a thread (or pool of threads) to periodically update a set of monitored update sources at a specified @@ -132,25 +131,30 @@ public static PerformanceEntry createUpdatePerformanceEntry( private final Thread refreshThread; private volatile boolean running = true; + /** + * {@link ScheduledExecutorService} used for scheduling the {@link #watchDogTimeoutProcedure}. + */ + private final ScheduledExecutorService watchdogScheduler; + /** * If this is set to a positive value, then we will call the {@link #watchDogTimeoutProcedure} if any single run * loop takes longer than this value. The intention is to use this for strategies, or other queries, where a * PeriodicUpdateGraph loop that is "stuck" is the equivalent of an error. Set the value with * {@link #setWatchDogMillis(int)}. */ - private int watchDogMillis = 0; + private volatile int watchDogMillis = 0; /** * If a timeout time has been {@link #setWatchDogMillis(int) set}, this procedure will be called if any single run * loop takes longer than the value specified. Set the value with * {@link #setWatchDogTimeoutProcedure(LongConsumer)}. */ - private LongConsumer watchDogTimeoutProcedure = null; + private volatile LongConsumer watchDogTimeoutProcedure; public static final String ALLOW_UNIT_TEST_MODE_PROP = "PeriodicUpdateGraph.allowUnitTestMode"; private final boolean allowUnitTestMode; - private int notificationAdditionDelay = 0; + private int notificationAdditionDelay; private Random notificationRandomizer = new Random(0); - private boolean unitTestMode = false; + private boolean unitTestMode; private ExecutorService unitTestRefreshThreadPool; public static final String DEFAULT_TARGET_CYCLE_DURATION_MILLIS_PROP = @@ -162,27 +166,27 @@ public static PerformanceEntry createUpdatePerformanceEntry( private final long minimumCycleDurationToLogNanos; /** when to next flush the performance tracker; initializes to zero to force a flush on start */ - private long nextUpdatePerformanceTrackerFlushTime = 0; + private long nextUpdatePerformanceTrackerFlushTimeNanos; /** * How many cycles we have not logged, but were non-zero. */ - private long suppressedCycles = 0; - private long suppressedCyclesTotalNanos = 0; - private long suppressedCyclesTotalSafePointTimeMillis = 0; + private long suppressedCycles; + private long suppressedCyclesTotalNanos; + private long suppressedCyclesTotalSafePointTimeMillis; /** * Accumulated UpdateGraph exclusive lock waits for the current cycle (or previous, if idle). */ - private long currentCycleLockWaitTotalNanos = 0; + private long currentCycleLockWaitTotalNanos; /** * Accumulated delays due to intracycle yields for the current cycle (or previous, if idle). */ - private long currentCycleYieldTotalNanos = 0L; + private long currentCycleYieldTotalNanos; /** * Accumulated delays due to intracycle sleeps for the current cycle (or previous, if idle). */ - private long currentCycleSleepTotalNanos = 0L; + private long currentCycleSleepTotalNanos; public static class AccumulatedCycleStats { /** @@ -331,6 +335,14 @@ public PeriodicUpdateGraph( } }), "PeriodicUpdateGraph." + name + ".refreshThread"); refreshThread.setDaemon(true); + watchdogScheduler = Executors.newSingleThreadScheduledExecutor( + new NamingThreadFactory(PeriodicUpdateGraph.class, "watchdogScheduler", true) { + @Override + public Thread newThread(@NotNull final Runnable r) { + // Not a refresh thread, but should still be instrumented for debugging purposes. + return super.newThread(ThreadInitializationFactory.wrapRunnable(r)); + } + }); updatePerformanceTracker = new UpdatePerformanceTracker(this); } @@ -593,15 +605,6 @@ public void setWatchDogTimeoutProcedure(LongConsumer procedure) { this.watchDogTimeoutProcedure = procedure; } - private class WatchdogJob extends TimedJob { - @Override - public void timedOut() { - if (watchDogTimeoutProcedure != null) { - watchDogTimeoutProcedure.accept(watchDogMillis); - } - } - } - /** * Install a real NotificationProcessor and start the primary refresh thread. * @@ -1129,7 +1132,7 @@ public Runnable flushAllNormalNotificationsForUnitTests(@NotNull final BooleanSu final ControlledNotificationProcessor controlledNotificationProcessor = new ControlledNotificationProcessor(); notificationProcessor = controlledNotificationProcessor; final Future flushJobFuture = unitTestRefreshThreadPool.submit(() -> { - final long deadlineNanoTime = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + final long deadlineNanoTime = System.nanoTime() + MILLISECONDS.toNanos(timeoutMillis); boolean flushed; while ((flushed = flushOneNotificationForUnitTestsInternal(false)) || !done.getAsBoolean()) { if (!flushed) { @@ -1676,8 +1679,6 @@ private static LogEntry appendAsMillisFromNanos(final LogEntry entry, final long * {@link #getTargetCycleDurationMillis() minimum cycle time}. */ private void refreshTablesAndFlushNotifications() { - final Scheduler sched = CommBase.getScheduler(); - final long startTime = sched.currentTimeMillis(); final long startTimeNanos = System.nanoTime(); jvmIntrospectionContext.startSample(); @@ -1686,17 +1687,20 @@ private void refreshTablesAndFlushNotifications() { } else { currentCycleLockWaitTotalNanos = currentCycleYieldTotalNanos = currentCycleSleepTotalNanos = 0L; - WatchdogJob watchdogJob = null; + ScheduledFuture watchdogFuture = null; - if ((watchDogMillis > 0) && (watchDogTimeoutProcedure != null)) { - watchdogJob = new WatchdogJob(); - sched.installJob(watchdogJob, startTime + watchDogMillis); + final long localWatchdogMillis = watchDogMillis; + final LongConsumer localWatchdogTimeoutProcedure = watchDogTimeoutProcedure; + if ((localWatchdogMillis > 0) && (localWatchdogTimeoutProcedure != null)) { + watchdogFuture = watchdogScheduler.schedule( + () -> localWatchdogTimeoutProcedure.accept(localWatchdogMillis), + localWatchdogMillis, MILLISECONDS); } refreshAllTables(); - if (watchdogJob != null) { - sched.cancelJob(watchdogJob); + if (watchdogFuture != null) { + watchdogFuture.cancel(true); } jvmIntrospectionContext.endSample(); final long cycleTimeNanos = System.nanoTime() - startTimeNanos; @@ -1707,7 +1711,7 @@ private void refreshTablesAndFlushNotifications() { Thread.yield(); } - waitForNextCycle(startTime, sched); + waitForNextCycle(startTimeNanos); } private void computeStatsAndLogCycle(final long cycleTimeNanos) { @@ -1791,24 +1795,25 @@ private void logSuppressedCycles() { * wait the remaining period. *

* - * @param startTime The start time of the last run cycle - * @param timeSource The source of time that startTime was based on + * @param startTimeNanos The start time of the last run cycle as reported by {@link System#nanoTime()} */ - private void waitForNextCycle(final long startTime, final Scheduler timeSource) { - final long now = timeSource.currentTimeMillis(); - long expectedEndTime = startTime + targetCycleDurationMillis; + private void waitForNextCycle(final long startTimeNanos) { + final long nowNanos = System.nanoTime(); + long expectedEndTimeNanos = startTimeNanos + MILLISECONDS.toNanos(targetCycleDurationMillis); if (minimumInterCycleSleep > 0) { - expectedEndTime = Math.max(expectedEndTime, now + minimumInterCycleSleep); + expectedEndTimeNanos = + Math.max(expectedEndTimeNanos, nowNanos + MILLISECONDS.toNanos(minimumInterCycleSleep)); } - if (expectedEndTime >= nextUpdatePerformanceTrackerFlushTime) { - nextUpdatePerformanceTrackerFlushTime = now + UpdatePerformanceTracker.REPORT_INTERVAL_MILLIS; + if (expectedEndTimeNanos >= nextUpdatePerformanceTrackerFlushTimeNanos) { + nextUpdatePerformanceTrackerFlushTimeNanos = + nowNanos + MILLISECONDS.toNanos(UpdatePerformanceTracker.REPORT_INTERVAL_MILLIS); try { updatePerformanceTracker.flush(); } catch (Exception err) { log.error().append("Error flushing UpdatePerformanceTracker: ").append(err).endl(); } } - waitForEndTime(expectedEndTime, timeSource); + waitForEndTime(expectedEndTimeNanos); } /** @@ -1819,12 +1824,11 @@ private void waitForNextCycle(final long startTime, final Scheduler timeSource) * If the delay is interrupted for any other {@link InterruptedException reason}, it will be logged and continue to * wait the remaining period. * - * @param expectedEndTime The time which we should sleep until - * @param timeSource The source of time that startTime was based on + * @param expectedEndTimeNanos The time (as reported by {@link System#nanoTime()}) which we should sleep until */ - private void waitForEndTime(final long expectedEndTime, final Scheduler timeSource) { - long remainingMillis; - while ((remainingMillis = expectedEndTime - timeSource.currentTimeMillis()) > 0) { + private void waitForEndTime(final long expectedEndTimeNanos) { + long remainingNanos; + while ((remainingNanos = expectedEndTimeNanos - System.nanoTime()) > 0) { if (refreshRequested.get()) { return; } @@ -1832,8 +1836,10 @@ private void waitForEndTime(final long expectedEndTime, final Scheduler timeSour if (refreshRequested.get()) { return; } + final long millisToWait = remainingNanos / 1_000_000; + final int extraNanosToWait = (int) (remainingNanos - (millisToWait * 1_000_000)); try { - refreshRequested.wait(remainingMillis); + refreshRequested.wait(millisToWait, extraNanosToWait); } catch (final InterruptedException logAndIgnore) { log.warn().append("Interrupted while waiting on refreshRequested. Ignoring: ").append(logAndIgnore) .endl(); @@ -2031,7 +2037,7 @@ public static final class Builder { Configuration.getInstance().getBooleanWithDefault(ALLOW_UNIT_TEST_MODE_PROP, false); private long targetCycleDurationMillis = Configuration.getInstance().getIntegerWithDefault(DEFAULT_TARGET_CYCLE_DURATION_MILLIS_PROP, 1000); - private long minimumCycleDurationToLogNanos = TimeUnit.MILLISECONDS.toNanos( + private long minimumCycleDurationToLogNanos = MILLISECONDS.toNanos( Configuration.getInstance().getIntegerWithDefault(MINIMUM_CYCLE_DURATION_TO_LOG_MILLIS_PROP, 25)); private String name; diff --git a/engine/table/src/main/java/io/deephaven/engine/util/file/TrackedFileHandleFactory.java b/engine/table/src/main/java/io/deephaven/engine/util/file/TrackedFileHandleFactory.java index 7d89b355700..18f23de97fd 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/file/TrackedFileHandleFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/file/TrackedFileHandleFactory.java @@ -3,13 +3,12 @@ */ package io.deephaven.engine.util.file; -import io.deephaven.net.CommBase; +import io.deephaven.UncheckedDeephavenException; import io.deephaven.base.verify.Require; import io.deephaven.configuration.Configuration; -import io.deephaven.io.logger.Logger; -import io.deephaven.io.sched.Scheduler; -import io.deephaven.io.sched.TimedJob; +import io.deephaven.util.thread.NamingThreadFactory; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.VisibleForTesting; import java.io.File; import java.io.IOException; @@ -17,8 +16,13 @@ import java.nio.channels.FileChannel; import java.nio.file.OpenOption; -import java.util.*; +import java.util.Iterator; +import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -26,7 +30,7 @@ * Simple least-recently-opened "cache" for FileHandles, to avoid running up against ulimits. Will probably not achieve * satisfactory results if the number of file handles concurrently in active use exceeds capacity. Note that returned * FileHandles may be closed asynchronously by the factory. - * + *

* TODO: Consider adding a lookup to enable handle sharing. Not necessary for current usage. */ public class TrackedFileHandleFactory implements FileHandleFactory { @@ -38,9 +42,16 @@ public static TrackedFileHandleFactory getInstance() { synchronized (TrackedFileHandleFactory.class) { if (instance == null) { instance = new TrackedFileHandleFactory( - CommBase.singleThreadedScheduler("TrackedFileHandleFactory.CleanupScheduler", Logger.NULL) - .start(), - Configuration.getInstance().getInteger("TrackedFileHandleFactory.maxOpenFiles")); + Executors.newSingleThreadScheduledExecutor( + new NamingThreadFactory(TrackedFileHandleFactory.class, "cleanupScheduler", true)), + Configuration.getInstance().getInteger("TrackedFileHandleFactory.maxOpenFiles")) { + + @Override + public void shutdown() { + super.shutdown(); + getScheduler().shutdown(); + } + }; } } } @@ -50,11 +61,12 @@ public static TrackedFileHandleFactory getInstance() { private final static double DEFAULT_TARGET_USAGE_RATIO = 0.9; private final static long DEFAULT_CLEANUP_INTERVAL_MILLIS = 60_000; - private final Scheduler scheduler; + private final ScheduledExecutorService scheduler; private final int capacity; private final double targetUsageRatio; private final int targetUsageThreshold; + private final ScheduledFuture cleanupJobFuture; private final AtomicInteger size = new AtomicInteger(0); private final Queue handleReferences = new ConcurrentLinkedQueue<>(); @@ -70,32 +82,39 @@ public static TrackedFileHandleFactory getInstance() { /** * Full constructor. * - * @param scheduler The scheduler to use for cleanup + * @param scheduler The {@link ScheduledExecutorService} to use for cleanup * @param capacity The total number of file handles to allow outstanding * @param targetUsageRatio The target usage threshold as a ratio of capacity, in [0.1, 0.9] * @param cleanupIntervalMillis The interval for asynchronous cleanup attempts */ - public TrackedFileHandleFactory(@NotNull final Scheduler scheduler, final int capacity, - final double targetUsageRatio, final long cleanupIntervalMillis) { + @VisibleForTesting + TrackedFileHandleFactory( + @NotNull final ScheduledExecutorService scheduler, + final int capacity, + final double targetUsageRatio, + final long cleanupIntervalMillis) { this.scheduler = scheduler; this.capacity = Require.gtZero(capacity, "capacity"); this.targetUsageRatio = Require.inRange(targetUsageRatio, 0.1, 0.9, "targetUsageRatio"); targetUsageThreshold = Require.gtZero((int) (capacity * targetUsageRatio), "targetUsageThreshold"); - new CleanupJob(cleanupIntervalMillis).schedule(); + cleanupJobFuture = scheduler.scheduleAtFixedRate( + new CleanupJob(), cleanupIntervalMillis, cleanupIntervalMillis, TimeUnit.MILLISECONDS); } /** * Constructor with default target usage ratio of 0.9 (90%) and cleanup attempts every 60 seconds. - * - * @param scheduler The scheduler to use for cleanup + * + * @param scheduler The {@link ScheduledExecutorService} to use for cleanup * @param capacity The total number of file handles to allow outstanding */ - public TrackedFileHandleFactory(@NotNull final Scheduler scheduler, final int capacity) { + @VisibleForTesting + TrackedFileHandleFactory(@NotNull final ScheduledExecutorService scheduler, final int capacity) { this(scheduler, capacity, DEFAULT_TARGET_USAGE_RATIO, DEFAULT_CLEANUP_INTERVAL_MILLIS); } - public Scheduler getScheduler() { + @VisibleForTesting + ScheduledExecutorService getScheduler() { return scheduler; } @@ -160,26 +179,18 @@ public void closeAll() { } } - private class CleanupJob extends TimedJob { - - private final long intervalMills; - - private CleanupJob(final long intervalMills) { - this.intervalMills = intervalMills; - } + public void shutdown() { + cleanupJobFuture.cancel(true); + } - private void schedule() { - scheduler.installJob(this, scheduler.currentTimeMillis() + intervalMills); - } + private class CleanupJob implements Runnable { - @Override - public void timedOut() { + public void run() { try { cleanup(); } catch (Exception e) { - throw new RuntimeException("TrackedFileHandleFactory.CleanupJob: Unexpected exception", e); + throw new UncheckedDeephavenException("TrackedFileHandleFactory.CleanupJob: Unexpected exception", e); } - schedule(); } } diff --git a/engine/table/src/test/java/io/deephaven/engine/util/file/TestTrackedFileHandleFactory.java b/engine/table/src/test/java/io/deephaven/engine/util/file/TestTrackedFileHandleFactory.java index 8d28daf9d7c..d1f832086c5 100644 --- a/engine/table/src/test/java/io/deephaven/engine/util/file/TestTrackedFileHandleFactory.java +++ b/engine/table/src/test/java/io/deephaven/engine/util/file/TestTrackedFileHandleFactory.java @@ -5,8 +5,6 @@ import io.deephaven.base.testing.BaseCachedJMockTestCase; import io.deephaven.base.verify.RequirementFailure; -import io.deephaven.io.sched.Scheduler; -import io.deephaven.io.sched.TimedJob; import junit.framework.TestCase; import org.junit.After; import org.junit.Before; @@ -15,6 +13,8 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; public class TestTrackedFileHandleFactory extends BaseCachedJMockTestCase { @@ -23,7 +23,7 @@ public class TestTrackedFileHandleFactory extends BaseCachedJMockTestCase { private static final double TARGET_USAGE_RATIO = 0.9; private static final int TARGET_USAGE_THRESHOLD = 90; - private Scheduler scheduler; + private ScheduledExecutorService scheduler; private TrackedFileHandleFactory FHCUT; @@ -33,13 +33,15 @@ public void setUp() throws Exception { FILE = Files.createTempFile(TestTrackedFileHandleFactory.class.getName(), ".dat").toFile(); - scheduler = mock(Scheduler.class); + scheduler = mock(ScheduledExecutorService.class); checking(new Expectations() { { - one(scheduler).currentTimeMillis(); - will(returnValue(0L)); - one(scheduler).installJob(with(any(TimedJob.class)), with(equal(60000L))); + one(scheduler).scheduleAtFixedRate( + with(any(Runnable.class)), + with(equal(60000L)), + with(equal(60000L)), + with(equal(TimeUnit.MILLISECONDS))); } }); diff --git a/engine/test-utils/build.gradle b/engine/test-utils/build.gradle index b11d14731e2..743e05a1dc5 100644 --- a/engine/test-utils/build.gradle +++ b/engine/test-utils/build.gradle @@ -13,7 +13,6 @@ dependencies { implementation project(':engine-tuple') implementation project(':base-test-utils') implementation project(':engine-rowset-test-utils') - implementation project(':FishUtil') implementation project(':extensions-source-support') implementation depCommonsLang3 diff --git a/engine/time/build.gradle b/engine/time/build.gradle index 697af28e26f..3dac58f277c 100644 --- a/engine/time/build.gradle +++ b/engine/time/build.gradle @@ -16,7 +16,6 @@ dependencies { implementation project(':engine-function') implementation project(':Configuration') implementation project(':log-factory') - implementation project(':FishUtil') implementation depJdom2 testImplementation TestTools.projectDependency(project, 'Base') diff --git a/engine/updategraph/build.gradle b/engine/updategraph/build.gradle index 1194f5ed0a9..69063b30bd0 100644 --- a/engine/updategraph/build.gradle +++ b/engine/updategraph/build.gradle @@ -12,8 +12,6 @@ dependencies { implementation project(':hotspot') implementation project(':log-factory') implementation project(':Configuration') - implementation project(':Net') - implementation project(':FishUtil') implementation depCommonsLang3 compileOnly 'com.google.code.findbugs:jsr305:3.0.2' diff --git a/engine/updategraph/src/main/java/io/deephaven/engine/liveness/Liveness.java b/engine/updategraph/src/main/java/io/deephaven/engine/liveness/Liveness.java index 9b6476ff4e8..f00655d5e14 100644 --- a/engine/updategraph/src/main/java/io/deephaven/engine/liveness/Liveness.java +++ b/engine/updategraph/src/main/java/io/deephaven/engine/liveness/Liveness.java @@ -5,14 +5,15 @@ import io.deephaven.configuration.Configuration; import io.deephaven.io.logger.Logger; -import io.deephaven.io.sched.Scheduler; -import io.deephaven.io.sched.TimedJob; import io.deephaven.engine.updategraph.DynamicNode; import io.deephaven.util.HeapDump; import io.deephaven.internal.log.LoggerFactory; import org.jetbrains.annotations.NotNull; import java.io.IOException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; /** * Utility class for liveness-related instrumentation. @@ -77,14 +78,18 @@ private static void maybeLogOutstandingCount() { intervalLastOutstandingCount = intervalMinOutstandingCount = intervalMaxOutstandingCount = outstandingCount; } - public static void scheduleCountReport(@NotNull final Scheduler scheduler) { - scheduler.installJob(new TimedJob() { - @Override - public final void timedOut() { - maybeLogOutstandingCount(); - scheduler.installJob(this, scheduler.currentTimeMillis() + OUTSTANDING_COUNT_LOG_INTERVAL_MILLIS); - } - }, 0L); + /** + * Schedule a job to log the count of known outstanding {@link LivenessReferent LivenessReferents}. + * + * @param scheduler The {@link ScheduledExecutorService} to use + * @return The {@link ScheduledFuture} for the scheduled job + */ + public static ScheduledFuture scheduleCountReport(@NotNull final ScheduledExecutorService scheduler) { + return scheduler.scheduleAtFixedRate( + Liveness::maybeLogOutstandingCount, + 0L, + OUTSTANDING_COUNT_LOG_INTERVAL_MILLIS, + TimeUnit.MILLISECONDS); } private Liveness() {} diff --git a/proto/raw-js-openapi/Dockerfile b/proto/raw-js-openapi/Dockerfile index 55cd2d950fb..9f39309d334 100644 --- a/proto/raw-js-openapi/Dockerfile +++ b/proto/raw-js-openapi/Dockerfile @@ -1,5 +1,3 @@ -FROM deephaven/proto-backplane-grpc:local-build AS proto-backplane-grpc - FROM deephaven/node:local-build WORKDIR /usr/src/app # Note: we are setting CI=true, even for local development, otherwise commands may run in dev-mode (ie, @@ -12,9 +10,6 @@ COPY . . RUN set -eux; \ npm ci --unsafe-perm -# TODO: this gets TS files which we don't need -COPY --from=proto-backplane-grpc generated/js raw-js-openapi/build/js-src - WORKDIR /usr/src/app/raw-js-openapi RUN set -eux; \ ../node_modules/.bin/webpack diff --git a/server/build.gradle b/server/build.gradle index 2996be8a521..2ab80c615d6 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -14,7 +14,6 @@ dependencies { implementation project(':extensions-jdbc') implementation project(':Util'); implementation project(':Integrations') - implementation project(':FishUtil') implementation depCommonsLang3 Classpaths.inheritCommonsText(project, 'implementation') diff --git a/settings.gradle b/settings.gradle index f1474fcba81..587ce55a9d7 100644 --- a/settings.gradle +++ b/settings.gradle @@ -137,10 +137,6 @@ include(':DataStructures') include(':Configuration') -include(':FishUtil') - -include(':Net') - include(':Stats') include(':Container')