diff --git a/engine/function/src/templates/Numeric.ftl b/engine/function/src/templates/Numeric.ftl index ccedc0d79d0..8710a195b83 100644 --- a/engine/function/src/templates/Numeric.ftl +++ b/engine/function/src/templates/Numeric.ftl @@ -14,6 +14,7 @@ import java.util.Arrays; import static io.deephaven.base.CompareUtils.compare; import static io.deephaven.util.QueryConstants.*; import static io.deephaven.function.Basic.*; +import static io.deephaven.function.Sort.*; import static io.deephaven.function.Cast.castDouble; /** @@ -364,20 +365,29 @@ public class Numeric { double sum = 0; double count = 0; + long nullCount = 0; try ( final ${pt.vectorIterator} vi = values.iterator() ) { while ( vi.hasNext() ) { final ${pt.primitive} c = vi.${pt.iteratorNext}(); + <#if pt.valueType.isFloat > if (isNaN(c)) { return Double.NaN; } + if (!isNull(c)) { sum += c; count++; + } else { + nullCount++; } } } + if (nullCount == values.size()) { + return NULL_DOUBLE; + } + return sum / count; } @@ -418,23 +428,32 @@ public class Numeric { double sum = 0; double count = 0; + long nullCount = 0; try ( final ${pt.vectorIterator} vi = values.iterator() ) { while ( vi.hasNext() ) { final ${pt.primitive} c = vi.${pt.iteratorNext}(); + <#if pt.valueType.isFloat > if (isNaN(c)) { return Double.NaN; } if (isInf(c)) { return Double.POSITIVE_INFINITY; } + if (!isNull(c)) { sum += Math.abs(c); count++; + } else { + nullCount++; } } } + if (nullCount == values.size()) { + return NULL_DOUBLE; + } + return sum / count; } @@ -485,20 +504,29 @@ public class Numeric { double sum = 0; double sum2 = 0; long count = 0; + long nullCount = 0; try ( final ${pt.vectorIterator} vi = values.iterator() ) { while ( vi.hasNext() ) { final ${pt.primitive} c = vi.${pt.iteratorNext}(); + <#if pt.valueType.isFloat > if (isNaN(c) || isInf(c)) { return Double.NaN; } + if (!isNull(c)) { sum += (double)c; sum2 += (double)c * (double)c; count++; + } else { + nullCount++; } } } + if (nullCount == values.size()) { + return NULL_DOUBLE; + } + // Return NaN if overflow or too few values to compute variance. if (count <= 1 || Double.isInfinite(sum) || Double.isInfinite(sum2)) { return Double.NaN; @@ -596,6 +624,8 @@ public class Numeric { double sum2 = 0; double count = 0; double count2 = 0; + long nullCount = 0; + long valueCount = 0; try ( final ${pt.vectorIterator} vi = values.iterator(); @@ -604,23 +634,34 @@ public class Numeric { while (vi.hasNext()) { final ${pt.primitive} c = vi.${pt.iteratorNext}(); final ${pt2.primitive} w = wi.${pt2.iteratorNext}(); + <#if pt.valueType.isFloat > if (isNaN(c) || isInf(c)) { return Double.NaN; } + + <#if pt2.valueType.isFloat > if (isNaN(w) || isInf(w)) { return Double.NaN; } + if (!isNull(c) && !isNull(w)) { sum += w * c; sum2 += w * c * c; count += w; count2 += w * w; + valueCount++; + } else { + nullCount++; } } } + if (nullCount == values.size()) { + return NULL_DOUBLE; + } + // Return NaN if overflow or too few values to compute variance. - if (count <= 1 || Double.isInfinite(sum) || Double.isInfinite(sum2)) { + if (valueCount <= 1 || Double.isInfinite(sum) || Double.isInfinite(sum2)) { return Double.NaN; } @@ -868,6 +909,11 @@ public class Numeric { throw new IllegalArgumentException("Incompatible input sizes: " + values.size() + ", " + weights.size()); } + final double s = wstd(values, weights); + if (s == NULL_DOUBLE) { + return NULL_DOUBLE; + } + // see https://stats.stackexchange.com/questions/25895/computing-standard-error-in-weighted-mean-estimation double sumw = 0; double sumw2 = 0; @@ -887,8 +933,7 @@ public class Numeric { } } - final double s = wstd(values, weights); - return s == NULL_DOUBLE ? NULL_DOUBLE : s * Math.sqrt(sumw2/sumw/sumw); + return s * Math.sqrt(sumw2/sumw/sumw); } @@ -996,7 +1041,15 @@ public class Numeric { } final double a = wavg(values, weights); + if (a == NULL_DOUBLE) { + return NULL_DOUBLE; + } + final double s = wste(values, weights); + if (s == NULL_DOUBLE) { + return NULL_DOUBLE; + } + return a / s; } @@ -1189,7 +1242,8 @@ public class Numeric { } /** - * Returns the median. + * Returns the median. {@code null} input values are ignored but {@code NaN} values will poison the computation, + * and {@code NaN} will be returned * * @param values values. * @return median. @@ -1199,7 +1253,8 @@ public class Numeric { } /** - * Returns the median. + * Returns the median. {@code null} input values are ignored but {@code NaN} values will poison the computation, + * and {@code NaN} will be returned * * @param values values. * @return median. @@ -1213,7 +1268,8 @@ public class Numeric { } /** - * Returns the median. + * Returns the median. {@code null} input values are ignored but {@code NaN} values will poison the computation, + * and {@code NaN} will be returned * * @param values values. * @return median. @@ -1226,18 +1282,71 @@ public class Numeric { int n = values.intSize("median"); if (n == 0) { - return Double.NaN; - } else { - ${pt.primitive}[] copy = values.copyToArray(); - Arrays.sort(copy); - if (n % 2 == 0) - return 0.5 * (copy[n / 2 - 1] + copy[n / 2]); - else return copy[n / 2]; + return NULL_DOUBLE; + } + + ${pt.primitive}[] sorted = values.copyToArray(); + Arrays.sort(sorted); + + <#if pt.valueType.isFloat > + if (isNaN(sorted[sorted.length - 1])) { + return Double.NaN; // Any NaN will pollute the result and NaN always sorted to the end. + } + + int nullStart = -1; + int nullCount = 0; + for (int i = 0; i < n; i++) { + final ${pt.primitive} val = sorted[i]; + if (val > ${pt.null}) { + break; // no more NULL possible + } + if (isNull(val)) { + nullCount++; + if (nullStart == -1) { + nullStart = i; + } + } + } + <#else> + int nullCount = 0; + for (int i = 0; i < n && isNull(sorted[i]); i++) { + nullCount++; } + + + if (nullCount == n) { + return NULL_DOUBLE; + } + if (nullCount > 0) { + n -= nullCount; + final int medianIndex = n / 2; + if (n % 2 == 0) { + <#if pt.valueType.isFloat > + final int idx1 = (medianIndex - 1) < nullStart ? (medianIndex - 1) : (medianIndex - 1) + nullCount; + final int idx2 = medianIndex < nullStart ? medianIndex : medianIndex + nullCount; + <#else> + final int idx1 = (medianIndex - 1) + nullCount; + final int idx2 = medianIndex + nullCount; + + return 0.5 * (sorted[idx1] + sorted[idx2]); + } + <#if pt.valueType.isFloat > + final int adjustedIndex = medianIndex < nullStart ? medianIndex : medianIndex + nullCount; + <#else> + final int adjustedIndex = medianIndex + nullCount; + + return sorted[adjustedIndex]; + } + final int medianIndex = n / 2; + if (n % 2 == 0) { + return 0.5 * (sorted[medianIndex - 1] + sorted[medianIndex]); + } + return sorted[medianIndex]; } /** - * Returns the percentile. + * Returns the percentile. {@code null} input values are ignored but {@code NaN} values will poison the computation, + * and {@code NaN} will be returned * * @param percentile percentile to compute. * @param values values. @@ -1252,7 +1361,8 @@ public class Numeric { } /** - * Returns the percentile. + * Returns the percentile. {@code null} input values are ignored but {@code NaN} values will poison the computation, + * and {@code NaN} will be returned * * @param percentile percentile to compute. * @param values values. @@ -1268,11 +1378,51 @@ public class Numeric { } int n = values.intSize("percentile"); - ${pt.primitive}[] copy = values.copyToArray(); - Arrays.sort(copy); + ${pt.primitive}[] sorted = values.copyToArray(); + Arrays.sort(sorted); + + <#if pt.valueType.isFloat > + if (isNaN(sorted[sorted.length - 1])) { + return ${pt.boxed}.NaN; // Any NaN will pollute the result and NaN always sorted to the end. + } + int nullStart = -1; + int nullCount = 0; + for (int i = 0; i < n; i++) { + final ${pt.primitive} val = sorted[i]; + if (val > ${pt.null}) { + break; // no more NULL possible + } + if (isNull(val)) { + nullCount++; + if (nullStart == -1) { + nullStart = i; + } + } + } + <#else> + int nullCount = 0; + for (int i = 0; i < n && isNull(sorted[i]); i++) { + nullCount++; + } + + + if (nullCount == n) { + return ${pt.null}; + } + if (nullCount > 0) { + n -= nullCount; + <#if pt.valueType.isFloat > + final int idx = (int) Math.round(percentile * (n - 1)); + final int adjustedIndex = idx < nullStart ? idx : idx + nullCount; + return sorted[adjustedIndex]; + <#else> + int idx = (int) Math.round(percentile * (n - 1)); + return sorted[idx + nullCount]; + + } int idx = (int) Math.round(percentile * (n - 1)); - return copy[idx]; + return sorted[idx]; } @@ -1344,6 +1494,7 @@ public class Numeric { double sum1 = 0; double sum01 = 0; double count = 0; + long nullCount = 0; try ( final ${pt.vectorIterator} v0i = values0.iterator(); @@ -1352,22 +1503,32 @@ public class Numeric { while (v0i.hasNext()) { final ${pt.primitive} v0 = v0i.${pt.iteratorNext}(); final ${pt2.primitive} v1 = v1i.${pt2.iteratorNext}(); + <#if pt.valueType.isFloat > if (isNaN(v0) || isInf(v0)) { return Double.NaN; } + + <#if pt2.valueType.isFloat > if (isNaN(v1) || isInf(v1)) { return Double.NaN; } + if (!isNull(v0) && !isNull(v1)) { sum0 += v0; sum1 += v1; sum01 += v0 * v1; count++; + } else { + nullCount++; } } } + if (nullCount == values0.size()) { + return NULL_DOUBLE; + } + return sum01 / count - sum0 * sum1 / count / count; } @@ -1438,6 +1599,7 @@ public class Numeric { double sum1Sq = 0; double sum01 = 0; double count = 0; + long nullCount = 0; try ( final ${pt.vectorIterator} v0i = values0.iterator(); @@ -1446,12 +1608,16 @@ public class Numeric { while (v0i.hasNext()) { final ${pt.primitive} v0 = v0i.${pt.iteratorNext}(); final ${pt2.primitive} v1 = v1i.${pt2.iteratorNext}(); + <#if pt.valueType.isFloat > if (isNaN(v0) || isInf(v0)) { return Double.NaN; } + + <#if pt2.valueType.isFloat > if (isNaN(v1) || isInf(v1)) { return Double.NaN; } + if (!isNull(v0) && !isNull(v1)) { sum0 += v0; @@ -1460,10 +1626,16 @@ public class Numeric { sum1Sq += v1 * v1; sum01 += v0 * v1; count++; + } else { + nullCount++; } } } + if (nullCount == values0.size()) { + return NULL_DOUBLE; + } + double cov = sum01 / count - sum0 * sum1 / count / count; double var0 = sum0Sq / count - sum0 * sum0 / count / count; double var1 = sum1Sq / count - sum1 * sum1 / count / count; @@ -2756,12 +2928,16 @@ public class Numeric { while (vi.hasNext()) { final ${pt.primitive} c = vi.${pt.iteratorNext}(); final ${pt2.primitive} w = wi.${pt2.iteratorNext}(); + <#if pt.valueType.isFloat > if (isNaN(c)) { return Double.NaN; } + + <#if pt2.valueType.isFloat > if (isNaN(w)) { return Double.NaN; } + if (!isNull(c) && !isNull(w)) { vsum += c * w; wsum += w; diff --git a/engine/function/src/templates/TestNumeric.ftl b/engine/function/src/templates/TestNumeric.ftl index 1805eb3a396..5e51d5445cd 100644 --- a/engine/function/src/templates/TestNumeric.ftl +++ b/engine/function/src/templates/TestNumeric.ftl @@ -91,25 +91,34 @@ public class TestNumeric extends BaseArrayTestCase { public void test${pt.boxed}Avg() { assertEquals(50.0, avg(new ${pt.primitive}[]{40, 50, 60})); assertEquals(45.5, avg(new ${pt.primitive}[]{40, 51})); - assertTrue(Double.isNaN(avg(new ${pt.primitive}[]{}))); - assertTrue(Double.isNaN(avg(new ${pt.primitive}[]{${pt.null}}))); + assertEquals(NULL_DOUBLE, avg(new ${pt.primitive}[]{})); + assertEquals(NULL_DOUBLE, avg(new ${pt.primitive}[]{${pt.null}})); assertEquals(10.0, avg(new ${pt.primitive}[]{5, ${pt.null}, 15})); assertEquals(NULL_DOUBLE, avg((${pt.primitive}[])null)); assertEquals(50.0, avg(new ${pt.boxed}[]{(${pt.primitive})40, (${pt.primitive})50, (${pt.primitive})60})); assertEquals(45.5, avg(new ${pt.boxed}[]{(${pt.primitive})40, (${pt.primitive})51})); - assertTrue(Double.isNaN(avg(new ${pt.boxed}[]{}))); - assertTrue(Double.isNaN(avg(new ${pt.boxed}[]{${pt.null}}))); + assertEquals(NULL_DOUBLE, avg(new ${pt.boxed}[]{})); + assertEquals(NULL_DOUBLE, avg(new ${pt.boxed}[]{${pt.null}})); assertEquals(10.0, avg(new ${pt.boxed}[]{(${pt.primitive})5, ${pt.null}, (${pt.primitive})15})); assertEquals(NULL_DOUBLE, avg((${pt.boxed}[])null)); assertEquals(50.0, avg(new ${pt.vectorDirect}(new ${pt.primitive}[]{40, 50, 60}))); assertEquals(45.5, avg(new ${pt.vectorDirect}(new ${pt.primitive}[]{40, 51}))); - assertTrue(Double.isNaN(avg(new ${pt.vectorDirect}()))); - assertTrue(Double.isNaN(avg(new ${pt.vectorDirect}(${pt.null})))); + assertEquals(NULL_DOUBLE, avg(new ${pt.vectorDirect}())); + assertEquals(NULL_DOUBLE, avg(new ${pt.vectorDirect}(${pt.null}))); assertEquals(10.0, avg(new ${pt.vectorDirect}(new ${pt.primitive}[]{5, ${pt.null}, 15}))); assertEquals(NULL_DOUBLE, avg((${pt.vectorDirect})null)); + // verify the all-null case returns null + assertEquals(NULL_DOUBLE, avg(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}})); + assertEquals(NULL_DOUBLE, avg(new ${pt.boxed}[]{${pt.null}, ${pt.null}, ${pt.null}})); + + <#if pt.valueType.isFloat > + // verify the NaN short-circuit case + assertEquals(Double.NaN, avg(new ${pt.primitive}[]{40, ${pt.boxed}.NaN, 60})); + + // check that functions can be resolved with varargs assertEquals(45.0, avg((${pt.primitive})40, (${pt.primitive})50)); } @@ -117,25 +126,29 @@ public class TestNumeric extends BaseArrayTestCase { public void test${pt.boxed}AbsAvg() { assertEquals(50.0, absAvg(new ${pt.primitive}[]{40, (${pt.primitive}) 50, 60})); assertEquals(45.5, absAvg(new ${pt.primitive}[]{(${pt.primitive}) 40, 51})); - assertTrue(Double.isNaN(absAvg(new ${pt.primitive}[]{}))); - assertTrue(Double.isNaN(absAvg(new ${pt.primitive}[]{${pt.null}}))); + assertEquals(NULL_DOUBLE, absAvg(new ${pt.primitive}[]{})); + assertEquals(NULL_DOUBLE, absAvg(new ${pt.primitive}[]{${pt.null}})); assertEquals(10.0, absAvg(new ${pt.primitive}[]{(${pt.primitive}) 5, ${pt.null}, (${pt.primitive}) 15})); assertEquals(NULL_DOUBLE, absAvg((${pt.primitive}[])null)); assertEquals(50.0, absAvg(new ${pt.boxed}[]{(${pt.primitive})40, (${pt.primitive}) 50, (${pt.primitive})60})); assertEquals(45.5, absAvg(new ${pt.boxed}[]{(${pt.primitive}) 40, (${pt.primitive})51})); - assertTrue(Double.isNaN(absAvg(new ${pt.boxed}[]{}))); - assertTrue(Double.isNaN(absAvg(new ${pt.boxed}[]{${pt.null}}))); + assertEquals(NULL_DOUBLE, absAvg(new ${pt.boxed}[]{})); + assertEquals(NULL_DOUBLE, absAvg(new ${pt.boxed}[]{${pt.null}})); assertEquals(10.0, absAvg(new ${pt.boxed}[]{(${pt.primitive}) 5, ${pt.null}, (${pt.primitive}) 15})); assertEquals(NULL_DOUBLE, absAvg((${pt.boxed}[])null)); assertEquals(50.0, absAvg(new ${pt.vectorDirect}(new ${pt.primitive}[]{40, (${pt.primitive}) 50, 60}))); assertEquals(45.5, absAvg(new ${pt.vectorDirect}(new ${pt.primitive}[]{(${pt.primitive}) 40, 51}))); - assertTrue(Double.isNaN(absAvg(new ${pt.vectorDirect}()))); - assertTrue(Double.isNaN(absAvg(new ${pt.vectorDirect}(${pt.null})))); + assertEquals(NULL_DOUBLE, absAvg(new ${pt.vectorDirect}())); + assertEquals(NULL_DOUBLE, absAvg(new ${pt.vectorDirect}(${pt.null}))); assertEquals(10.0, absAvg(new ${pt.vectorDirect}((${pt.primitive}) 5, ${pt.null}, (${pt.primitive}) 15))); assertEquals(NULL_DOUBLE, absAvg((${pt.vectorDirect})null)); + // verify the all-null case returns null + assertEquals(NULL_DOUBLE, absAvg(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}})); + assertEquals(NULL_DOUBLE, absAvg(new ${pt.boxed}[]{${pt.null}, ${pt.null}, ${pt.null}})); + // check that functions can be resolved with varargs assertEquals(45.0, absAvg((${pt.primitive})40, (${pt.primitive})50)); } @@ -309,6 +322,18 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(var, var(new ${pt.vectorDirect}(v))); assertEquals(NULL_DOUBLE, var((${pt.vectorDirect})null)); + // verify the all-null case returns null + assertEquals(NULL_DOUBLE, var(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}})); + assertEquals(NULL_DOUBLE, var(new ${pt.boxed}[]{${pt.null}, ${pt.null}, ${pt.null}})); + + // verify size==1 + assertEquals(Double.NaN, var(new ${pt.primitive}[]{40})); + + <#if pt.valueType.isFloat > + // verify the NaN short-circuit case + assertEquals(Double.NaN, var(new ${pt.primitive}[]{40, ${pt.boxed}.NaN, 60})); + + // check that functions can be resolved with varargs assertEquals(var, var((${pt.primitive})0, (${pt.primitive})40, ${pt.null}, (${pt.primitive})50, (${pt.primitive})60, (${pt.primitive}) -1, (${pt.primitive})0)); } @@ -326,6 +351,10 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(Math.sqrt(var(new ${pt.vectorDirect}(v))), std(new ${pt.vectorDirect}(v))); assertEquals(NULL_DOUBLE, std((${pt.vectorDirect})null)); + // verify the all-null case returns null + assertEquals(NULL_DOUBLE, std(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}})); + assertEquals(NULL_DOUBLE, std(new ${pt.boxed}[]{${pt.null}, ${pt.null}, ${pt.null}})); + // check that functions can be resolved with varargs assertEquals(std(v), std((${pt.primitive})0, (${pt.primitive})40, ${pt.null}, (${pt.primitive})50, (${pt.primitive})60, (${pt.primitive}) -1, (${pt.primitive})0)); } @@ -343,6 +372,10 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(std(new ${pt.vectorDirect}(v)) / Math.sqrt(count(new ${pt.vectorDirect}(v))), ste(new ${pt.vectorDirect}(v))); assertEquals(NULL_DOUBLE, ste((${pt.vectorDirect})null)); + // verify the all-null case returns null + assertEquals(NULL_DOUBLE, ste(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}})); + assertEquals(NULL_DOUBLE, ste(new ${pt.boxed}[]{${pt.null}, ${pt.null}, ${pt.null}})); + // check that functions can be resolved with varargs assertEquals(ste(v), ste((${pt.primitive})0, (${pt.primitive})40, ${pt.null}, (${pt.primitive})50, (${pt.primitive})60, (${pt.primitive}) -1, (${pt.primitive})0)); } @@ -360,6 +393,10 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(avg(new ${pt.vectorDirect}(v)) / ste(new ${pt.vectorDirect}(v)), tstat(new ${pt.vectorDirect}(v))); assertEquals(NULL_DOUBLE, tstat((${pt.vectorDirect})null)); + // verify the all-null case returns null + assertEquals(NULL_DOUBLE, tstat(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}})); + assertEquals(NULL_DOUBLE, tstat(new ${pt.boxed}[]{${pt.null}, ${pt.null}, ${pt.null}})); + // check that functions can be resolved with varargs assertEquals(tstat(v), tstat((${pt.primitive})0, (${pt.primitive})40, ${pt.null}, (${pt.primitive})50, (${pt.primitive})60, (${pt.primitive}) -1, (${pt.primitive})0)); } @@ -509,6 +546,19 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(NULL_DOUBLE, cov((${pt.vectorDirect})null, new ${pt2.vectorDirect}(b))); assertEquals(NULL_DOUBLE, cov((${pt.vectorDirect})null, (${pt2.vectorDirect})null)); + // verify the all-null cases return null + assertEquals(NULL_DOUBLE, cov(new ${pt.primitive}[]{1, 2, 3}, new ${pt2.primitive}[]{${pt2.null}, ${pt2.null}, ${pt2.null}})); + assertEquals(NULL_DOUBLE, cov(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}}, new ${pt2.primitive}[]{1, 2, 3})); + assertEquals(NULL_DOUBLE, cov(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}}, new ${pt2.primitive}[]{${pt2.null}, ${pt2.null}, ${pt2.null}})); + + <#if pt.valueType.isFloat > + // verify the NaN short-circuit case + assertEquals(Double.NaN, cov(new ${pt.primitive}[]{1, 2, ${pt.boxed}.NaN}, new ${pt2.primitive}[]{1, 2, 3})); + + <#if pt2.valueType.isFloat > + // verify the NaN short-circuit case + assertEquals(Double.NaN, cov(new ${pt.primitive}[]{1, 2, 3}, new ${pt2.primitive}[]{1, 2, ${pt2.boxed}.NaN})); + try { cov(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3,${pt.null},5}), new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5})); @@ -552,6 +602,20 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(NULL_DOUBLE, cor((${pt.vectorDirect})null, new ${pt2.vectorDirect}(b))); assertEquals(NULL_DOUBLE, cor((${pt.vectorDirect})null, (${pt2.vectorDirect})null)); + // verify the all-null cases return null + assertEquals(NULL_DOUBLE, cor(new ${pt.primitive}[]{1, 2, 3}, new ${pt2.primitive}[]{${pt2.null}, ${pt2.null}, ${pt2.null}})); + assertEquals(NULL_DOUBLE, cor(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}}, new ${pt2.primitive}[]{1, 2, 3})); + assertEquals(NULL_DOUBLE, cor(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}}, new ${pt2.primitive}[]{${pt2.null}, ${pt2.null}, ${pt2.null}})); + + <#if pt.valueType.isFloat > + // verify the NaN short-circuit case + assertEquals(Double.NaN, cor(new ${pt.primitive}[]{1, 2, ${pt.boxed}.NaN}, new ${pt2.primitive}[]{1, 2, 3})); + + <#if pt2.valueType.isFloat > + // verify the NaN short-circuit case + assertEquals(Double.NaN, cor(new ${pt.primitive}[]{1, 2, 3}, new ${pt2.primitive}[]{1, 2, ${pt2.boxed}.NaN})); + + try { cor(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3,${pt.null},5}), new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5})); fail("Mismatched arguments"); @@ -1092,7 +1156,7 @@ public class TestNumeric extends BaseArrayTestCase { } public void test${pt.boxed}Median() { - assertEquals(Double.NaN, median(new ${pt.primitive}[]{})); + assertEquals(NULL_DOUBLE, median(new ${pt.primitive}[]{})); assertEquals(3.0, median(new ${pt.primitive}[]{4,2,3})); assertEquals(3.5, median(new ${pt.primitive}[]{5,4,2,3})); @@ -1106,6 +1170,23 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(3.5, median(new ${pt.vectorDirect}(new ${pt.primitive}[]{5,4,2,3}))); assertEquals(NULL_DOUBLE, median((${pt.vector}) null)); + // verify the all-null case returns null + assertEquals(NULL_DOUBLE, median(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}})); + assertEquals(NULL_DOUBLE, median(new ${pt.boxed}[]{${pt.null}, ${pt.null}, ${pt.null}})); + + // verify the mixed-null cases + assertEquals(3.0, median(new ${pt.primitive}[]{(${pt.primitive})4,(${pt.primitive})2,(${pt.primitive})3,${pt.null},${pt.null},${pt.null}})); + assertEquals(3.5, median(new ${pt.primitive}[]{(${pt.primitive})4,(${pt.primitive})2,(${pt.primitive})3,(${pt.primitive})5, ${pt.null},${pt.null},${pt.null}})); + + assertEquals(3.0, median(new ${pt.boxed}[]{(${pt.primitive})4,(${pt.primitive})2,(${pt.primitive})3,${pt.null},${pt.null}})); + assertEquals(3.5, median(new ${pt.boxed}[]{(${pt.primitive})4,(${pt.primitive})2,(${pt.primitive})3,(${pt.primitive})5,${pt.null},${pt.null}})); + + <#if pt.valueType.isFloat > + assertEquals(Double.NaN, median(new ${pt.primitive}[]{4,2,3, ${pt.boxed}.NaN})); + assertEquals(3.0, median(new ${pt.primitive}[]{4,2,3, ${pt.boxed}.POSITIVE_INFINITY, ${pt.null}, ${pt.null}, ${pt.boxed}.NEGATIVE_INFINITY})); + assertEquals(3.5, median(new ${pt.primitive}[]{4,2,3,5, ${pt.boxed}.POSITIVE_INFINITY, ${pt.null}, ${pt.null}, ${pt.boxed}.NEGATIVE_INFINITY})); + + // check that functions can be resolved with varargs assertEquals(3.0, median((${pt.primitive})4, (${pt.primitive})2, (${pt.primitive})3)); } @@ -1135,6 +1216,28 @@ public class TestNumeric extends BaseArrayTestCase { // pass } + // verify the all-null case returns null + assertEquals(${pt.null}, percentile(0.00, new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}})); + assertEquals(${pt.null}, percentile(0.25, new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}})); + assertEquals(${pt.null}, percentile(0.50, new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}})); + + // verify the mixed-null cases + assertEquals((${pt.primitive})2, percentile(0.00, new ${pt.primitive}[]{4,2,3,${pt.null}})); + assertEquals((${pt.primitive})3, percentile(0.50, new ${pt.primitive}[]{4,2,3,${pt.null},${pt.null}})); + assertEquals((${pt.primitive})4, percentile(1.0, new ${pt.primitive}[]{4,2,3,${pt.null},${pt.null},${pt.null}})); + + // verify the empty array case + assertEquals(${pt.null}, percentile(0.00, new ${pt.vectorDirect}(new ${pt.primitive}[]{}))); + + <#if pt.valueType.isFloat > + assertEquals(${pt.boxed}.NaN, percentile(1.0, new ${pt.primitive}[]{4,2,3, ${pt.boxed}.NaN})); + + assertEquals(${pt.boxed}.NEGATIVE_INFINITY, percentile(0.0, new ${pt.primitive}[]{4,2,3, ${pt.boxed}.POSITIVE_INFINITY, ${pt.null}, ${pt.null}, ${pt.boxed}.NEGATIVE_INFINITY})); + assertEquals((${pt.primitive})2, percentile(0.25, new ${pt.primitive}[]{4,2,3, ${pt.boxed}.POSITIVE_INFINITY, ${pt.null}, ${pt.null}, ${pt.boxed}.NEGATIVE_INFINITY})); + assertEquals((${pt.primitive})3, percentile(0.5, new ${pt.primitive}[]{4,2,3, ${pt.boxed}.POSITIVE_INFINITY, ${pt.null}, ${pt.null}, ${pt.boxed}.NEGATIVE_INFINITY})); + assertEquals((${pt.primitive})4, percentile(0.75, new ${pt.primitive}[]{4,2,3, ${pt.boxed}.POSITIVE_INFINITY, ${pt.null}, ${pt.null}, ${pt.boxed}.NEGATIVE_INFINITY})); + assertEquals(${pt.boxed}.POSITIVE_INFINITY, percentile(1.0, new ${pt.primitive}[]{4,2,3, ${pt.boxed}.POSITIVE_INFINITY, ${pt.null}, ${pt.null}, ${pt.boxed}.NEGATIVE_INFINITY})); + } public void test${pt.boxed}Wsum() { @@ -1286,6 +1389,21 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(var(new ${pt.primitive}[]{1,2,3}), wvar(new ${pt.primitive}[]{1,2,3,${pt.null},5}, new ${pt2.primitive}[]{1,1,1,7,${pt2.null}})); assertEquals(var(new ${pt.primitive}[]{1,2,3}), wvar(new ${pt.primitive}[]{1,2,3,${pt.null},5}, new ${pt2.primitive}[]{2,2,2,7,${pt2.null}})); + // verify the all-null cases return null + assertEquals(NULL_DOUBLE, wvar(new ${pt.primitive}[]{1, 2, 3}, new ${pt2.primitive}[]{${pt2.null}, ${pt2.null}, ${pt2.null}})); + assertEquals(NULL_DOUBLE, wvar(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}}, new ${pt2.primitive}[]{1, 2, 3})); + assertEquals(NULL_DOUBLE, wvar(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}}, new ${pt2.primitive}[]{${pt2.null}, ${pt2.null}, ${pt2.null}})); + + // verify size==1 + assertEquals(Double.NaN, wvar(new ${pt.primitive}[]{1}, new ${pt2.primitive}[]{4})); + + <#if pt2.valueType.isFloat > + // verify NaN poisoning + assertEquals(Double.NaN, wvar(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3,${pt.null},5}), new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5,6,Float.NaN,${pt2.null}}))); + + + // verify the zero-weight case returns null + assertEquals(Double.NaN, wvar(new ${pt.primitive}[]{1, 2, 3}, new ${pt2.primitive}[]{0, 0, 0})); } @@ -1316,6 +1434,10 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(NULL_DOUBLE, wstd((${pt.vector}) null, new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5,6}))); assertEquals(NULL_DOUBLE, wstd(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3}), (${pt2.vector}) null)); + // verify the all-null cases return null + assertEquals(NULL_DOUBLE, wstd(new ${pt.primitive}[]{1, 2, 3}, new ${pt2.primitive}[]{${pt2.null}, ${pt2.null}, ${pt2.null}})); + assertEquals(NULL_DOUBLE, wstd(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}}, new ${pt2.primitive}[]{1, 2, 3})); + assertEquals(NULL_DOUBLE, wstd(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}}, new ${pt2.primitive}[]{${pt2.null}, ${pt2.null}, ${pt2.null}})); } @@ -1355,6 +1477,10 @@ public class TestNumeric extends BaseArrayTestCase { // pass } + // verify the all-null cases return null + assertEquals(NULL_DOUBLE, wste(new ${pt.primitive}[]{1, 2, 3}, new ${pt2.primitive}[]{${pt2.null}, ${pt2.null}, ${pt2.null}})); + assertEquals(NULL_DOUBLE, wste(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}}, new ${pt2.primitive}[]{1, 2, 3})); + assertEquals(NULL_DOUBLE, wste(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}}, new ${pt2.primitive}[]{${pt2.null}, ${pt2.null}, ${pt2.null}})); @@ -1382,6 +1508,11 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(NULL_DOUBLE, wtstat((${pt.vector}) null, new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5,6}))); assertEquals(NULL_DOUBLE, wtstat(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3}), (${pt2.vector}) null)); + // verify the all-null cases return null + assertEquals(NULL_DOUBLE, wtstat(new ${pt.primitive}[]{1, 2, 3}, new ${pt2.primitive}[]{${pt2.null}, ${pt2.null}, ${pt2.null}})); + assertEquals(NULL_DOUBLE, wtstat(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}}, new ${pt2.primitive}[]{1, 2, 3})); + assertEquals(NULL_DOUBLE, wtstat(new ${pt.primitive}[]{${pt.null}, ${pt.null}, ${pt.null}}, new ${pt2.primitive}[]{${pt2.null}, ${pt2.null}, ${pt2.null}})); + } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/ByteChunkedAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/ByteChunkedAvgOperator.java index ac8a08af83c..34fdf0b5392 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/ByteChunkedAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/ByteChunkedAvgOperator.java @@ -24,6 +24,7 @@ import static io.deephaven.engine.table.impl.by.RollupConstants.*; import static io.deephaven.engine.util.NullSafeAddition.plusLong; import static io.deephaven.engine.util.NullSafeAddition.minusLong; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; /** * Iterative average operator. @@ -92,7 +93,7 @@ private boolean addChunk(ByteChunk values, long destination, i runningSum.set(destination, newSum); resultColumn.set(destination, (double) newSum / newCount); } else if (nonNullCount.onlyNullsUnsafe(destination)) { - resultColumn.set(destination, Double.NaN); + resultColumn.set(destination, NULL_DOUBLE); } else { return false; } @@ -110,8 +111,11 @@ private boolean removeChunk(ByteChunk values, long destination final long newCount = nonNullCount.addNonNullUnsafe(destination, -chunkNonNull.get()); final long newSum = minusLong(runningSum.getUnsafe(destination), chunkSum); runningSum.set(destination, newSum); - resultColumn.set(destination, (double) newSum / newCount); - + if (newCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + } else { + resultColumn.set(destination, (double) newSum / newCount); + } return true; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/ByteChunkedVarOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/ByteChunkedVarOperator.java index bedc7465bb8..a2f8f971ba9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/ByteChunkedVarOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/ByteChunkedVarOperator.java @@ -7,6 +7,7 @@ // @formatter:off package io.deephaven.engine.table.impl.by; +import io.deephaven.base.verify.Assert; import io.deephaven.chunk.attributes.ChunkLengths; import io.deephaven.chunk.attributes.ChunkPositions; import io.deephaven.chunk.attributes.Values; @@ -23,6 +24,7 @@ import static io.deephaven.engine.table.impl.by.RollupConstants.*; import static io.deephaven.engine.util.NullSafeAddition.plusDouble; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; /** * Iterative variance operator. @@ -86,21 +88,27 @@ private boolean addChunk(ByteChunk values, long destination, i final double sum = SumByteChunk.sum2ByteChunk(values, chunkStart, chunkSize, chunkNonNull, sum2); if (chunkNonNull.get() > 0) { - final long nonNullCount = nonNullCounter.addNonNullUnsafe(destination, chunkNonNull.get()); + final long totalNormalCount = nonNullCounter.addNonNullUnsafe(destination, chunkNonNull.get()); final double newSum = plusDouble(sumSource.getUnsafe(destination), sum); final double newSum2 = plusDouble(sum2Source.getUnsafe(destination), sum2.doubleValue()); sumSource.set(destination, newSum); sum2Source.set(destination, newSum2); - if (nonNullCount <= 1) { + Assert.neqZero(totalNormalCount, "totalNormalCount"); + if (totalNormalCount == 1) { resultColumn.set(destination, Double.NaN); } else { - final double variance = (newSum2 - (newSum * newSum / nonNullCount)) / (nonNullCount - 1); + final double variance = (newSum2 - (newSum * newSum / totalNormalCount)) / (totalNormalCount - 1); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); } - } else if (nonNullCounter.getCountUnsafe(destination) <= 1) { - resultColumn.set(destination, Double.NaN); + } else { + final long totalNormalCount = nonNullCounter.getCountUnsafe(destination); + if (totalNormalCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + } else if (totalNormalCount == 1) { + resultColumn.set(destination, Double.NaN); + } } return true; } @@ -114,12 +122,12 @@ private boolean removeChunk(ByteChunk values, long destination return false; } - final long nonNullCount = nonNullCounter.addNonNullUnsafe(destination, -chunkNonNull.get()); + final long totalNormalCount = nonNullCounter.addNonNullUnsafe(destination, -chunkNonNull.get()); final double newSum; final double newSum2; - if (nonNullCount == 0) { + if (totalNormalCount == 0) { newSum = newSum2 = 0; } else { newSum = plusDouble(sumSource.getUnsafe(destination), -sum); @@ -129,12 +137,16 @@ private boolean removeChunk(ByteChunk values, long destination sumSource.set(destination, newSum); sum2Source.set(destination, newSum2); - if (nonNullCount <= 1) { + if (totalNormalCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + return true; + } + if (totalNormalCount == 1) { resultColumn.set(destination, Double.NaN); return true; } - final double variance = (newSum2 - (newSum * newSum / nonNullCount)) / (nonNullCount - 1); + final double variance = (newSum2 - (newSum * newSum / totalNormalCount)) / (totalNormalCount - 1); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); return true; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/CharChunkedAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/CharChunkedAvgOperator.java index 130c588f20e..b5c2c12aec7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/CharChunkedAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/CharChunkedAvgOperator.java @@ -20,6 +20,7 @@ import static io.deephaven.engine.table.impl.by.RollupConstants.*; import static io.deephaven.engine.util.NullSafeAddition.plusLong; import static io.deephaven.engine.util.NullSafeAddition.minusLong; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; /** * Iterative average operator. @@ -88,7 +89,7 @@ private boolean addChunk(CharChunk values, long destination, i runningSum.set(destination, newSum); resultColumn.set(destination, (double) newSum / newCount); } else if (nonNullCount.onlyNullsUnsafe(destination)) { - resultColumn.set(destination, Double.NaN); + resultColumn.set(destination, NULL_DOUBLE); } else { return false; } @@ -106,8 +107,11 @@ private boolean removeChunk(CharChunk values, long destination final long newCount = nonNullCount.addNonNullUnsafe(destination, -chunkNonNull.get()); final long newSum = minusLong(runningSum.getUnsafe(destination), chunkSum); runningSum.set(destination, newSum); - resultColumn.set(destination, (double) newSum / newCount); - + if (newCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + } else { + resultColumn.set(destination, (double) newSum / newCount); + } return true; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/CharChunkedVarOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/CharChunkedVarOperator.java index bb653cce70e..51599f8a9fd 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/CharChunkedVarOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/CharChunkedVarOperator.java @@ -3,6 +3,7 @@ // package io.deephaven.engine.table.impl.by; +import io.deephaven.base.verify.Assert; import io.deephaven.chunk.attributes.ChunkLengths; import io.deephaven.chunk.attributes.ChunkPositions; import io.deephaven.chunk.attributes.Values; @@ -19,6 +20,7 @@ import static io.deephaven.engine.table.impl.by.RollupConstants.*; import static io.deephaven.engine.util.NullSafeAddition.plusDouble; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; /** * Iterative variance operator. @@ -82,21 +84,27 @@ private boolean addChunk(CharChunk values, long destination, i final double sum = SumCharChunk.sum2CharChunk(values, chunkStart, chunkSize, chunkNonNull, sum2); if (chunkNonNull.get() > 0) { - final long nonNullCount = nonNullCounter.addNonNullUnsafe(destination, chunkNonNull.get()); + final long totalNormalCount = nonNullCounter.addNonNullUnsafe(destination, chunkNonNull.get()); final double newSum = plusDouble(sumSource.getUnsafe(destination), sum); final double newSum2 = plusDouble(sum2Source.getUnsafe(destination), sum2.doubleValue()); sumSource.set(destination, newSum); sum2Source.set(destination, newSum2); - if (nonNullCount <= 1) { + Assert.neqZero(totalNormalCount, "totalNormalCount"); + if (totalNormalCount == 1) { resultColumn.set(destination, Double.NaN); } else { - final double variance = (newSum2 - (newSum * newSum / nonNullCount)) / (nonNullCount - 1); + final double variance = (newSum2 - (newSum * newSum / totalNormalCount)) / (totalNormalCount - 1); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); } - } else if (nonNullCounter.getCountUnsafe(destination) <= 1) { - resultColumn.set(destination, Double.NaN); + } else { + final long totalNormalCount = nonNullCounter.getCountUnsafe(destination); + if (totalNormalCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + } else if (totalNormalCount == 1) { + resultColumn.set(destination, Double.NaN); + } } return true; } @@ -110,12 +118,12 @@ private boolean removeChunk(CharChunk values, long destination return false; } - final long nonNullCount = nonNullCounter.addNonNullUnsafe(destination, -chunkNonNull.get()); + final long totalNormalCount = nonNullCounter.addNonNullUnsafe(destination, -chunkNonNull.get()); final double newSum; final double newSum2; - if (nonNullCount == 0) { + if (totalNormalCount == 0) { newSum = newSum2 = 0; } else { newSum = plusDouble(sumSource.getUnsafe(destination), -sum); @@ -125,12 +133,16 @@ private boolean removeChunk(CharChunk values, long destination sumSource.set(destination, newSum); sum2Source.set(destination, newSum2); - if (nonNullCount <= 1) { + if (totalNormalCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + return true; + } + if (totalNormalCount == 1) { resultColumn.set(destination, Double.NaN); return true; } - final double variance = (newSum2 - (newSum * newSum / nonNullCount)) / (nonNullCount - 1); + final double variance = (newSum2 - (newSum * newSum / totalNormalCount)) / (totalNormalCount - 1); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); return true; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/DoubleChunkedAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/DoubleChunkedAvgOperator.java index 15666e608e0..ed0313ada8a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/DoubleChunkedAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/DoubleChunkedAvgOperator.java @@ -22,6 +22,7 @@ import static io.deephaven.engine.table.impl.by.RollupConstants.*; import static io.deephaven.engine.util.NullSafeAddition.plusDouble; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; class DoubleChunkedAvgOperator extends FpChunkedNonNormalCounter implements IterativeChunkedAggregationOperator { private final String name; @@ -141,6 +142,8 @@ private void updateResultWithNewSum(long destination, long totalNormal, long tot resultColumn.set(destination, Double.POSITIVE_INFINITY); } else if (totalNegativeInfinityCount > 0) { resultColumn.set(destination, Double.NEGATIVE_INFINITY); + } else if (totalNormal == 0) { + resultColumn.set(destination, NULL_DOUBLE); } else { resultColumn.set(destination, newSum / totalNormal); } @@ -148,12 +151,14 @@ private void updateResultWithNewSum(long destination, long totalNormal, long tot private void updateResultSumUnchanged(long destination, long totalNormal, long totalNanCount, long totalInfinityCount, long totalNegativeInfinityCount) { - if (totalNanCount > 0 || totalNormal == 0 || (totalInfinityCount > 0 && totalNegativeInfinityCount > 0)) { + if (totalNanCount > 0 || (totalInfinityCount > 0 && totalNegativeInfinityCount > 0)) { resultColumn.set(destination, Double.NaN); } else if (totalInfinityCount > 0) { resultColumn.set(destination, Double.POSITIVE_INFINITY); } else if (totalNegativeInfinityCount > 0) { resultColumn.set(destination, Double.NEGATIVE_INFINITY); + } else if (totalNormal == 0) { + resultColumn.set(destination, NULL_DOUBLE); } else { resultColumn.set(destination, runningSum.getUnsafe(destination) / totalNormal); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/DoubleChunkedReAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/DoubleChunkedReAvgOperator.java index 4bdffdf6183..a8610d282b2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/DoubleChunkedReAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/DoubleChunkedReAvgOperator.java @@ -20,6 +20,8 @@ import java.util.Collections; import java.util.Map; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; + /** * Iterative average operator. */ @@ -127,12 +129,14 @@ private boolean updateResult(long destination) { private boolean updateResult(long destination, long nncValue, long nanValue, long picValue, long nicValue, double sumSumValue) { - if (nanValue > 0 || (picValue > 0 && nicValue > 0) || nncValue == 0) { + if (nanValue > 0 || (picValue > 0 && nicValue > 0)) { return !Double.isNaN(resultColumn.getAndSetUnsafe(destination, Double.NaN)); } else if (picValue > 0) { return resultColumn.getAndSetUnsafe(destination, Double.POSITIVE_INFINITY) != Double.POSITIVE_INFINITY; } else if (nicValue > 0) { return resultColumn.getAndSetUnsafe(destination, Double.NEGATIVE_INFINITY) != Double.NEGATIVE_INFINITY; + } else if (nncValue == 0) { + return resultColumn.getAndSetUnsafe(destination, NULL_DOUBLE) != NULL_DOUBLE; } else { final double newValue = (double) (sumSumValue / nncValue); return resultColumn.getAndSetUnsafe(destination, newValue) != newValue; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/DoubleChunkedVarOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/DoubleChunkedVarOperator.java index db5991d75e4..42916045fc0 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/DoubleChunkedVarOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/DoubleChunkedVarOperator.java @@ -7,6 +7,7 @@ // @formatter:off package io.deephaven.engine.table.impl.by; +import io.deephaven.base.verify.Assert; import io.deephaven.chunk.attributes.ChunkLengths; import io.deephaven.chunk.attributes.ChunkPositions; import io.deephaven.chunk.attributes.Values; @@ -23,6 +24,7 @@ import java.util.Map; import static io.deephaven.engine.table.impl.by.RollupConstants.*; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; /** * Iterative variance operator. @@ -95,14 +97,15 @@ private boolean addChunk(DoubleChunk values, long destination, final boolean forceNanResult = totalNegativeInfinities > 0 || totalPositiveInfinities > 0 || totalNanCount > 0; if (chunkNormalCount.get() > 0) { - final long nonNullCount = nonNullCounter.addNonNullUnsafe(destination, chunkNormalCount.get()); + final long totalNormalCount = nonNullCounter.addNonNullUnsafe(destination, chunkNormalCount.get()); final double newSum = NullSafeAddition.plusDouble(sumSource.getUnsafe(destination), sum); final double newSum2 = NullSafeAddition.plusDouble(sum2Source.getUnsafe(destination), sum2.doubleValue()); sumSource.set(destination, newSum); sum2Source.set(destination, newSum2); - if (forceNanResult || nonNullCount <= 1) { + Assert.neqZero(totalNormalCount, "totalNormalCount"); + if (forceNanResult || totalNormalCount == 1) { resultColumn.set(destination, Double.NaN); } else { // If the sum or sumSquared has reached +/-Infinity, we are stuck with NaN forever. @@ -110,16 +113,24 @@ private boolean addChunk(DoubleChunk values, long destination, resultColumn.set(destination, Double.NaN); return true; } - final double variance = computeVariance(nonNullCount, newSum, newSum2); + final double variance = computeVariance(totalNormalCount, newSum, newSum2); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); } return true; - } else if (forceNanResult || (nonNullCounter.getCountUnsafe(destination) <= 1)) { + } + if (forceNanResult) { + resultColumn.set(destination, Double.NaN); + return true; + } + final long totalNormalCount = nonNullCounter.getCountUnsafe(destination); + if (totalNormalCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + return true; + } else if (totalNormalCount == 1) { resultColumn.set(destination, Double.NaN); return true; - } else { - return false; } + return false; } private static double computeVariance(long nonNullCount, double newSum, double newSum2) { @@ -165,16 +176,17 @@ private boolean removeChunk(DoubleChunk values, long destinati } sumSource.set(destination, newSum); sum2Source.set(destination, newSum2); - } else if (totalNormalCount <= 1 || forceNanResult) { - resultColumn.set(destination, Double.NaN); - return true; } else { newSum = sumSource.getUnsafe(destination); newSum2 = sum2Source.getUnsafe(destination); } - if (totalNormalCount <= 1) { + + if (totalNormalCount == 1 || forceNanResult) { resultColumn.set(destination, Double.NaN); return true; + } else if (totalNormalCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + return true; } // If the sum has reach +/-Infinity, we are stuck with NaN forever. @@ -186,6 +198,7 @@ private boolean removeChunk(DoubleChunk values, long destinati // Perform the calculation in a way that minimizes the impact of FP error. final double variance = computeVariance(totalNormalCount, newSum, newSum2); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); + return true; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedAvgOperator.java index f227389aa6b..e3cf530f7d7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedAvgOperator.java @@ -18,6 +18,7 @@ import static io.deephaven.engine.table.impl.by.RollupConstants.*; import static io.deephaven.engine.util.NullSafeAddition.plusDouble; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; class FloatChunkedAvgOperator extends FpChunkedNonNormalCounter implements IterativeChunkedAggregationOperator { private final String name; @@ -137,6 +138,8 @@ private void updateResultWithNewSum(long destination, long totalNormal, long tot resultColumn.set(destination, Double.POSITIVE_INFINITY); } else if (totalNegativeInfinityCount > 0) { resultColumn.set(destination, Double.NEGATIVE_INFINITY); + } else if (totalNormal == 0) { + resultColumn.set(destination, NULL_DOUBLE); } else { resultColumn.set(destination, newSum / totalNormal); } @@ -144,12 +147,14 @@ private void updateResultWithNewSum(long destination, long totalNormal, long tot private void updateResultSumUnchanged(long destination, long totalNormal, long totalNanCount, long totalInfinityCount, long totalNegativeInfinityCount) { - if (totalNanCount > 0 || totalNormal == 0 || (totalInfinityCount > 0 && totalNegativeInfinityCount > 0)) { + if (totalNanCount > 0 || (totalInfinityCount > 0 && totalNegativeInfinityCount > 0)) { resultColumn.set(destination, Double.NaN); } else if (totalInfinityCount > 0) { resultColumn.set(destination, Double.POSITIVE_INFINITY); } else if (totalNegativeInfinityCount > 0) { resultColumn.set(destination, Double.NEGATIVE_INFINITY); + } else if (totalNormal == 0) { + resultColumn.set(destination, NULL_DOUBLE); } else { resultColumn.set(destination, runningSum.getUnsafe(destination) / totalNormal); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedReAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedReAvgOperator.java index 7c42869077f..4d3258a8d4f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedReAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedReAvgOperator.java @@ -16,6 +16,8 @@ import java.util.Collections; import java.util.Map; +import static io.deephaven.util.QueryConstants.NULL_FLOAT; + /** * Iterative average operator. */ @@ -123,12 +125,14 @@ private boolean updateResult(long destination) { private boolean updateResult(long destination, long nncValue, long nanValue, long picValue, long nicValue, double sumSumValue) { - if (nanValue > 0 || (picValue > 0 && nicValue > 0) || nncValue == 0) { + if (nanValue > 0 || (picValue > 0 && nicValue > 0)) { return !Float.isNaN(resultColumn.getAndSetUnsafe(destination, Float.NaN)); } else if (picValue > 0) { return resultColumn.getAndSetUnsafe(destination, Float.POSITIVE_INFINITY) != Float.POSITIVE_INFINITY; } else if (nicValue > 0) { return resultColumn.getAndSetUnsafe(destination, Float.NEGATIVE_INFINITY) != Float.NEGATIVE_INFINITY; + } else if (nncValue == 0) { + return resultColumn.getAndSetUnsafe(destination, NULL_FLOAT) != NULL_FLOAT; } else { final float newValue = (float) (sumSumValue / nncValue); return resultColumn.getAndSetUnsafe(destination, newValue) != newValue; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedReVarOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedReVarOperator.java index 12df11fca34..7ce77d6c68f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedReVarOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedReVarOperator.java @@ -16,6 +16,8 @@ import java.util.Collections; import java.util.Map; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; + /** * Iterative average operator. */ @@ -91,7 +93,7 @@ private void updateResult(ReVarContext reVarContext, RowSequence destinationOk, nicSum.getChunk(reVarContext.nicContext, destinationOk).asLongChunk(); final int size = reVarContext.keyIndices.size(); - final boolean ordered = reVarContext.ordered;; + final boolean ordered = reVarContext.ordered; for (int ii = 0; ii < size; ++ii) { final boolean changed = updateResult(reVarContext.keyIndices.get(ii), nncSumChunk.get(ii), nanSumChunk.get(ii), @@ -131,8 +133,10 @@ private boolean updateResult(long destination) { private boolean updateResult(long destination, long nncValue, long nanValue, long picValue, long nicValue, double newSum, double newSum2) { - if (nanValue > 0 || picValue > 0 || nicValue > 0 || nncValue <= 1) { + if (nanValue > 0 || picValue > 0 || nicValue > 0 || nncValue == 1) { return !Double.isNaN(resultColumn.getAndSetUnsafe(destination, Double.NaN)); + } else if (nncValue == 0) { + return resultColumn.getAndSetUnsafe(destination, NULL_DOUBLE) != NULL_DOUBLE; } else { final double variance = (newSum2 - newSum * newSum / nncValue) / (nncValue - 1); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedVarOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedVarOperator.java index e3881adf0f2..3d843d604f5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedVarOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedVarOperator.java @@ -3,6 +3,7 @@ // package io.deephaven.engine.table.impl.by; +import io.deephaven.base.verify.Assert; import io.deephaven.chunk.attributes.ChunkLengths; import io.deephaven.chunk.attributes.ChunkPositions; import io.deephaven.chunk.attributes.Values; @@ -19,6 +20,7 @@ import java.util.Map; import static io.deephaven.engine.table.impl.by.RollupConstants.*; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; /** * Iterative variance operator. @@ -91,14 +93,15 @@ private boolean addChunk(FloatChunk values, long destination, final boolean forceNanResult = totalNegativeInfinities > 0 || totalPositiveInfinities > 0 || totalNanCount > 0; if (chunkNormalCount.get() > 0) { - final long nonNullCount = nonNullCounter.addNonNullUnsafe(destination, chunkNormalCount.get()); + final long totalNormalCount = nonNullCounter.addNonNullUnsafe(destination, chunkNormalCount.get()); final double newSum = NullSafeAddition.plusDouble(sumSource.getUnsafe(destination), sum); final double newSum2 = NullSafeAddition.plusDouble(sum2Source.getUnsafe(destination), sum2.doubleValue()); sumSource.set(destination, newSum); sum2Source.set(destination, newSum2); - if (forceNanResult || nonNullCount <= 1) { + Assert.neqZero(totalNormalCount, "totalNormalCount"); + if (forceNanResult || totalNormalCount == 1) { resultColumn.set(destination, Double.NaN); } else { // If the sum or sumSquared has reached +/-Infinity, we are stuck with NaN forever. @@ -106,16 +109,24 @@ private boolean addChunk(FloatChunk values, long destination, resultColumn.set(destination, Double.NaN); return true; } - final double variance = computeVariance(nonNullCount, newSum, newSum2); + final double variance = computeVariance(totalNormalCount, newSum, newSum2); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); } return true; - } else if (forceNanResult || (nonNullCounter.getCountUnsafe(destination) <= 1)) { + } + if (forceNanResult) { + resultColumn.set(destination, Double.NaN); + return true; + } + final long totalNormalCount = nonNullCounter.getCountUnsafe(destination); + if (totalNormalCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + return true; + } else if (totalNormalCount == 1) { resultColumn.set(destination, Double.NaN); return true; - } else { - return false; } + return false; } private static double computeVariance(long nonNullCount, double newSum, double newSum2) { @@ -161,16 +172,17 @@ private boolean removeChunk(FloatChunk values, long destinatio } sumSource.set(destination, newSum); sum2Source.set(destination, newSum2); - } else if (totalNormalCount <= 1 || forceNanResult) { - resultColumn.set(destination, Double.NaN); - return true; } else { newSum = sumSource.getUnsafe(destination); newSum2 = sum2Source.getUnsafe(destination); } - if (totalNormalCount <= 1) { + + if (totalNormalCount == 1 || forceNanResult) { resultColumn.set(destination, Double.NaN); return true; + } else if (totalNormalCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + return true; } // If the sum has reach +/-Infinity, we are stuck with NaN forever. @@ -182,6 +194,7 @@ private boolean removeChunk(FloatChunk values, long destinatio // Perform the calculation in a way that minimizes the impact of FP error. final double variance = computeVariance(totalNormalCount, newSum, newSum2); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); + return true; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/IntChunkedAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/IntChunkedAvgOperator.java index 81e66b75ff8..1adec564276 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/IntChunkedAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/IntChunkedAvgOperator.java @@ -24,6 +24,7 @@ import static io.deephaven.engine.table.impl.by.RollupConstants.*; import static io.deephaven.engine.util.NullSafeAddition.plusLong; import static io.deephaven.engine.util.NullSafeAddition.minusLong; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; /** * Iterative average operator. @@ -92,7 +93,7 @@ private boolean addChunk(IntChunk values, long destination, in runningSum.set(destination, newSum); resultColumn.set(destination, (double) newSum / newCount); } else if (nonNullCount.onlyNullsUnsafe(destination)) { - resultColumn.set(destination, Double.NaN); + resultColumn.set(destination, NULL_DOUBLE); } else { return false; } @@ -110,8 +111,11 @@ private boolean removeChunk(IntChunk values, long destination, final long newCount = nonNullCount.addNonNullUnsafe(destination, -chunkNonNull.get()); final long newSum = minusLong(runningSum.getUnsafe(destination), chunkSum); runningSum.set(destination, newSum); - resultColumn.set(destination, (double) newSum / newCount); - + if (newCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + } else { + resultColumn.set(destination, (double) newSum / newCount); + } return true; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/IntChunkedVarOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/IntChunkedVarOperator.java index 882dc975285..482c9af19d5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/IntChunkedVarOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/IntChunkedVarOperator.java @@ -7,6 +7,7 @@ // @formatter:off package io.deephaven.engine.table.impl.by; +import io.deephaven.base.verify.Assert; import io.deephaven.chunk.attributes.ChunkLengths; import io.deephaven.chunk.attributes.ChunkPositions; import io.deephaven.chunk.attributes.Values; @@ -23,6 +24,7 @@ import static io.deephaven.engine.table.impl.by.RollupConstants.*; import static io.deephaven.engine.util.NullSafeAddition.plusDouble; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; /** * Iterative variance operator. @@ -86,21 +88,27 @@ private boolean addChunk(IntChunk values, long destination, in final double sum = SumIntChunk.sum2IntChunk(values, chunkStart, chunkSize, chunkNonNull, sum2); if (chunkNonNull.get() > 0) { - final long nonNullCount = nonNullCounter.addNonNullUnsafe(destination, chunkNonNull.get()); + final long totalNormalCount = nonNullCounter.addNonNullUnsafe(destination, chunkNonNull.get()); final double newSum = plusDouble(sumSource.getUnsafe(destination), sum); final double newSum2 = plusDouble(sum2Source.getUnsafe(destination), sum2.doubleValue()); sumSource.set(destination, newSum); sum2Source.set(destination, newSum2); - if (nonNullCount <= 1) { + Assert.neqZero(totalNormalCount, "totalNormalCount"); + if (totalNormalCount == 1) { resultColumn.set(destination, Double.NaN); } else { - final double variance = (newSum2 - (newSum * newSum / nonNullCount)) / (nonNullCount - 1); + final double variance = (newSum2 - (newSum * newSum / totalNormalCount)) / (totalNormalCount - 1); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); } - } else if (nonNullCounter.getCountUnsafe(destination) <= 1) { - resultColumn.set(destination, Double.NaN); + } else { + final long totalNormalCount = nonNullCounter.getCountUnsafe(destination); + if (totalNormalCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + } else if (totalNormalCount == 1) { + resultColumn.set(destination, Double.NaN); + } } return true; } @@ -114,12 +122,12 @@ private boolean removeChunk(IntChunk values, long destination, return false; } - final long nonNullCount = nonNullCounter.addNonNullUnsafe(destination, -chunkNonNull.get()); + final long totalNormalCount = nonNullCounter.addNonNullUnsafe(destination, -chunkNonNull.get()); final double newSum; final double newSum2; - if (nonNullCount == 0) { + if (totalNormalCount == 0) { newSum = newSum2 = 0; } else { newSum = plusDouble(sumSource.getUnsafe(destination), -sum); @@ -129,12 +137,16 @@ private boolean removeChunk(IntChunk values, long destination, sumSource.set(destination, newSum); sum2Source.set(destination, newSum2); - if (nonNullCount <= 1) { + if (totalNormalCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + return true; + } + if (totalNormalCount == 1) { resultColumn.set(destination, Double.NaN); return true; } - final double variance = (newSum2 - (newSum * newSum / nonNullCount)) / (nonNullCount - 1); + final double variance = (newSum2 - (newSum * newSum / totalNormalCount)) / (totalNormalCount - 1); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); return true; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/LongChunkedAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/LongChunkedAvgOperator.java index 8c58a17b6d7..ce71da4691e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/LongChunkedAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/LongChunkedAvgOperator.java @@ -24,6 +24,7 @@ import static io.deephaven.engine.table.impl.by.RollupConstants.*; import static io.deephaven.engine.util.NullSafeAddition.plusLong; import static io.deephaven.engine.util.NullSafeAddition.minusLong; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; /** * Iterative average operator. @@ -92,7 +93,7 @@ private boolean addChunk(LongChunk values, long destination, i runningSum.set(destination, newSum); resultColumn.set(destination, (double) newSum / newCount); } else if (nonNullCount.onlyNullsUnsafe(destination)) { - resultColumn.set(destination, Double.NaN); + resultColumn.set(destination, NULL_DOUBLE); } else { return false; } @@ -110,8 +111,11 @@ private boolean removeChunk(LongChunk values, long destination final long newCount = nonNullCount.addNonNullUnsafe(destination, -chunkNonNull.get()); final long newSum = minusLong(runningSum.getUnsafe(destination), chunkSum); runningSum.set(destination, newSum); - resultColumn.set(destination, (double) newSum / newCount); - + if (newCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + } else { + resultColumn.set(destination, (double) newSum / newCount); + } return true; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/LongChunkedVarOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/LongChunkedVarOperator.java index 2b7b9a14bfa..d5d4929a7ab 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/LongChunkedVarOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/LongChunkedVarOperator.java @@ -7,6 +7,7 @@ // @formatter:off package io.deephaven.engine.table.impl.by; +import io.deephaven.base.verify.Assert; import io.deephaven.chunk.attributes.ChunkLengths; import io.deephaven.chunk.attributes.ChunkPositions; import io.deephaven.chunk.attributes.Values; @@ -23,6 +24,7 @@ import static io.deephaven.engine.table.impl.by.RollupConstants.*; import static io.deephaven.engine.util.NullSafeAddition.plusDouble; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; /** * Iterative variance operator. @@ -86,21 +88,27 @@ private boolean addChunk(LongChunk values, long destination, i final double sum = SumLongChunk.sum2LongChunk(values, chunkStart, chunkSize, chunkNonNull, sum2); if (chunkNonNull.get() > 0) { - final long nonNullCount = nonNullCounter.addNonNullUnsafe(destination, chunkNonNull.get()); + final long totalNormalCount = nonNullCounter.addNonNullUnsafe(destination, chunkNonNull.get()); final double newSum = plusDouble(sumSource.getUnsafe(destination), sum); final double newSum2 = plusDouble(sum2Source.getUnsafe(destination), sum2.doubleValue()); sumSource.set(destination, newSum); sum2Source.set(destination, newSum2); - if (nonNullCount <= 1) { + Assert.neqZero(totalNormalCount, "totalNormalCount"); + if (totalNormalCount == 1) { resultColumn.set(destination, Double.NaN); } else { - final double variance = (newSum2 - (newSum * newSum / nonNullCount)) / (nonNullCount - 1); + final double variance = (newSum2 - (newSum * newSum / totalNormalCount)) / (totalNormalCount - 1); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); } - } else if (nonNullCounter.getCountUnsafe(destination) <= 1) { - resultColumn.set(destination, Double.NaN); + } else { + final long totalNormalCount = nonNullCounter.getCountUnsafe(destination); + if (totalNormalCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + } else if (totalNormalCount == 1) { + resultColumn.set(destination, Double.NaN); + } } return true; } @@ -114,12 +122,12 @@ private boolean removeChunk(LongChunk values, long destination return false; } - final long nonNullCount = nonNullCounter.addNonNullUnsafe(destination, -chunkNonNull.get()); + final long totalNormalCount = nonNullCounter.addNonNullUnsafe(destination, -chunkNonNull.get()); final double newSum; final double newSum2; - if (nonNullCount == 0) { + if (totalNormalCount == 0) { newSum = newSum2 = 0; } else { newSum = plusDouble(sumSource.getUnsafe(destination), -sum); @@ -129,12 +137,16 @@ private boolean removeChunk(LongChunk values, long destination sumSource.set(destination, newSum); sum2Source.set(destination, newSum2); - if (nonNullCount <= 1) { + if (totalNormalCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + return true; + } + if (totalNormalCount == 1) { resultColumn.set(destination, Double.NaN); return true; } - final double variance = (newSum2 - (newSum * newSum / nonNullCount)) / (nonNullCount - 1); + final double variance = (newSum2 - (newSum * newSum / totalNormalCount)) / (totalNormalCount - 1); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); return true; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/ShortChunkedAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/ShortChunkedAvgOperator.java index 40bb2fd41b5..27c3ebdfbd9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/ShortChunkedAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/ShortChunkedAvgOperator.java @@ -24,6 +24,7 @@ import static io.deephaven.engine.table.impl.by.RollupConstants.*; import static io.deephaven.engine.util.NullSafeAddition.plusLong; import static io.deephaven.engine.util.NullSafeAddition.minusLong; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; /** * Iterative average operator. @@ -92,7 +93,7 @@ private boolean addChunk(ShortChunk values, long destination, runningSum.set(destination, newSum); resultColumn.set(destination, (double) newSum / newCount); } else if (nonNullCount.onlyNullsUnsafe(destination)) { - resultColumn.set(destination, Double.NaN); + resultColumn.set(destination, NULL_DOUBLE); } else { return false; } @@ -110,8 +111,11 @@ private boolean removeChunk(ShortChunk values, long destinatio final long newCount = nonNullCount.addNonNullUnsafe(destination, -chunkNonNull.get()); final long newSum = minusLong(runningSum.getUnsafe(destination), chunkSum); runningSum.set(destination, newSum); - resultColumn.set(destination, (double) newSum / newCount); - + if (newCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + } else { + resultColumn.set(destination, (double) newSum / newCount); + } return true; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/ShortChunkedVarOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/ShortChunkedVarOperator.java index 6d24d6f75d6..b8a77cfefc7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/ShortChunkedVarOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/ShortChunkedVarOperator.java @@ -7,6 +7,7 @@ // @formatter:off package io.deephaven.engine.table.impl.by; +import io.deephaven.base.verify.Assert; import io.deephaven.chunk.attributes.ChunkLengths; import io.deephaven.chunk.attributes.ChunkPositions; import io.deephaven.chunk.attributes.Values; @@ -23,6 +24,7 @@ import static io.deephaven.engine.table.impl.by.RollupConstants.*; import static io.deephaven.engine.util.NullSafeAddition.plusDouble; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; /** * Iterative variance operator. @@ -86,21 +88,27 @@ private boolean addChunk(ShortChunk values, long destination, final double sum = SumShortChunk.sum2ShortChunk(values, chunkStart, chunkSize, chunkNonNull, sum2); if (chunkNonNull.get() > 0) { - final long nonNullCount = nonNullCounter.addNonNullUnsafe(destination, chunkNonNull.get()); + final long totalNormalCount = nonNullCounter.addNonNullUnsafe(destination, chunkNonNull.get()); final double newSum = plusDouble(sumSource.getUnsafe(destination), sum); final double newSum2 = plusDouble(sum2Source.getUnsafe(destination), sum2.doubleValue()); sumSource.set(destination, newSum); sum2Source.set(destination, newSum2); - if (nonNullCount <= 1) { + Assert.neqZero(totalNormalCount, "totalNormalCount"); + if (totalNormalCount == 1) { resultColumn.set(destination, Double.NaN); } else { - final double variance = (newSum2 - (newSum * newSum / nonNullCount)) / (nonNullCount - 1); + final double variance = (newSum2 - (newSum * newSum / totalNormalCount)) / (totalNormalCount - 1); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); } - } else if (nonNullCounter.getCountUnsafe(destination) <= 1) { - resultColumn.set(destination, Double.NaN); + } else { + final long totalNormalCount = nonNullCounter.getCountUnsafe(destination); + if (totalNormalCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + } else if (totalNormalCount == 1) { + resultColumn.set(destination, Double.NaN); + } } return true; } @@ -114,12 +122,12 @@ private boolean removeChunk(ShortChunk values, long destinatio return false; } - final long nonNullCount = nonNullCounter.addNonNullUnsafe(destination, -chunkNonNull.get()); + final long totalNormalCount = nonNullCounter.addNonNullUnsafe(destination, -chunkNonNull.get()); final double newSum; final double newSum2; - if (nonNullCount == 0) { + if (totalNormalCount == 0) { newSum = newSum2 = 0; } else { newSum = plusDouble(sumSource.getUnsafe(destination), -sum); @@ -129,12 +137,16 @@ private boolean removeChunk(ShortChunk values, long destinatio sumSource.set(destination, newSum); sum2Source.set(destination, newSum2); - if (nonNullCount <= 1) { + if (totalNormalCount == 0) { + resultColumn.set(destination, NULL_DOUBLE); + return true; + } + if (totalNormalCount == 1) { resultColumn.set(destination, Double.NaN); return true; } - final double variance = (newSum2 - (newSum * newSum / nonNullCount)) / (nonNullCount - 1); + final double variance = (newSum2 - (newSum * newSum / totalNormalCount)) / (totalNormalCount - 1); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); return true; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/ByteRollingAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/ByteRollingAvgOperator.java index adc9ca532d4..8a3ec4ccd8f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/ByteRollingAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/ByteRollingAvgOperator.java @@ -87,12 +87,12 @@ public void pop(int count) { @Override public void writeToOutputChunk(int outIdx) { - if (byteWindowValues.size() == 0) { + if (byteWindowValues.isEmpty()) { outputValues.set(outIdx, NULL_DOUBLE); } else { final int count = byteWindowValues.size() - nullCount; if (count == 0) { - outputValues.set(outIdx, Double.NaN); + outputValues.set(outIdx, NULL_DOUBLE); } else { outputValues.set(outIdx, curVal / (double) count); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/CharRollingAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/CharRollingAvgOperator.java index e5620d15cc2..97dc280c981 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/CharRollingAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/CharRollingAvgOperator.java @@ -82,12 +82,12 @@ public void pop(int count) { @Override public void writeToOutputChunk(int outIdx) { - if (charWindowValues.size() == 0) { + if (charWindowValues.isEmpty()) { outputValues.set(outIdx, NULL_DOUBLE); } else { final int count = charWindowValues.size() - nullCount; if (count == 0) { - outputValues.set(outIdx, Double.NaN); + outputValues.set(outIdx, NULL_DOUBLE); } else { outputValues.set(outIdx, curVal / (double) count); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/DoubleRollingAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/DoubleRollingAvgOperator.java index 3e53befcd5c..7bc1cf11e02 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/DoubleRollingAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/DoubleRollingAvgOperator.java @@ -87,7 +87,7 @@ public void writeToOutputChunk(int outIdx) { } else { final int count = aggSum.size() - nullCount; if (count == 0) { - outputValues.set(outIdx, Double.NaN); + outputValues.set(outIdx, NULL_DOUBLE); } else { outputValues.set(outIdx, aggSum.evaluate() / (double) count); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/FloatRollingAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/FloatRollingAvgOperator.java index 4f4eaa67053..0dcb3855312 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/FloatRollingAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/FloatRollingAvgOperator.java @@ -83,7 +83,7 @@ public void writeToOutputChunk(int outIdx) { } else { final int count = aggSum.size() - nullCount; if (count == 0) { - outputValues.set(outIdx, Double.NaN); + outputValues.set(outIdx, NULL_DOUBLE); } else { outputValues.set(outIdx, aggSum.evaluate() / (double) count); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/IntRollingAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/IntRollingAvgOperator.java index fa73349b73f..2f6c5192691 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/IntRollingAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/IntRollingAvgOperator.java @@ -86,12 +86,12 @@ public void pop(int count) { @Override public void writeToOutputChunk(int outIdx) { - if (intWindowValues.size() == 0) { + if (intWindowValues.isEmpty()) { outputValues.set(outIdx, NULL_DOUBLE); } else { final int count = intWindowValues.size() - nullCount; if (count == 0) { - outputValues.set(outIdx, Double.NaN); + outputValues.set(outIdx, NULL_DOUBLE); } else { outputValues.set(outIdx, curVal / (double) count); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/LongRollingAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/LongRollingAvgOperator.java index 72c6e7b639f..81541eae9c2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/LongRollingAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/LongRollingAvgOperator.java @@ -86,12 +86,12 @@ public void pop(int count) { @Override public void writeToOutputChunk(int outIdx) { - if (longWindowValues.size() == 0) { + if (longWindowValues.isEmpty()) { outputValues.set(outIdx, NULL_DOUBLE); } else { final int count = longWindowValues.size() - nullCount; if (count == 0) { - outputValues.set(outIdx, Double.NaN); + outputValues.set(outIdx, NULL_DOUBLE); } else { outputValues.set(outIdx, curVal / (double) count); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/ShortRollingAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/ShortRollingAvgOperator.java index 23512f1ab4d..f256a88c3a8 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/ShortRollingAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/ShortRollingAvgOperator.java @@ -86,12 +86,12 @@ public void pop(int count) { @Override public void writeToOutputChunk(int outIdx) { - if (shortWindowValues.size() == 0) { + if (shortWindowValues.isEmpty()) { outputValues.set(outIdx, NULL_DOUBLE); } else { final int count = shortWindowValues.size() - nullCount; if (count == 0) { - outputValues.set(outIdx, Double.NaN); + outputValues.set(outIdx, NULL_DOUBLE); } else { outputValues.set(outIdx, curVal / (double) count); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/ByteRollingStdOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/ByteRollingStdOperator.java index 15c40b34176..a87a630b841 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/ByteRollingStdOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/ByteRollingStdOperator.java @@ -110,9 +110,14 @@ public void pop(int count) { @Override public void writeToOutputChunk(int outIdx) { - if (valueBuffer.size() == 0) { + if (valueBuffer.isEmpty()) { outputValues.set(outIdx, NULL_DOUBLE); } else { + if (nullCount == valueBuffer.size()) { + outputValues.set(outIdx, NULL_DOUBLE); + return; + } + final int count = valueBuffer.size() - nullCount; if (count <= 1) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/CharRollingStdOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/CharRollingStdOperator.java index f622450770f..5ab4fb484de 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/CharRollingStdOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/CharRollingStdOperator.java @@ -105,9 +105,14 @@ public void pop(int count) { @Override public void writeToOutputChunk(int outIdx) { - if (valueBuffer.size() == 0) { + if (valueBuffer.isEmpty()) { outputValues.set(outIdx, NULL_DOUBLE); } else { + if (nullCount == valueBuffer.size()) { + outputValues.set(outIdx, NULL_DOUBLE); + return; + } + final int count = valueBuffer.size() - nullCount; if (count <= 1) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/DoubleRollingStdOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/DoubleRollingStdOperator.java index 1bd631bfd62..2a4b52b0493 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/DoubleRollingStdOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/DoubleRollingStdOperator.java @@ -109,9 +109,14 @@ public void pop(int count) { @Override public void writeToOutputChunk(int outIdx) { - if (valueBuffer.size() == 0) { + if (valueBuffer.isEmpty()) { outputValues.set(outIdx, NULL_DOUBLE); } else { + if (nullCount == valueBuffer.size()) { + outputValues.set(outIdx, NULL_DOUBLE); + return; + } + final int count = valueBuffer.size() - nullCount; if (count <= 1) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/FloatRollingStdOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/FloatRollingStdOperator.java index f15227cd488..0653990e4e4 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/FloatRollingStdOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/FloatRollingStdOperator.java @@ -109,9 +109,14 @@ public void pop(int count) { @Override public void writeToOutputChunk(int outIdx) { - if (valueBuffer.size() == 0) { + if (valueBuffer.isEmpty()) { outputValues.set(outIdx, NULL_DOUBLE); } else { + if (nullCount == valueBuffer.size()) { + outputValues.set(outIdx, NULL_DOUBLE); + return; + } + final int count = valueBuffer.size() - nullCount; if (count <= 1) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/IntRollingStdOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/IntRollingStdOperator.java index 2c71b961993..5ffe50e2657 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/IntRollingStdOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/IntRollingStdOperator.java @@ -109,9 +109,14 @@ public void pop(int count) { @Override public void writeToOutputChunk(int outIdx) { - if (valueBuffer.size() == 0) { + if (valueBuffer.isEmpty()) { outputValues.set(outIdx, NULL_DOUBLE); } else { + if (nullCount == valueBuffer.size()) { + outputValues.set(outIdx, NULL_DOUBLE); + return; + } + final int count = valueBuffer.size() - nullCount; if (count <= 1) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/LongRollingStdOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/LongRollingStdOperator.java index b4706c05d75..ba6527c6682 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/LongRollingStdOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/LongRollingStdOperator.java @@ -109,9 +109,14 @@ public void pop(int count) { @Override public void writeToOutputChunk(int outIdx) { - if (valueBuffer.size() == 0) { + if (valueBuffer.isEmpty()) { outputValues.set(outIdx, NULL_DOUBLE); } else { + if (nullCount == valueBuffer.size()) { + outputValues.set(outIdx, NULL_DOUBLE); + return; + } + final int count = valueBuffer.size() - nullCount; if (count <= 1) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/ShortRollingStdOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/ShortRollingStdOperator.java index 193216665f4..4ef7f286514 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/ShortRollingStdOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/ShortRollingStdOperator.java @@ -109,9 +109,14 @@ public void pop(int count) { @Override public void writeToOutputChunk(int outIdx) { - if (valueBuffer.size() == 0) { + if (valueBuffer.isEmpty()) { outputValues.set(outIdx, NULL_DOUBLE); } else { + if (nullCount == valueBuffer.size()) { + outputValues.set(outIdx, NULL_DOUBLE); + return; + } + final int count = valueBuffer.size() - nullCount; if (count <= 1) { diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTest.java index f573f40547e..138cc91b59c 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTest.java @@ -1646,9 +1646,7 @@ private String avgExpr(String c) { return c + "=" + c + "_Count == 0 ? null : " + c + "_Sum.divide(java.math.BigDecimal.valueOf(" + c + "_Count), java.math.BigDecimal.ROUND_HALF_UP)"; } - // I would expect us to return a null for an average of nothing, but we instead return a NaN - // return c + "=" + c + "_Count == 0 ? null : ((double)" + c + "_Sum / (double)" + c + "_Count)"; - return c + "=((double)(" + c + "_Count == 0 ? 0.0 : " + c + "_Sum) / (double)" + c + "_Count)"; + return c + "=" + c + "_Count == 0 ? null : ((double)" + c + "_Sum / (double)" + c + "_Count)"; } @Test @@ -1886,9 +1884,9 @@ public void testAvgInfinities() { TableTools.show(result.meta()); TestCase.assertEquals(1, result.size()); double avg = result.getColumnSource("IntCol", double.class).getDouble(result.getRowSet().firstRowKey()); - TestCase.assertEquals(Double.NaN, avg); + TestCase.assertEquals(NULL_DOUBLE, avg); double avgF = result.getColumnSource("FloatCol", double.class).getDouble(result.getRowSet().firstRowKey()); - TestCase.assertEquals(Double.NaN, avgF); + TestCase.assertEquals(NULL_DOUBLE, avgF); final ControlledUpdateGraph updateGraph = ExecutionContext.getContext().getUpdateGraph().cast(); updateGraph.runWithinUnitTestCycle(() -> { @@ -1965,9 +1963,9 @@ public void testVarInfinities() { TableTools.show(result.meta()); TestCase.assertEquals(1, result.size()); double var = result.getColumnSource("IntCol", double.class).getDouble(result.getRowSet().firstRowKey()); - TestCase.assertEquals(Double.NaN, var); + TestCase.assertEquals(NULL_DOUBLE, var); double varF = result.getColumnSource("FloatCol", double.class).getDouble(result.getRowSet().firstRowKey()); - TestCase.assertEquals(Double.NaN, varF); + TestCase.assertEquals(NULL_DOUBLE, varF); final ControlledUpdateGraph updateGraph = ExecutionContext.getContext().getUpdateGraph().cast(); updateGraph.runWithinUnitTestCycle(() -> { diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTestFormulaStaticMethods.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTestFormulaStaticMethods.java index e25376a5579..c0633350819 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTestFormulaStaticMethods.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTestFormulaStaticMethods.java @@ -14,6 +14,8 @@ import java.math.BigDecimal; import java.math.BigInteger; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; + public class QueryTableAggregationTestFormulaStaticMethods { public static ByteVector abs(ByteVector values) { final byte[] result = new byte[values.intSize()]; @@ -357,6 +359,7 @@ public static double varChar(CharVector values) { double sum = 0; double sum2 = 0; int count = 0; + int nullCount = 0; for (int ii = 0; ii < values.size(); ++ii) { final char c = values.get(ii); @@ -364,12 +367,26 @@ public static double varChar(CharVector values) { sum += c; sum2 += c * c; count++; + } else { + nullCount++; } } + if (nullCount == values.size()) { + return NULL_DOUBLE; + } return (sum2 - sum * sum / count) / (count - 1); } + public static double stdChar(CharVector values) { + if (values == null) { + return NULL_DOUBLE; + } + + final double v = varChar(values); + return v == NULL_DOUBLE ? NULL_DOUBLE : Math.sqrt(v); + } + public static BigDecimal varBigInt(ObjectVector values) { BigInteger sum = BigInteger.ZERO; BigInteger sum2 = BigInteger.ZERO; @@ -614,11 +631,13 @@ static String varFunction(String col) { static String stdFunction(String col) { switch (col) { + case "charCol": + return QueryTableAggregationTestFormulaStaticMethods.class.getCanonicalName() + ".stdChar(" + col + ")"; case "bigI": case "bigD": return "io.deephaven.util.BigDecimalUtils.sqrt(" + varFunction(col) + ", 10)"; default: - return "Math.sqrt(" + varFunction(col) + ")"; + return "std(" + col + ")"; } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingAvg.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingAvg.java index 05b93595eab..3c9fdfec4d8 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingAvg.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingAvg.java @@ -78,8 +78,7 @@ public class TestRollingAvg extends BaseUpdateByTest { private String[] getFormulas(String[] columns) { return Arrays.stream(columns) - // Force null instead of NaN when vector size == 0 - .map(c -> String.format("%s=%s.size() == 0 ? null : avg(%s)", c, c, c)) + .map(c -> String.format("%s=avg(%s)", c, c)) .toArray(String[]::new); } @@ -290,6 +289,14 @@ private void doTestStaticBucketedTimedBigNumbers(final QueryTable t, final Durat // region Static Zero Key Tests + @Test + public void testStaticZeroKeyAllNullVector() { + final int prevTicks = 1; + final int postTicks = 0; + + doTestStaticZeroKey(prevTicks, postTicks); + } + @Test public void testStaticZeroKeyRev() { final int prevTicks = 100; diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingCount.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingCount.java index d80c33edeb9..8fe83dfb661 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingCount.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingCount.java @@ -74,8 +74,7 @@ public class TestRollingCount extends BaseUpdateByTest { private String[] getFormulas(String[] columns) { return Arrays.stream(columns) - // Force null instead of NaN when vector size == 0 - .map(c -> String.format("%s=count(%s)", c, c, c)) + .map(c -> String.format("%s=count(%s)", c, c)) .toArray(String[]::new); } @@ -235,6 +234,13 @@ private void doTestStaticBucketedTimedBigNumbers(final QueryTable t, final Durat // endregion Object Helper functions // region Static Zero Key Tests + @Test + public void testStaticZeroKeyAllNullVector() { + final int prevTicks = 1; + final int postTicks = 0; + + doTestStaticZeroKey(prevTicks, postTicks); + } @Test public void testStaticZeroKeyRev() { diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingFormula.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingFormula.java index 685f19dde43..7238a47c1d0 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingFormula.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingFormula.java @@ -111,6 +111,14 @@ public static BigInteger sumBigInteger(ObjectVector bigIntegerObject // region Static Zero Key Tests + @Test + public void testStaticZeroKeyAllNullVector() { + final int prevTicks = 1; + final int postTicks = 0; + + doTestStaticZeroKey(prevTicks, postTicks); + } + @Test public void testStaticZeroKeyRev() { final int prevTicks = 100; @@ -263,12 +271,7 @@ private void doTestStaticZeroKey(final int prevTicks, final int postTicks) { //////////////////////////////////////////////////////////////////////////////////////////////////// actual = t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "avg(x)", "x", primitiveColumns)); - - // avg return Double.NaN when the window is empty, so we should adjust our comparison table. - updateStrings = Arrays.stream(primitiveColumns).map(c -> c + "=isNull(" + c + ") ? Double.NaN : " + c) - .toArray(String[]::new); - expected = t.updateBy(UpdateByOperation.RollingAvg(prevTicks, postTicks, primitiveColumns)) - .update(updateStrings); + expected = t.updateBy(UpdateByOperation.RollingAvg(prevTicks, postTicks, primitiveColumns)); TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); @@ -386,12 +389,7 @@ private void doTestStaticZeroKeyTimed(final Duration prevTime, final Duration po actual = t .updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, "avg(x)", "x", primitiveColumns)); - - // avg return Double.NaN when the window is empty, so we should adjust our comparison table. - updateStrings = Arrays.stream(primitiveColumns).map(c -> c + "=isNull(" + c + ") ? Double.NaN : " + c) - .toArray(String[]::new); - expected = t.updateBy(UpdateByOperation.RollingAvg("ts", prevTime, postTime, primitiveColumns)) - .update(updateStrings); + expected = t.updateBy(UpdateByOperation.RollingAvg("ts", prevTime, postTime, primitiveColumns)); TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); @@ -601,12 +599,7 @@ private void doTestStaticBucketed(boolean grouped, int prevTicks, int postTicks) actual = t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "avg(x)", "x", primitiveColumns), "Sym"); - - // avg return Double.NaN when the window is empty, so we should adjust our comparison table. - updateStrings = Arrays.stream(primitiveColumns).map(c -> c + "=isNull(" + c + ") ? Double.NaN : " + c) - .toArray(String[]::new); - expected = t.updateBy(UpdateByOperation.RollingAvg(prevTicks, postTicks, primitiveColumns), "Sym") - .update(updateStrings); + expected = t.updateBy(UpdateByOperation.RollingAvg(prevTicks, postTicks, primitiveColumns), "Sym"); TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); @@ -726,12 +719,7 @@ private void doTestStaticBucketedTimed(boolean grouped, Duration prevTime, Durat actual = t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, "avg(x)", "x", primitiveColumns), "Sym"); - - // avg return Double.NaN when the window is empty, so we should adjust our comparison table. - updateStrings = Arrays.stream(primitiveColumns).map(c -> c + "=isNull(" + c + ") ? Double.NaN : " + c) - .toArray(String[]::new); - expected = t.updateBy(UpdateByOperation.RollingAvg("ts", prevTime, postTime, primitiveColumns), "Sym") - .update(updateStrings); + expected = t.updateBy(UpdateByOperation.RollingAvg("ts", prevTime, postTime, primitiveColumns), "Sym"); TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingGroup.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingGroup.java index e9faa243e10..2c6befbb805 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingGroup.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingGroup.java @@ -64,6 +64,14 @@ public class TestRollingGroup extends BaseUpdateByTest { // region Static Zero Key Tests + @Test + public void testStaticZeroKeyAllNullVector() { + final int prevTicks = 1; + final int postTicks = 0; + + doTestStaticZeroKey(prevTicks, postTicks); + } + @Test public void testStaticZeroKeyRev() { final int prevTicks = 100; diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingMinMax.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingMinMax.java index d5598242124..c8d51cd5394 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingMinMax.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingMinMax.java @@ -473,6 +473,14 @@ private void doTestStaticBucketedTimedBigNumbers(final QueryTable t, final Durat // region Static Zero Key Tests + @Test + public void testStaticZeroKeyAllNullVector() { + final int prevTicks = 1; + final int postTicks = 0; + + doTestStaticZeroKey(prevTicks, postTicks); + } + @Test public void testStaticZeroKeyRev() { final int prevTicks = 100; diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingProduct.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingProduct.java index 609035daa38..8107ce5e716 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingProduct.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingProduct.java @@ -91,8 +91,7 @@ public EnumSet diffItems() { private String[] getFormulas(String[] columns) { return Arrays.stream(columns) - // Force null instead of NaN when vector size == 0 - .map(c -> String.format("%s=%s.size() == 0 ? null : product(%s)", c, c, c)) + .map(c -> String.format("%s=product(%s)", c, c, c)) .toArray(String[]::new); } @@ -370,6 +369,14 @@ private void doTestStaticBucketedTimedBigNumbers(final QueryTable t, final Durat // region Static Zero Key Tests + @Test + public void testStaticZeroKeyAllNullVector() { + final int prevTicks = 1; + final int postTicks = 0; + + doTestStaticZeroKey(prevTicks, postTicks); + } + @Test public void testStaticZeroKeyRev() { final int prevTicks = 10; diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingStd.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingStd.java index 787eb3d1bda..52613920d6e 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingStd.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingStd.java @@ -78,8 +78,7 @@ public class TestRollingStd extends BaseUpdateByTest { private String[] getFormulas(String[] columns) { return Arrays.stream(columns) - // Force null instead of NaN when vector size == 0 - .map(c -> String.format("%s=%s.size() == 0 ? null : std(%s)", c, c, c)) + .map(c -> String.format("%s=std(%s)", c, c)) .toArray(String[]::new); } @@ -316,6 +315,14 @@ private void doTestStaticBucketedTimedBigNumbers(final QueryTable t, final Durat // region Static Zero Key Tests + @Test + public void testStaticZeroKeyAllNullVector() { + final int prevTicks = 1; + final int postTicks = 0; + + doTestStaticZeroKey(prevTicks, postTicks); + } + @Test public void testStaticZeroKeyRev() { final int prevTicks = 100; diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingSum.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingSum.java index c985d4dd766..66a10af9287 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingSum.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingSum.java @@ -62,7 +62,7 @@ public EnumSet diffItems() { // region Static Zero Key Tests @Test - public void testStaticZeroKeyWithAllNullWindows() { + public void testStaticZeroKeyAllNullVector() { final QueryTable t = createTestTable(10000, false, false, false, 0x31313131).t; t.setRefreshing(false); diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingWAvg.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingWAvg.java index d8d9ffa0cb9..33b93422407 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingWAvg.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingWAvg.java @@ -682,6 +682,14 @@ private void doTestStaticTimed(boolean bucketed, Duration prevTime, Duration pos doTestStaticTimedBigNumbers(t, prevTime, postTime, bucketed, weightCol, "wavgBigIntBigDec", "wavgBigDecBigDec"); } + @Test + public void testStaticZeroKeyAllNullVector() { + final int prevTicks = 1; + final int postTicks = 0; + + doTestStatic(false, prevTicks, postTicks); + } + @Test public void testStaticZeroKeyRev() { final int prevTicks = 100;