From c221bd01f9f5b8da5b39f6f5a77b73470091488d Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Tue, 19 Mar 2024 11:10:32 -0700 Subject: [PATCH] Additional tests, improving coverage for MatchFilter CI String matching and RemappedDataIndex --- .../table/impl/AbstractColumnSource.java | 2 +- .../engine/table/impl/QueryTableTest.java | 34 +++++++++++ .../indexer/TestTransformedDataIndex.java | 59 +++++++++++++++++-- 3 files changed, 90 insertions(+), 5 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/AbstractColumnSource.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/AbstractColumnSource.java index 95adaeae048..65d434d6598 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/AbstractColumnSource.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/AbstractColumnSource.java @@ -146,7 +146,7 @@ public WritableRowSet match( final int thisChunkSize = chunkKeys.size(); for (int ii = 0; ii < thisChunkSize; ++ii) { final String key = chunkKeys.get(ii); - if (keySet.contains(key)) { + if (keySet.containsKey(key)) { matchingIndexRowsBuilder.appendKey(chunkRowKeys.get(ii)); } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableTest.java index 9300da3ff43..b9ee758b68f 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableTest.java @@ -933,6 +933,40 @@ public void testStringContainsFilter() { } } + public void testStringMatchFilterIndexed() { + // MatchFilters (currently) only use indexes on initial creation but this incremental test will recreate + // index-enabled match filtered tables and compare them against incremental non-indexed filtered tables. + + Function filter = ConditionFilter::createConditionFilter; + final Random random = new Random(0); + + final int size = 500; + + final ColumnInfo[] columnInfo; + final QueryTable table = getTable(size, random, columnInfo = initColumnInfos(new String[] {"S1", "S2"}, + new SetGenerator<>("aa", "bb", "cc", "dd", "AA", "BB", "CC", "DD"), + new SetGenerator<>("aaa", "bbb", "ccc", "ddd", "AAA", "BBB", "CCC", "DDD"))); + + DataIndexer.getOrCreateDataIndex(table, "S1"); + DataIndexer.getOrCreateDataIndex(table, "S2"); + + final EvalNuggetInterface[] en = new EvalNuggetInterface[] { + EvalNugget.from(() -> table.where("S1 in 'aa'")), + EvalNugget.from(() -> table.where("S2 in 'bbb'")), + EvalNugget.from(() -> table.where("S2 not in 'ccc', 'dddd'")), + EvalNugget.from(() -> table.where("S1 not in 'aa', 'bb'")), + + EvalNugget.from(() -> table.where("S1 icase in 'aa'")), + EvalNugget.from(() -> table.where("S2 icase in 'bbb'")), + EvalNugget.from(() -> table.where("S2 icase not in 'ccc', 'dddd'")), + EvalNugget.from(() -> table.where("S1 icase not in 'aa', 'bb'")), + }; + + for (int i = 0; i < 500; i++) { + simulateShiftAwareStep(size, random, table, columnInfo, en); + } + } + public void testDoubleRangeFilterSimple() { final Table t = TableTools.newTable(doubleCol("DV", 1.0, 2.0, -3.0, Double.NaN, QueryConstants.NULL_DOUBLE, 6.0, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, 9.0)).update("IV=i+1"); diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/indexer/TestTransformedDataIndex.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/indexer/TestTransformedDataIndex.java index ad392544c17..da3b735ad19 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/indexer/TestTransformedDataIndex.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/indexer/TestTransformedDataIndex.java @@ -10,6 +10,7 @@ import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.QueryTable; import io.deephaven.engine.table.impl.dataindex.DataIndexUtils; +import io.deephaven.engine.table.impl.sources.InMemoryColumnSource; import io.deephaven.engine.table.iterators.ChunkedColumnIterator; import io.deephaven.engine.testutil.ColumnInfo; import io.deephaven.engine.testutil.ControlledUpdateGraph; @@ -22,10 +23,7 @@ import org.junit.Test; import org.junit.experimental.categories.Category; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Random; +import java.util.*; import java.util.function.Function; import java.util.function.ToLongFunction; @@ -93,6 +91,59 @@ public void testFullIndexLookup() { } } + @Test + public void testRemappedDataIndexAllColumns() { + for (final DataIndex dataIndex : dataIndexes) { + // Map dummy columns to the key columns + final Map, ColumnSource> firstRemap = new HashMap<>(); + dataIndex.keyColumnNames().forEach(name -> firstRemap.put(testTable.getColumnSource(name), + InMemoryColumnSource.makeImmutableSource(Integer.class, null))); + final DataIndex firstRemappedIndex = dataIndex.remapKeyColumns(firstRemap); + // Verify that the original and remapped indexes point to the same index table columns + assertEquals(dataIndex.keyColumns(), firstRemappedIndex.keyColumns()); + + // Map new dummy columns to the old dummy columns (second-level) + final Map, ColumnSource> secondRemap = new HashMap<>(); + firstRemap.forEach((oldColumn, dummy) -> secondRemap.put(dummy, + InMemoryColumnSource.makeImmutableSource(Integer.class, null))); + final DataIndex secondRemappedIndex = firstRemappedIndex.remapKeyColumns(secondRemap); + // Verify that the original and remapped indexes point to the same index table columns + assertEquals(dataIndex.keyColumns(), secondRemappedIndex.keyColumns()); + + // Map even newer dummy columns to the old dummy columns (third-level) + final Map, ColumnSource> thirdRemap = new HashMap<>(); + secondRemap.forEach((oldColumn, dummy) -> thirdRemap.put(dummy, + InMemoryColumnSource.makeImmutableSource(Integer.class, null))); + final DataIndex thirdRemappedIndex = secondRemappedIndex.remapKeyColumns(secondRemap); + // Verify that the original and remapped indexes point to the same index table columns + assertEquals(dataIndex.keyColumns(), thirdRemappedIndex.keyColumns()); + } + } + + @Test + public void testRemappedDataIndexOnlyFirstColumns() { + for (final DataIndex dataIndex : dataIndexes) { + final ColumnSource firstDummy = InMemoryColumnSource.makeImmutableSource(Integer.class, null); + final Map, ColumnSource> firstRemap = + Map.of(testTable.getColumnSource(dataIndex.keyColumnNames().get(0)), firstDummy); + final DataIndex firstRemappedIndex = dataIndex.remapKeyColumns(firstRemap); + // Verify that the original and remapped indexes point to the same index table columns + assertEquals(dataIndex.keyColumns(), firstRemappedIndex.keyColumns()); + + final ColumnSource secondDummy = InMemoryColumnSource.makeImmutableSource(Integer.class, null); + final Map, ColumnSource> secondRemap = Map.of(firstDummy, secondDummy); + final DataIndex secondRemappedIndex = firstRemappedIndex.remapKeyColumns(secondRemap); + // Verify that the original and remapped indexes point to the same index table columns + assertEquals(dataIndex.keyColumns(), firstRemappedIndex.keyColumns()); + + final ColumnSource thirdDummy = InMemoryColumnSource.makeImmutableSource(Integer.class, null); + final Map, ColumnSource> thirdRemap = Map.of(secondDummy, thirdDummy); + final DataIndex thirdRemappedIndex = secondRemappedIndex.remapKeyColumns(thirdRemap); + // Verify that the original and remapped indexes point to the same index table columns + assertEquals(dataIndex.keyColumns(), thirdRemappedIndex.keyColumns()); + } + } + @Test public void testMultiColumnOutOfOrderLookup() {