Additional tests, improving coverage for MatchFilter CI String matchi…

…ng and RemappedDataIndex
lbooker42 · Mar 19, 2024 · c221bd0 · c221bd0
1 parent c41d04a
commit c221bd0
Show file tree

Hide file tree

Showing 3 changed files with 90 additions and 5 deletions.
diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/AbstractColumnSource.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/AbstractColumnSource.java
@@ -146,7 +146,7 @@ public WritableRowSet match(
                         final int thisChunkSize = chunkKeys.size();
                         for (int ii = 0; ii < thisChunkSize; ++ii) {
                             final String key = chunkKeys.get(ii);
-                            if (keySet.contains(key)) {
+                            if (keySet.containsKey(key)) {
                                 matchingIndexRowsBuilder.appendKey(chunkRowKeys.get(ii));
                             }
                         }

diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableTest.java
@@ -933,6 +933,40 @@ public void testStringContainsFilter() {
         }
     }
 
+    public void testStringMatchFilterIndexed() {
+        // MatchFilters (currently) only use indexes on initial creation but this incremental test will recreate
+        // index-enabled match filtered tables and compare them against incremental non-indexed filtered tables.
+
+        Function<String, WhereFilter> filter = ConditionFilter::createConditionFilter;
+        final Random random = new Random(0);
+
+        final int size = 500;
+
+        final ColumnInfo<?, ?>[] columnInfo;
+        final QueryTable table = getTable(size, random, columnInfo = initColumnInfos(new String[] {"S1", "S2"},
+                new SetGenerator<>("aa", "bb", "cc", "dd", "AA", "BB", "CC", "DD"),
+                new SetGenerator<>("aaa", "bbb", "ccc", "ddd", "AAA", "BBB", "CCC", "DDD")));
+
+        DataIndexer.getOrCreateDataIndex(table, "S1");
+        DataIndexer.getOrCreateDataIndex(table, "S2");
+
+        final EvalNuggetInterface[] en = new EvalNuggetInterface[] {
+                EvalNugget.from(() -> table.where("S1 in 'aa'")),
+                EvalNugget.from(() -> table.where("S2 in 'bbb'")),
+                EvalNugget.from(() -> table.where("S2 not in 'ccc', 'dddd'")),
+                EvalNugget.from(() -> table.where("S1 not in 'aa', 'bb'")),
+
+                EvalNugget.from(() -> table.where("S1 icase in 'aa'")),
+                EvalNugget.from(() -> table.where("S2 icase in 'bbb'")),
+                EvalNugget.from(() -> table.where("S2 icase not in 'ccc', 'dddd'")),
+                EvalNugget.from(() -> table.where("S1 icase not in 'aa', 'bb'")),
+        };
+
+        for (int i = 0; i < 500; i++) {
+            simulateShiftAwareStep(size, random, table, columnInfo, en);
+        }
+    }
+
     public void testDoubleRangeFilterSimple() {
         final Table t = TableTools.newTable(doubleCol("DV", 1.0, 2.0, -3.0, Double.NaN, QueryConstants.NULL_DOUBLE, 6.0,
                 Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, 9.0)).update("IV=i+1");

diff --git a/.../table/src/test/java/io/deephaven/engine/table/impl/indexer/TestTransformedDataIndex.java b/.../table/src/test/java/io/deephaven/engine/table/impl/indexer/TestTransformedDataIndex.java
@@ -10,6 +10,7 @@
 import io.deephaven.engine.table.*;
 import io.deephaven.engine.table.impl.QueryTable;
 import io.deephaven.engine.table.impl.dataindex.DataIndexUtils;
+import io.deephaven.engine.table.impl.sources.InMemoryColumnSource;
 import io.deephaven.engine.table.iterators.ChunkedColumnIterator;
 import io.deephaven.engine.testutil.ColumnInfo;
 import io.deephaven.engine.testutil.ControlledUpdateGraph;
@@ -22,10 +23,7 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Random;
+import java.util.*;
 import java.util.function.Function;
 import java.util.function.ToLongFunction;
 
@@ -93,6 +91,59 @@ public void testFullIndexLookup() {
         }
     }
 
+    @Test
+    public void testRemappedDataIndexAllColumns() {
+        for (final DataIndex dataIndex : dataIndexes) {
+            // Map dummy columns to the key columns
+            final Map<ColumnSource<?>, ColumnSource<?>> firstRemap = new HashMap<>();
+            dataIndex.keyColumnNames().forEach(name -> firstRemap.put(testTable.getColumnSource(name),
+                    InMemoryColumnSource.makeImmutableSource(Integer.class, null)));
+            final DataIndex firstRemappedIndex = dataIndex.remapKeyColumns(firstRemap);
+            // Verify that the original and remapped indexes point to the same index table columns
+            assertEquals(dataIndex.keyColumns(), firstRemappedIndex.keyColumns());
+
+            // Map new dummy columns to the old dummy columns (second-level)
+            final Map<ColumnSource<?>, ColumnSource<?>> secondRemap = new HashMap<>();
+            firstRemap.forEach((oldColumn, dummy) -> secondRemap.put(dummy,
+                    InMemoryColumnSource.makeImmutableSource(Integer.class, null)));
+            final DataIndex secondRemappedIndex = firstRemappedIndex.remapKeyColumns(secondRemap);
+            // Verify that the original and remapped indexes point to the same index table columns
+            assertEquals(dataIndex.keyColumns(), secondRemappedIndex.keyColumns());
+
+            // Map even newer dummy columns to the old dummy columns (third-level)
+            final Map<ColumnSource<?>, ColumnSource<?>> thirdRemap = new HashMap<>();
+            secondRemap.forEach((oldColumn, dummy) -> thirdRemap.put(dummy,
+                    InMemoryColumnSource.makeImmutableSource(Integer.class, null)));
+            final DataIndex thirdRemappedIndex = secondRemappedIndex.remapKeyColumns(secondRemap);
+            // Verify that the original and remapped indexes point to the same index table columns
+            assertEquals(dataIndex.keyColumns(), thirdRemappedIndex.keyColumns());
+        }
+    }
+
+    @Test
+    public void testRemappedDataIndexOnlyFirstColumns() {
+        for (final DataIndex dataIndex : dataIndexes) {
+            final ColumnSource<?> firstDummy = InMemoryColumnSource.makeImmutableSource(Integer.class, null);
+            final Map<ColumnSource<?>, ColumnSource<?>> firstRemap =
+                    Map.of(testTable.getColumnSource(dataIndex.keyColumnNames().get(0)), firstDummy);
+            final DataIndex firstRemappedIndex = dataIndex.remapKeyColumns(firstRemap);
+            // Verify that the original and remapped indexes point to the same index table columns
+            assertEquals(dataIndex.keyColumns(), firstRemappedIndex.keyColumns());
+
+            final ColumnSource<?> secondDummy = InMemoryColumnSource.makeImmutableSource(Integer.class, null);
+            final Map<ColumnSource<?>, ColumnSource<?>> secondRemap = Map.of(firstDummy, secondDummy);
+            final DataIndex secondRemappedIndex = firstRemappedIndex.remapKeyColumns(secondRemap);
+            // Verify that the original and remapped indexes point to the same index table columns
+            assertEquals(dataIndex.keyColumns(), firstRemappedIndex.keyColumns());
+
+            final ColumnSource<?> thirdDummy = InMemoryColumnSource.makeImmutableSource(Integer.class, null);
+            final Map<ColumnSource<?>, ColumnSource<?>> thirdRemap = Map.of(secondDummy, thirdDummy);
+            final DataIndex thirdRemappedIndex = secondRemappedIndex.remapKeyColumns(thirdRemap);
+            // Verify that the original and remapped indexes point to the same index table columns
+            assertEquals(dataIndex.keyColumns(), thirdRemappedIndex.keyColumns());
+        }
+    }
+
 
     @Test
     public void testMultiColumnOutOfOrderLookup() {