Merge branch 'master' into cache_dumps

apache · Sep 27, 2023 · 7fe7e16 · 7fe7e16
2 parents 084580b + 1185941
commit 7fe7e16
Show file tree

Hide file tree

Showing 14 changed files with 267 additions and 51 deletions.
diff --git a/...utility/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingCheckSizeTest.java b/...utility/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingCheckSizeTest.java
@@ -436,7 +436,12 @@ private void checkNoCheckSizeInCaseBrokenData(String cacheName) {
         );
 
         String out = testOut.toString();
+
         assertContains(log, out, "issues found (listed above)");
+
+        assertContains(log, out, String.format("Index issues found on node %s [consistentId='%s']:",
+            crd.localNode().id().toString(), crd.localNode().consistentId()));
+
         assertNotContains(log, out, "Size check");
     }
 
@@ -471,8 +476,7 @@ private void validateCheckSizes(
         assertContains(log, out, "Size check");
 
         Map<String, ValidateIndexesCheckSizeResult> valIdxCheckSizeResults =
-            ((ValidateIndexesTaskResult)lastOperationResult).results().get(node.localNode().id())
-                .checkSizeResult();
+            ((ValidateIndexesTaskResult)lastOperationResult).jobResult(node.localNode()).checkSizeResult();
 
         assertEquals(rmvByTbl.size(), valIdxCheckSizeResults.size());
 

diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -34,6 +34,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
@@ -52,6 +53,9 @@
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
+import javax.cache.expiry.CreatedExpiryPolicy;
+import javax.cache.expiry.Duration;
+import javax.cache.expiry.ExpiryPolicy;
 import javax.cache.processor.EntryProcessor;
 import javax.cache.processor.EntryProcessorException;
 import javax.cache.processor.MutableEntry;
@@ -2393,6 +2397,38 @@ public void testCacheIdleVerifyMovingParts() throws Exception {
         assertContains(log, testOut.toString(), "MOVING partitions");
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    @Test
+    public void testCacheIdleVerifyExpiringEntries() throws Exception {
+        IgniteEx ignite = startGrids(3);
+
+        ignite.cluster().state(ACTIVE);
+
+        IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+            .setAffinity(new RendezvousAffinityFunction(false, 32))
+            .setBackups(1));
+
+        Random rnd = new Random();
+
+        // Put without expiry policy.
+        for (int i = 0; i < 5_000; i++)
+            cache.put(i, i);
+
+        // Put with expiry policy.
+        for (int i = 5_000; i < 10_000; i++) {
+            ExpiryPolicy expPol = new CreatedExpiryPolicy(new Duration(TimeUnit.MILLISECONDS, rnd.nextInt(1_000)));
+            cache.withExpiryPolicy(expPol).put(i, i);
+        }
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify"));
+
+        assertContains(log, testOut.toString(), "no conflicts have been found");
+    }
+
     /** */
     @Test
     public void testCacheSequence() throws Exception {

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/api/CommandUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/management/api/CommandUtils.java
@@ -549,17 +549,31 @@ public static boolean printErrors(Map<UUID, Exception> exceptions, String infoMs
 
         printer.accept(infoMsg);
 
-        for (Map.Entry<UUID, Exception> e : exceptions.entrySet()) {
-            printer.accept(INDENT + "Node ID: " + e.getKey());
-
-            printer.accept(INDENT + "Exception message:");
-            printer.accept(DOUBLE_INDENT + e.getValue().getMessage());
-            printer.accept("");
-        }
+        exceptions.forEach((nodeId, err) -> printNodeError(printer, nodeId, null, err));
 
         return true;
     }
 
+    /**
+     * Prints single node exception message to the log.
+     *
+     * @param printer Printer to use.
+     * @param nodeId Node id.
+     * @param consistentId Node consistent id.
+     * @param err Exception.
+     */
+    public static void printNodeError(
+        Consumer<String> printer,
+        UUID nodeId,
+        @Nullable Object consistentId,
+        Exception err
+    ) {
+        printer.accept(INDENT + "Node ID: " + nodeId + (consistentId == null ? "" : " [consistentId='" + consistentId + "']"));
+        printer.accept(INDENT + "Exception message:");
+        printer.accept(DOUBLE_INDENT + err.getMessage());
+        printer.accept("");
+    }
+
     /** */
     public static boolean experimental(Command<?, ?> cmd) {
         return cmd.getClass().isAnnotationPresent(IgniteExperimental.class);

diff --git a/...rc/main/java/org/apache/ignite/internal/management/cache/CacheValidateIndexesCommand.java b/...rc/main/java/org/apache/ignite/internal/management/cache/CacheValidateIndexesCommand.java
@@ -19,7 +19,6 @@
 
 import java.util.Collection;
 import java.util.Map;
-import java.util.UUID;
 import java.util.function.Consumer;
 import org.apache.ignite.internal.client.GridClientNode;
 import org.apache.ignite.internal.management.api.CommandUtils;
@@ -64,17 +63,24 @@ public class CacheValidateIndexesCommand
         ValidateIndexesTaskResult res0,
         Consumer<String> printer
     ) {
-        boolean errors = CommandUtils.printErrors(res0.exceptions(), "Index validation failed on nodes:", printer);
+        boolean errors = !F.isEmpty(res0.exceptions());
 
-        for (Map.Entry<UUID, ValidateIndexesJobResult> nodeEntry : res0.results().entrySet()) {
+        if (errors) {
+            printer.accept("Index validation failed on nodes:");
+
+            res0.exceptions().forEach((node, err) -> CommandUtils.printNodeError(printer, node.id(), node.consistentId(), err));
+        }
+
+        for (Map.Entry<ValidateIndexesTaskResult.NodeInfo, ValidateIndexesJobResult> nodeEntry : res0.results().entrySet()) {
             ValidateIndexesJobResult jobRes = nodeEntry.getValue();
 
             if (!jobRes.hasIssues())
                 continue;
 
             errors = true;
 
-            printer.accept("Index issues found on node " + nodeEntry.getKey() + ":");
+            printer.accept("Index issues found on node " + nodeEntry.getKey().id() + " [consistentId='"
+                + nodeEntry.getKey().consistentId() + "']:");
 
             for (IndexIntegrityCheckIssue is : jobRes.integrityCheckFailures())
                 printer.accept(INDENT + is);

diff --git a/...s/core/src/main/java/org/apache/ignite/internal/management/cache/ValidateIndexesTask.java b/...s/core/src/main/java/org/apache/ignite/internal/management/cache/ValidateIndexesTask.java
@@ -20,10 +20,8 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
 import org.apache.ignite.IgniteException;
@@ -52,17 +50,16 @@ public class ValidateIndexesTask extends VisorMultiNodeTask<CacheValidateIndexes
 
     /** {@inheritDoc} */
     @Nullable @Override protected ValidateIndexesTaskResult reduce0(List<ComputeJobResult> list) throws IgniteException {
-        Map<UUID, Exception> exceptions = new HashMap<>();
-        Map<UUID, ValidateIndexesJobResult> jobResults = new HashMap<>();
+        ValidateIndexesTaskResult taskResult = new ValidateIndexesTaskResult();
 
         for (ComputeJobResult res : list) {
             if (res.getException() != null)
-                exceptions.put(res.getNode().id(), res.getException());
+                taskResult.addException(res.getNode(), res.getException());
             else
-                jobResults.put(res.getNode().id(), res.getData());
+                taskResult.addResult(res.getNode(), res.getData());
         }
 
-        return new ValidateIndexesTaskResult(jobResults, exceptions);
+        return taskResult;
     }
 
     /** {@inheritDoc} */

diff --git a/.../src/main/java/org/apache/ignite/internal/management/cache/ValidateIndexesTaskResult.java b/.../src/main/java/org/apache/ignite/internal/management/cache/ValidateIndexesTaskResult.java
@@ -17,14 +17,21 @@
 
 package org.apache.ignite.internal.management.cache;
 
+import java.io.Externalizable;
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.Objects;
 import java.util.UUID;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorDataTransferObject;
+import org.jetbrains.annotations.Nullable;
 
 /**
  *
@@ -34,39 +41,50 @@ public class ValidateIndexesTaskResult extends VisorDataTransferObject {
     private static final long serialVersionUID = 0L;
 
     /** Exceptions. */
-    private Map<UUID, Exception> exceptions;
+    private @Nullable Map<NodeInfo, Exception> exceptions;
 
     /** Results from cluster. */
-    private Map<UUID, ValidateIndexesJobResult> results;
+    private @Nullable Map<NodeInfo, ValidateIndexesJobResult> results;
 
     /**
-     * @param results Results.
-     * @param exceptions Exceptions.
+     * Adds single node job result.
      */
-    public ValidateIndexesTaskResult(Map<UUID, ValidateIndexesJobResult> results,
-        Map<UUID, Exception> exceptions) {
-        this.exceptions = exceptions;
-        this.results = results;
+    public void addResult(ClusterNode clusterNode, ValidateIndexesJobResult jobResult) {
+        if (results == null)
+            results = new HashMap<>();
+
+        results.put(new NodeInfo(clusterNode.id(), clusterNode.consistentId()), jobResult);
     }
 
     /**
-     * For externalization only.
+     * @return Single node job result or {@code null} if not found.
      */
-    public ValidateIndexesTaskResult() {
+    public @Nullable ValidateIndexesJobResult jobResult(ClusterNode clusterNode) {
+        return results().get(new NodeInfo(clusterNode.id(), clusterNode.consistentId()));
     }
 
     /**
-     * @return Exceptions.
+     * @return Results from cluster.
      */
-    public Map<UUID, Exception> exceptions() {
-        return exceptions;
+    public Map<NodeInfo, ValidateIndexesJobResult> results() {
+        return results == null ? Collections.emptyMap() : results;
     }
 
     /**
-     * @return Results from cluster.
+     * Adds single node job failure.
      */
-    public Map<UUID, ValidateIndexesJobResult> results() {
-        return results;
+    public void addException(ClusterNode clusterNode, Exception exception) {
+        if (exceptions == null)
+            exceptions = new HashMap<>();
+
+        exceptions.put(new NodeInfo(clusterNode.id(), clusterNode.consistentId()), exception);
+    }
+
+    /**
+     * @return Exceptions.
+     */
+    public Map<NodeInfo, Exception> exceptions() {
+        return exceptions == null ? Collections.emptyMap() : exceptions;
     }
 
     /** {@inheritDoc} */
@@ -85,4 +103,54 @@ public Map<UUID, ValidateIndexesJobResult> results() {
     @Override public String toString() {
         return S.toString(ValidateIndexesTaskResult.class, this);
     }
+
+    /**
+     * Holds node id and consistent id.
+     */
+    public static final class NodeInfo implements Serializable {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private final UUID id;
+
+        /** */
+        private final Object consistentId;
+
+        /** */
+        private NodeInfo(UUID id, Object consistentId) {
+            assert consistentId instanceof Serializable || consistentId instanceof Externalizable;
+
+            this.id = id;
+            this.consistentId = consistentId;
+        }
+
+        /** */
+        public UUID id() {
+            return id;
+        }
+
+        /** */
+        public Object consistentId() {
+            return consistentId;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            NodeInfo id1 = (NodeInfo)o;
+
+            return id.equals(id1.id) && consistentId.equals(id1.consistentId);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return Objects.hash(id, consistentId);
+        }
+    }
 }
diff --git a/.../internal/processors/cache/persistence/snapshot/SnapshotPartitionsQuickVerifyHandler.java b/.../internal/processors/cache/persistence/snapshot/SnapshotPartitionsQuickVerifyHandler.java
@@ -89,7 +89,8 @@ public SnapshotPartitionsQuickVerifyHandler(GridCacheSharedContext<?, ?> cctx) {
             if (other == null)
                 return;
 
-            if (val.size() != other.size() || !Objects.equals(val.updateCounter(), other.updateCounter()))
+            if ((!val.hasExpiringEntries() && !other.hasExpiringEntries() && val.size() != other.size())
+                || !Objects.equals(val.updateCounter(), other.updateCounter()))
                 wrnGrps.add(part.groupId());
         }));