Merge pull request #6897 from popa-raluca/dedup-connector

Deduplication - performance fixes
odpi · Sep 26, 2022 · dad5789 · dad5789
2 parents a21908d + 234b38b
commit dad5789
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 9 deletions.
diff --git a/...rs/governanceactions/remediation/QualifiedNamePeerDuplicateGovernanceActionConnector.java b/...rs/governanceactions/remediation/QualifiedNamePeerDuplicateGovernanceActionConnector.java
@@ -78,6 +78,7 @@ public void start() throws ConnectorCheckedException {
                                 null, null, null, null, null, true);
                         outputGuards.add(QualifiedNamePeerDuplicateGovernanceActionProvider.DUPLICATE_ASSIGNED_GUARD);
                         completionStatus = CompletionStatus.ACTIONED;
+                        break;
                     }
                 }
             }

diff --git a/.../java/org/odpi/openmetadata/commonservices/repositoryhandler/DuplicateEntityIterator.java b/.../java/org/odpi/openmetadata/commonservices/repositoryhandler/DuplicateEntityIterator.java
@@ -10,6 +10,7 @@
 import org.odpi.openmetadata.repositoryservices.connectors.stores.metadatacollectionstore.properties.instances.Classification;
 import org.odpi.openmetadata.repositoryservices.connectors.stores.metadatacollectionstore.properties.instances.EntityDetail;
 import org.odpi.openmetadata.repositoryservices.connectors.stores.metadatacollectionstore.properties.instances.EntityProxy;
+import org.odpi.openmetadata.repositoryservices.connectors.stores.metadatacollectionstore.properties.instances.InstanceHeader;
 import org.odpi.openmetadata.repositoryservices.connectors.stores.metadatacollectionstore.properties.instances.Relationship;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -19,6 +20,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * DuplicateEntityIterator retrieves the list of entities that need to be processed for a specific entity.
@@ -180,6 +182,7 @@ private void getNextEntity() throws InvalidParameterException,
                                 {
                                     log.debug("KnownDuplicate classification detected");
                                     deduplicationNeeded = true;
+                                    break;
                                 }
                             }
                         }
@@ -269,17 +272,23 @@ private void getNextEntity() throws InvalidParameterException,
                                      */
                                     final String guidParameterName = "peerProxy.getGUID()";
 
-                                    EntityDetail peerEntity = repositoryHandler.validateEntityGUID(userId,
-                                                                                                   peerProxy.getGUID(),
-                                                                                                   guidParameterName,
-                                                                                                   entityTypeName,
-                                                                                                   methodName);
-
                                     /*
-                                     * Save the entity for later processing since it may link to peer entities that the current processing entity
-                                     * does not know about.
+                                     * Filter out the peers that were already added to the list of unprocessed peers.
                                      */
-                                    unprocessedPeers.add(peerEntity);
+                                    if(!unprocessedPeers.stream().map(InstanceHeader::getGUID).collect(Collectors.toList()).contains(peerProxy.getGUID()))
+                                    {
+                                        EntityDetail peerEntity = repositoryHandler.validateEntityGUID(userId,
+                                                                                                       peerProxy.getGUID(),
+                                                                                                       guidParameterName,
+                                                                                                       entityTypeName,
+                                                                                                       methodName);
+
+                                        /*
+                                        * Save the entity for later processing since it may link to peer entities that the current processing entity
+                                        * does not know about.
+                                        */
+                                        unprocessedPeers.add(peerEntity);
+                                    }
                                 }
                             }
                         }
-Original file line number
+Diff line change
@@ Expand Up / @@ -78,6 +78,7 @@ public void start() throws ConnectorCheckedException { @@
                                     null, null, null, null, null, true);
                             outputGuards.add(QualifiedNamePeerDuplicateGovernanceActionProvider.DUPLICATE_ASSIGNED_GUARD);
                             completionStatus = CompletionStatus.ACTIONED;
+                            break;
                         }
                     }
                 }
@@ Expand Down @@