From ad090ca3b3563a79cb25ebd1ddc61d960ce188b8 Mon Sep 17 00:00:00 2001 From: nkumar2 Date: Tue, 9 Jan 2024 13:21:57 +0000 Subject: [PATCH 1/2] recovering continuous id blocks one by one --- .../MonotonicAccessionGenerator.java | 12 +++++- .../service/ContiguousIdBlockService.java | 1 + ...ataRepositoryMonotonicDatabaseService.java | 16 ++++---- ...icAccessioningWithAlternateRangesTest.java | 39 +++++++++++++++++++ 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/accession-commons-monotonic-generator-jpa/src/main/java/uk/ac/ebi/ampt2d/commons/accession/generators/monotonic/MonotonicAccessionGenerator.java b/accession-commons-monotonic-generator-jpa/src/main/java/uk/ac/ebi/ampt2d/commons/accession/generators/monotonic/MonotonicAccessionGenerator.java index 73c8a88c..9abefd59 100644 --- a/accession-commons-monotonic-generator-jpa/src/main/java/uk/ac/ebi/ampt2d/commons/accession/generators/monotonic/MonotonicAccessionGenerator.java +++ b/accession-commons-monotonic-generator-jpa/src/main/java/uk/ac/ebi/ampt2d/commons/accession/generators/monotonic/MonotonicAccessionGenerator.java @@ -27,9 +27,9 @@ import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.MonotonicDatabaseService; import uk.ac.ebi.ampt2d.commons.accession.utils.ExponentialBackOff; -import uk.ac.ebi.ampt2d.commons.accession.utils.exceptions.ExponentialBackOffMaxRetriesRuntimeException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -52,7 +52,15 @@ public MonotonicAccessionGenerator(String categoryId, ContiguousIdBlockService contiguousIdBlockService, MonotonicDatabaseService databaseService) { this(categoryId, applicationInstanceId, contiguousIdBlockService); - recoverState(databaseService.getAccessionsInRanges(getAvailableRanges())); + // As we are going through the available ranges and at the same time we are also going to manipulate/update them + // Need to make a copy of the original for iteration to avoid ConcurrentModificationException + MonotonicRangePriorityQueue copyOfAvailableRanges = new MonotonicRangePriorityQueue(); + for (MonotonicRange range : getAvailableRanges()) { + copyOfAvailableRanges.offer(range); + } + for (MonotonicRange monotonicRange : copyOfAvailableRanges) { + recoverState(databaseService.getAccessionsInRanges(Collections.singletonList(monotonicRange))); + } } public MonotonicAccessionGenerator(String categoryId, diff --git a/accession-commons-monotonic-generator-jpa/src/main/java/uk/ac/ebi/ampt2d/commons/accession/persistence/jpa/monotonic/service/ContiguousIdBlockService.java b/accession-commons-monotonic-generator-jpa/src/main/java/uk/ac/ebi/ampt2d/commons/accession/persistence/jpa/monotonic/service/ContiguousIdBlockService.java index 9030bb16..2dc38869 100644 --- a/accession-commons-monotonic-generator-jpa/src/main/java/uk/ac/ebi/ampt2d/commons/accession/persistence/jpa/monotonic/service/ContiguousIdBlockService.java +++ b/accession-commons-monotonic-generator-jpa/src/main/java/uk/ac/ebi/ampt2d/commons/accession/persistence/jpa/monotonic/service/ContiguousIdBlockService.java @@ -48,6 +48,7 @@ public ContiguousIdBlockService(ContiguousIdBlockRepository repository, Map blocks) { repository.saveAll(blocks); + entityManager.flush(); } @Transactional(isolation = Isolation.SERIALIZABLE) diff --git a/accession-commons-monotonic-generator-jpa/src/main/java/uk/ac/ebi/ampt2d/commons/accession/service/BasicSpringDataRepositoryMonotonicDatabaseService.java b/accession-commons-monotonic-generator-jpa/src/main/java/uk/ac/ebi/ampt2d/commons/accession/service/BasicSpringDataRepositoryMonotonicDatabaseService.java index 7bcc6207..f078c7c5 100644 --- a/accession-commons-monotonic-generator-jpa/src/main/java/uk/ac/ebi/ampt2d/commons/accession/service/BasicSpringDataRepositoryMonotonicDatabaseService.java +++ b/accession-commons-monotonic-generator-jpa/src/main/java/uk/ac/ebi/ampt2d/commons/accession/service/BasicSpringDataRepositoryMonotonicDatabaseService.java @@ -17,7 +17,6 @@ */ package uk.ac.ebi.ampt2d.commons.accession.service; -import uk.ac.ebi.ampt2d.commons.accession.core.DatabaseService; import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicRange; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.MonotonicDatabaseService; @@ -31,6 +30,7 @@ import java.util.Collection; import java.util.List; import java.util.function.Function; +import java.util.stream.Collectors; /** * Basic implementation of {@link MonotonicDatabaseService} that requires a Spring Data repository that extends @@ -62,18 +62,18 @@ public BasicSpringDataRepositoryMonotonicDatabaseService( @Override public long[] getAccessionsInRanges(Collection ranges) { - List> accessionsInRanges = new ArrayList<>(); + List accessionsInDB = new ArrayList<>(); for (MonotonicRange potentiallyBigRange : ranges) { for (MonotonicRange range : ensureRangeMaxSize(potentiallyBigRange, MAX_RANGE_SIZE)) { - accessionsInRanges.addAll( + List> accessionsInRange = repository.findByAccessionGreaterThanEqualAndAccessionLessThanEqual(range.getStart(), - range.getEnd())); + range.getEnd()); + + accessionsInDB.addAll(accessionsInRange.stream().map(ap -> ap.getAccession()).collect(Collectors.toList())); } } - long[] accessionArray = new long[accessionsInRanges.size()]; - for (int i = 0; i < accessionsInRanges.size(); i++) { - accessionArray[i] = accessionsInRanges.get(i).getAccession(); - } + + long[] accessionArray = accessionsInDB.stream().mapToLong(l -> l).toArray(); return accessionArray; } diff --git a/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithAlternateRangesTest.java b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithAlternateRangesTest.java index f234888b..08429b8a 100644 --- a/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithAlternateRangesTest.java +++ b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithAlternateRangesTest.java @@ -29,15 +29,18 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicAccessionGenerator; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; +import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService; import uk.ac.ebi.ampt2d.commons.accession.service.BasicSpringDataRepositoryMonotonicDatabaseService; import uk.ac.ebi.ampt2d.test.configuration.TestMonotonicDatabaseServiceTestConfiguration; import uk.ac.ebi.ampt2d.test.models.TestModel; import uk.ac.ebi.ampt2d.test.persistence.TestMonotonicEntity; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.LongStream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -62,6 +65,42 @@ public void testUnknownCategory() throws AccessionCouldNotBeGeneratedException { .getOrCreate(getObjectsForAccessionsInRange(1, 10)); } + @Test + public void testRecoverState() { + String categoryId = "eva_2"; + String instanceId2 = "test-instance_2"; + + // create 3 un-complete contiguous id blocks of size 10 + // block-1 : (100 to 109), block-2 : (110 to 119), block-3 : (120 to 129) + List uncompletedBlocks = new ArrayList<>(); + uncompletedBlocks.add(new ContiguousIdBlock(categoryId, instanceId2, 100, 10)); + uncompletedBlocks.add(new ContiguousIdBlock(categoryId, instanceId2, 110, 10)); + uncompletedBlocks.add(new ContiguousIdBlock(categoryId, instanceId2, 120, 10)); + contiguousIdBlockService.save(uncompletedBlocks); + + assertEquals(3, contiguousIdBlockService.getUncompletedBlocksByCategoryIdAndApplicationInstanceIdOrderByEndAsc(categoryId, instanceId2).size()); + + // create and save accessions in db (100 to 124) + List> accessions = LongStream.range(100l, 125l) + .boxed() + .map(longAcc -> new AccessionWrapper<>(longAcc, "hash" + longAcc, TestModel.of("test-obj-" + longAcc))) + .collect(Collectors.toList()); + databaseService.save(accessions); + + // run recover blocks + getGenerator(categoryId, instanceId2); + + // As we have already saved accessions in db from 100 to 124, the status should be + // block-1 (100 to 109) : fully complete + // block-2 (110 to 119) : fully complete + // block-3 (120 to 124) : partially complete + assertEquals(1, contiguousIdBlockService.getUncompletedBlocksByCategoryIdAndApplicationInstanceIdOrderByEndAsc(categoryId, instanceId2).size()); + ContiguousIdBlock uncompletedBlock = contiguousIdBlockService.getUncompletedBlocksByCategoryIdAndApplicationInstanceIdOrderByEndAsc(categoryId, instanceId2).get(0); + assertEquals(120l, uncompletedBlock.getFirstValue()); + assertEquals(129l, uncompletedBlock.getLastValue()); + assertEquals(124l, uncompletedBlock.getLastCommitted()); + } + @Test public void testAlternateRangesWithDifferentGenerators() throws AccessionCouldNotBeGeneratedException { /* blockStartValue= 0, blockSize= 10 , nextBlockInterval= 20 From 5c6382cfac6bf576f6c3fb6890b981e7da2fb60d Mon Sep 17 00:00:00 2001 From: nkumar2 Date: Tue, 9 Jan 2024 13:35:36 +0000 Subject: [PATCH 2/2] test available ranges after recovery --- ...asicMonotonicAccessioningWithAlternateRangesTest.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithAlternateRangesTest.java b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithAlternateRangesTest.java index 08429b8a..9ba72c5a 100644 --- a/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithAlternateRangesTest.java +++ b/accession-commons-monotonic-generator-jpa/src/test/java/uk/ac/ebi/ampt2d/commons/accession/core/BasicMonotonicAccessioningWithAlternateRangesTest.java @@ -28,6 +28,8 @@ import uk.ac.ebi.ampt2d.commons.accession.core.models.AccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.core.models.GetOrCreateAccessionWrapper; import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicAccessionGenerator; +import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicRange; +import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicRangePriorityQueue; import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService; @@ -88,7 +90,7 @@ public void testRecoverState() { databaseService.save(accessions); // run recover blocks - getGenerator(categoryId, instanceId2); + MonotonicAccessionGenerator generator = getGenerator(categoryId, instanceId2); // As we have already saved accessions in db from 100 to 124, the status should be // block-1 (100 to 109) : fully complete @@ -99,6 +101,11 @@ public void testRecoverState() { assertEquals(120l, uncompletedBlock.getFirstValue()); assertEquals(129l, uncompletedBlock.getLastValue()); assertEquals(124l, uncompletedBlock.getLastCommitted()); + + MonotonicRangePriorityQueue availableRanges = generator.getAvailableRanges(); + assertEquals(1, availableRanges.size()); + assertEquals(125l, availableRanges.peek().getStart()); + assertEquals(129l, availableRanges.peek().getEnd()); } @Test