From 4c68df58022e1e41681ebe2e13126facc5963ae7 Mon Sep 17 00:00:00 2001 From: Diogo Santos Date: Tue, 27 Aug 2024 14:43:42 +0100 Subject: [PATCH] ID-1570 - Fix dice-where and update dce-id to use the latest version (#113) * ID-1570 - Alternative way; * ID-1570 - working version; * ID-1570 - unnecessary code; * ID-1570 - unnecessary imports; --- .../downloader/destination/FileAcceptor.java | 4 +- .../destination/local/LocalFileAcceptor.java | 6 +- .../stream/StreamWithMD5Decorator.java | 19 ++- .../local/LocalFileAcceptorTest.java | 131 +++++++++++------- .../stream/StreamWithMD5DecoratorTest.java | 81 +++++++++++ .../test/resources/maxmind/maxmind-city-1.zip | Bin 0 -> 1909 bytes 6 files changed, 182 insertions(+), 59 deletions(-) create mode 100644 dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5DecoratorTest.java create mode 100644 dice-where-downloader-lib/src/test/resources/maxmind/maxmind-city-1.zip diff --git a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/FileAcceptor.java b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/FileAcceptor.java index e741f25..565f26e 100644 --- a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/FileAcceptor.java +++ b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/FileAcceptor.java @@ -8,8 +8,8 @@ public interface FileAcceptor { - StreamConsumer getStreamConsumer(MD5Checksum originalFileMd5, Instant originalFileTimestamp, - boolean noMd5Check); + StreamConsumer getStreamConsumer( + MD5Checksum originalFileMd5, Instant originalFileTimestamp, boolean noMd5Check); boolean destinationExists(); diff --git a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptor.java b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptor.java index 71adab3..83d9b3d 100644 --- a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptor.java +++ b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptor.java @@ -73,13 +73,11 @@ public Optional existingFileMd5() { BufferedInputStream bis = new BufferedInputStream(is); StreamWithMD5Decorator md5Is = StreamWithMD5Decorator.of(bis)) { byte[] buffer = new byte[BUFFER]; - while ((md5Is.read(buffer)) != -1) { - } + while ((md5Is.read(buffer)) != -1) {} return Optional.of(md5Is.md5()); } catch (IOException | NoSuchAlgorithmException e) { throw new RuntimeException( - "Could not obtain md5 of the file existing at the target: " + destination, - e); + "Could not obtain md5 of the file existing at the target: " + destination, e); } } return Optional.empty(); diff --git a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5Decorator.java b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5Decorator.java index 99f03a2..1583664 100644 --- a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5Decorator.java +++ b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5Decorator.java @@ -5,6 +5,7 @@ import java.security.DigestInputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.Optional; import javax.xml.bind.annotation.adapters.HexBinaryAdapter; import technology.dice.dicewhere.downloader.md5.MD5Checksum; @@ -12,6 +13,7 @@ public class StreamWithMD5Decorator extends InputStream { private final MessageDigest md5; DigestInputStream inputStream; + private Optional checksum = Optional.empty(); private StreamWithMD5Decorator(DigestInputStream inputStream, MessageDigest md5) { this.inputStream = inputStream; @@ -25,17 +27,26 @@ public static StreamWithMD5Decorator of(InputStream inputStream) throws NoSuchAl } public MD5Checksum md5() { - String hex = (new HexBinaryAdapter()).marshal(this.md5.digest()); - return MD5Checksum.of(hex); + return checksum.orElseGet( + () -> { + String hex = (new HexBinaryAdapter()).marshal(this.md5.digest()); + checksum = Optional.of(MD5Checksum.of(hex)); + return checksum.get(); + }); } @Override public int read() throws IOException { - return this.inputStream.read(); + return inputStream.read(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + return inputStream.read(b, off, len); } @Override public void close() throws IOException { - this.inputStream.close(); + inputStream.close(); } } diff --git a/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptorTest.java b/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptorTest.java index 121d2f7..c84bf7c 100644 --- a/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptorTest.java +++ b/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptorTest.java @@ -33,8 +33,7 @@ public class LocalFileAcceptorTest extends TestCase { private static final int TEST_FILE_SIZE = 1024 * 1024; - @ClassRule - static WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort()); + @ClassRule static WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort()); @BeforeClass public static void beforeClass() { @@ -45,16 +44,22 @@ public static void beforeClass() { public void corruptedFileEmptyPreexistingSet() throws IOException, NoSuchAlgorithmException { Pair tempFile = generateTempFile(); Path destinationDir = Files.createTempDirectory("dice-where"); - IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource( - new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb")); - wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn( - aResponse().withStatus(HttpStatus.SC_OK) - .withHeader("Etag", "aaa") - .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE)) - .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT"))); - wireMockRule.stubFor(WireMock.get(UrlPattern.ANY) - .willReturn(aResponse().withBody( - IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile()))))); + IpInfoSiteSource ipInfoSiteSource = + new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb")); + wireMockRule.stubFor( + WireMock.head(UrlPattern.ANY) + .willReturn( + aResponse() + .withStatus(HttpStatus.SC_OK) + .withHeader("Etag", "aaa") + .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE)) + .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT"))); + wireMockRule.stubFor( + WireMock.get(UrlPattern.ANY) + .willReturn( + aResponse() + .withBody( + IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile()))))); FileInfo fileInfo = ipInfoSiteSource.fileInfo(); ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false); @@ -68,72 +73,100 @@ public void corruptedFilePreexistingSet() throws IOException, NoSuchAlgorithmExc Pair existingFile = generateTempFile(); Path destinationDir = Files.createTempDirectory("dice-where"); Files.copy(existingFile.getLeft(), destinationDir.resolve("existingFile.mdb")); - IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource( - new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb")); - wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn( - aResponse().withStatus(HttpStatus.SC_OK) - .withHeader("Etag", "aaa") - .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE)) - .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT"))); - wireMockRule.stubFor(WireMock.get(UrlPattern.ANY) - .willReturn(aResponse().withBody( - IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile()))))); + IpInfoSiteSource ipInfoSiteSource = + new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb")); + wireMockRule.stubFor( + WireMock.head(UrlPattern.ANY) + .willReturn( + aResponse() + .withStatus(HttpStatus.SC_OK) + .withHeader("Etag", "aaa") + .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE)) + .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT"))); + wireMockRule.stubFor( + WireMock.get(UrlPattern.ANY) + .willReturn( + aResponse() + .withBody( + IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile()))))); FileInfo fileInfo = ipInfoSiteSource.fileInfo(); ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false); assertNotEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat()); assertEquals(Files.list(destinationDir).count(), 1); assertFalse(Files.exists(destinationDir.resolve("file.mdb"))); - assertTrue(Arrays.equals(Files.readAllBytes(existingFile.getLeft()), - Files.readAllBytes(Files.list(destinationDir).findFirst().get()))); + assertTrue( + Arrays.equals( + Files.readAllBytes(existingFile.getLeft()), + Files.readAllBytes(Files.list(destinationDir).findFirst().get()))); } @Test public void goodFileEmptyPreexistingSet() throws IOException, NoSuchAlgorithmException { Pair tempFile = generateTempFile(); Path destinationDir = Files.createTempDirectory("dice-where"); - IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource( - new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb")); - wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn( - aResponse().withStatus(HttpStatus.SC_OK) - .withHeader("Etag", tempFile.getRight()) - .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE)) - .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT"))); - wireMockRule.stubFor(WireMock.get(UrlPattern.ANY) - .willReturn(aResponse().withBody( - IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile()))))); + IpInfoSiteSource ipInfoSiteSource = + new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb")); + wireMockRule.stubFor( + WireMock.head(UrlPattern.ANY) + .willReturn( + aResponse() + .withStatus(HttpStatus.SC_OK) + .withHeader("Etag", tempFile.getRight()) + .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE)) + .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT"))); + wireMockRule.stubFor( + WireMock.get(UrlPattern.ANY) + .willReturn( + aResponse() + .withBody( + IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile()))))); FileInfo fileInfo = ipInfoSiteSource.fileInfo(); ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false); assertEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat()); assertEquals(1, Files.list(destinationDir).count()); - assertTrue(Arrays.equals(Files.readAllBytes(tempFile.getLeft()), - Files.readAllBytes(destinationDir.resolve("file.mdb")))); + assertTrue( + Arrays.equals( + Files.readAllBytes(tempFile.getLeft()), + Files.readAllBytes(destinationDir.resolve("file.mdb")))); } @Test public void goodFilePreexistingSet() throws IOException, NoSuchAlgorithmException { Pair tempFile = generateTempFile(); Pair existingFile = generateTempFile(); + Path destinationDir = Files.createTempDirectory("dice-where"); Files.copy(existingFile.getLeft(), destinationDir.resolve("existingFile.mdb")); - IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource( - new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb")); - wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn( - aResponse().withStatus(HttpStatus.SC_OK) - .withHeader("Etag", tempFile.getRight()) - .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE)) - .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT"))); - wireMockRule.stubFor(WireMock.get(UrlPattern.ANY) - .willReturn(aResponse().withBody( - IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile()))))); + wireMockRule.stubFor( + WireMock.head(UrlPattern.ANY) + .willReturn( + aResponse() + .withStatus(HttpStatus.SC_OK) + .withHeader("Etag", tempFile.getRight()) + .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE)) + .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT"))); + + wireMockRule.stubFor( + WireMock.get(UrlPattern.ANY) + .willReturn( + aResponse() + .withBody( + IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile()))))); + + IpInfoSiteSource ipInfoSiteSource = + new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb")); FileInfo fileInfo = ipInfoSiteSource.fileInfo(); + ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false); assertEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat()); assertEquals(2, Files.list(destinationDir).count()); - assertTrue(Arrays.equals(Files.readAllBytes(tempFile.getLeft()), - Files.readAllBytes(destinationDir.resolve("file.mdb")))); + assertTrue( + Arrays.equals( + Files.readAllBytes(tempFile.getLeft()), + Files.readAllBytes(destinationDir.resolve("file.mdb")))); } private Pair generateTempFile() throws IOException, NoSuchAlgorithmException { @@ -145,4 +178,4 @@ private Pair generateTempFile() throws IOException, NoSuchAlgorith String hex = (new HexBinaryAdapter()).marshal(md.digest(contents)); return Pair.of(tempFile, hex); } -} \ No newline at end of file +} diff --git a/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5DecoratorTest.java b/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5DecoratorTest.java new file mode 100644 index 0000000..d3a3de4 --- /dev/null +++ b/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5DecoratorTest.java @@ -0,0 +1,81 @@ +package technology.dice.dicewhere.downloader.stream; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; + +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.github.tomakehurst.wiremock.matching.UrlPattern; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.Charset; +import java.nio.file.Path; +import java.security.NoSuchAlgorithmException; +import junit.framework.TestCase; +import org.apache.commons.io.IOUtils; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.internal.runners.JUnit4ClassRunner; +import org.junit.runner.RunWith; + +@RunWith(JUnit4ClassRunner.class) +public class StreamWithMD5DecoratorTest extends TestCase { + + private static final String PATH = "/maxmind/maxmind-city-1.zip"; + + @ClassRule static WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort()); + + @BeforeClass + public static void beforeClass() { + wireMockRule.start(); + } + + @Test + public void shouldSuccessfullyReadAndCalculateDigestOfStream() + throws IOException, NoSuchAlgorithmException, URISyntaxException { + Path path = Path.of(getClass().getResource(PATH).toURI()); + StreamWithMD5Decorator is = StreamWithMD5Decorator.of(new FileInputStream(path.toFile())); + + // Exhaust stream for the complete hash digest + byte[] buffer = new byte[8192]; + while ((is.read(buffer)) != -1) {} + + String first = is.md5().stringFormat(); + IOUtils.toString(is, Charset.defaultCharset()); + + // Assert the Stream Hash before and after + assertEquals(first, is.md5().stringFormat()); + assertEquals(first, "9c7dd68c8352f1c59a33efe0dca04f06"); + } + + @Test + public void shouldSuccessfullyReadAndCalculateDigestOfStreamFromHttp() + throws IOException, NoSuchAlgorithmException, URISyntaxException { + Path path = Path.of(getClass().getResource(PATH).toURI()); + + wireMockRule.stubFor( + WireMock.get(UrlPattern.ANY) + .willReturn( + aResponse().withBody(IOUtils.toByteArray(new FileInputStream(path.toFile()))))); + + URL url = new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + + StreamWithMD5Decorator is = StreamWithMD5Decorator.of(connection.getInputStream()); + // Exhaust stream for the complete hash digest + byte[] buffer = new byte[8192]; + while ((is.read(buffer)) != -1) {} + + String first = is.md5().stringFormat(); + // Read from the stream + IOUtils.toString(is, Charset.defaultCharset()); + + // Assert the Stream Hash before and after + assertEquals(first, is.md5().stringFormat()); + assertEquals(first, "9c7dd68c8352f1c59a33efe0dca04f06"); + } +} diff --git a/dice-where-downloader-lib/src/test/resources/maxmind/maxmind-city-1.zip b/dice-where-downloader-lib/src/test/resources/maxmind/maxmind-city-1.zip new file mode 100644 index 0000000000000000000000000000000000000000..a2aabb4fc41c444614104bbb9aa453ba72f39b70 GIT binary patch literal 1909 zcmbuAc{CgN7RMvRQmRXcz0_8*q~kEPrHHkJp!SwTP)msvHI}xCN}G0UBX*^zP^wHU zZKfeghg2d_9eJ<3_QhURYHYRCQZL?lbKacy=A8Ls?mfSA?>)bBzMnroo=8zKIe?_F z%QL+E0RO3D!U+aw0C3n8Ty$KxQCRfVR3l>;4i_Hk>vI?&5%Fj#Y=F5G#umP$6ls8n z*oFuIAPfQkxI6o{1=)BjNuY+oEqnKL^#n`ZHk#{M^ zMlg_gVz*LCI@77va8)-jyu`W8`JD0o_*tr)mW5V%j!)O%N(Pr6Jwtee~g0{7FclCR?5O zvZUGDA`pBD@a5PJ_Xv2QlejBg^=ll(A&FE&B(87sm&Tc0-@1Dd&8kQ|2~>%{iOp)y zC*zeNUB{6}h&>;$NA>-)wgO(usZ&WW;fW{K!EzP#s4Ay28w@GPVj|J{GT8!OF)LMZ zUT__AvxD8P%wxi1Y~R=t6dS0z1MPKu)7)evcIzEC8`)nb!lY-&n3!~vVjsChCERCwH5-4wk{sqJx48(hko#=`y*K5ovWIm=w zU6EAt_V42a<$wgNfqQ>1dQA%*0Bc~{yK1L>9k2@i8J$72(aD!tMIX;OK7@z`lWms= z0{jaFnbU73Mfq-Z}$^q!2def`8mkCO?}ElPqonEmC&_+GzQyS((N0CSrjN_9V5@vPmNod0($GH#x-Lvy>*Sn?F+-oy>^ssL z=VgJot#dh!wRctd6QOSj@~(ppwOVKBljXacPaKsouxf4`PEsizS7@@kFdyi8jzTqR z);nY(v8<)-7WPxbUG5Hwv)%Y!k{@MY@RUx)*STJ2tW5gK)(bvssH;(}fSgvt$55PG zW@z)_G-6-%C`sU69sl-|Ptsk>ETE;mq1|NK$j7NG=sT1c{G7y%_@NJNG<-?JQ}Hvc zpN5ZiP;8WY6(h;vlD8nkxT#Sv&QibaY4wvIU-ghj)JNdpkN1-m9PC)>Ujr;Q?4sT8 zbt^6m0%c1tv=+Z_n(!yLkR*^bY5-dQ#k}Qt(&GfxZ&0u$bkH5-&#E1-jDiy)NfjLZ z{vnyNUg$PR!}j#WDZShp#oTg5a=UiYUTmOOsR%|q_ef(}DtBQBt-IkUCcwK?&mnKo z(vNh(67qE-^(t`aCXz$IEUN80X04m!vfq(B2qf%NWZ*KZB7wnz$2k8bQS}@IsJSZ?!h+yv%a&Jl~ACS0PnB zVo&=qvD8WT2jX`-RZY)zXBjG@v@h$iE8X69=4JaOQUvLKqcvHdSl!` zrhD6Qb=NUR^evh9^&pfr+gMv2{wEkeLiE zN53iU8Jhz;+zG|awSfSD0|$5_MSyam|8>&u&2iwQ-->?oVEZ#g{DGnV7w~)dfq;H1 a`pv7t^{?s36A2PO$O8&{P&o7def