Skip to content

Commit

Permalink
ID-1570 - Fix dice-where and update dce-id to use the latest version (#…
Browse files Browse the repository at this point in the history
…113)

* ID-1570 - Alternative way;

* ID-1570 - working version;

* ID-1570 - unnecessary code;

* ID-1570 - unnecessary imports;
  • Loading branch information
snackk authored Aug 27, 2024
1 parent 6703b9d commit 4c68df5
Show file tree
Hide file tree
Showing 6 changed files with 182 additions and 59 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@

public interface FileAcceptor<T> {

StreamConsumer<T> getStreamConsumer(MD5Checksum originalFileMd5, Instant originalFileTimestamp,
boolean noMd5Check);
StreamConsumer<T> getStreamConsumer(
MD5Checksum originalFileMd5, Instant originalFileTimestamp, boolean noMd5Check);

boolean destinationExists();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,13 +73,11 @@ public Optional<MD5Checksum> existingFileMd5() {
BufferedInputStream bis = new BufferedInputStream(is);
StreamWithMD5Decorator md5Is = StreamWithMD5Decorator.of(bis)) {
byte[] buffer = new byte[BUFFER];
while ((md5Is.read(buffer)) != -1) {
}
while ((md5Is.read(buffer)) != -1) {}
return Optional.of(md5Is.md5());
} catch (IOException | NoSuchAlgorithmException e) {
throw new RuntimeException(
"Could not obtain md5 of the file existing at the target: " + destination,
e);
"Could not obtain md5 of the file existing at the target: " + destination, e);
}
}
return Optional.empty();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Optional;
import javax.xml.bind.annotation.adapters.HexBinaryAdapter;
import technology.dice.dicewhere.downloader.md5.MD5Checksum;

public class StreamWithMD5Decorator extends InputStream {

private final MessageDigest md5;
DigestInputStream inputStream;
private Optional<MD5Checksum> checksum = Optional.empty();

private StreamWithMD5Decorator(DigestInputStream inputStream, MessageDigest md5) {
this.inputStream = inputStream;
Expand All @@ -25,17 +27,26 @@ public static StreamWithMD5Decorator of(InputStream inputStream) throws NoSuchAl
}

public MD5Checksum md5() {
String hex = (new HexBinaryAdapter()).marshal(this.md5.digest());
return MD5Checksum.of(hex);
return checksum.orElseGet(
() -> {
String hex = (new HexBinaryAdapter()).marshal(this.md5.digest());
checksum = Optional.of(MD5Checksum.of(hex));
return checksum.get();
});
}

@Override
public int read() throws IOException {
return this.inputStream.read();
return inputStream.read();
}

@Override
public int read(byte[] b, int off, int len) throws IOException {
return inputStream.read(b, off, len);
}

@Override
public void close() throws IOException {
this.inputStream.close();
inputStream.close();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,7 @@
public class LocalFileAcceptorTest extends TestCase {

private static final int TEST_FILE_SIZE = 1024 * 1024;
@ClassRule
static WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort());
@ClassRule static WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort());

@BeforeClass
public static void beforeClass() {
Expand All @@ -45,16 +44,22 @@ public static void beforeClass() {
public void corruptedFileEmptyPreexistingSet() throws IOException, NoSuchAlgorithmException {
Pair<Path, String> tempFile = generateTempFile();
Path destinationDir = Files.createTempDirectory("dice-where");
IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource(
new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn(
aResponse().withStatus(HttpStatus.SC_OK)
.withHeader("Etag", "aaa")
.withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
.withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
wireMockRule.stubFor(WireMock.get(UrlPattern.ANY)
.willReturn(aResponse().withBody(
IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
IpInfoSiteSource ipInfoSiteSource =
new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
wireMockRule.stubFor(
WireMock.head(UrlPattern.ANY)
.willReturn(
aResponse()
.withStatus(HttpStatus.SC_OK)
.withHeader("Etag", "aaa")
.withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
.withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
wireMockRule.stubFor(
WireMock.get(UrlPattern.ANY)
.willReturn(
aResponse()
.withBody(
IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));

FileInfo fileInfo = ipInfoSiteSource.fileInfo();
ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false);
Expand All @@ -68,72 +73,100 @@ public void corruptedFilePreexistingSet() throws IOException, NoSuchAlgorithmExc
Pair<Path, String> existingFile = generateTempFile();
Path destinationDir = Files.createTempDirectory("dice-where");
Files.copy(existingFile.getLeft(), destinationDir.resolve("existingFile.mdb"));
IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource(
new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn(
aResponse().withStatus(HttpStatus.SC_OK)
.withHeader("Etag", "aaa")
.withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
.withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
wireMockRule.stubFor(WireMock.get(UrlPattern.ANY)
.willReturn(aResponse().withBody(
IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
IpInfoSiteSource ipInfoSiteSource =
new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
wireMockRule.stubFor(
WireMock.head(UrlPattern.ANY)
.willReturn(
aResponse()
.withStatus(HttpStatus.SC_OK)
.withHeader("Etag", "aaa")
.withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
.withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
wireMockRule.stubFor(
WireMock.get(UrlPattern.ANY)
.willReturn(
aResponse()
.withBody(
IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));

FileInfo fileInfo = ipInfoSiteSource.fileInfo();
ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false);
assertNotEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat());
assertEquals(Files.list(destinationDir).count(), 1);
assertFalse(Files.exists(destinationDir.resolve("file.mdb")));
assertTrue(Arrays.equals(Files.readAllBytes(existingFile.getLeft()),
Files.readAllBytes(Files.list(destinationDir).findFirst().get())));
assertTrue(
Arrays.equals(
Files.readAllBytes(existingFile.getLeft()),
Files.readAllBytes(Files.list(destinationDir).findFirst().get())));
}

@Test
public void goodFileEmptyPreexistingSet() throws IOException, NoSuchAlgorithmException {
Pair<Path, String> tempFile = generateTempFile();
Path destinationDir = Files.createTempDirectory("dice-where");
IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource(
new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn(
aResponse().withStatus(HttpStatus.SC_OK)
.withHeader("Etag", tempFile.getRight())
.withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
.withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
wireMockRule.stubFor(WireMock.get(UrlPattern.ANY)
.willReturn(aResponse().withBody(
IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
IpInfoSiteSource ipInfoSiteSource =
new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
wireMockRule.stubFor(
WireMock.head(UrlPattern.ANY)
.willReturn(
aResponse()
.withStatus(HttpStatus.SC_OK)
.withHeader("Etag", tempFile.getRight())
.withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
.withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
wireMockRule.stubFor(
WireMock.get(UrlPattern.ANY)
.willReturn(
aResponse()
.withBody(
IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));

FileInfo fileInfo = ipInfoSiteSource.fileInfo();
ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false);
assertEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat());
assertEquals(1, Files.list(destinationDir).count());
assertTrue(Arrays.equals(Files.readAllBytes(tempFile.getLeft()),
Files.readAllBytes(destinationDir.resolve("file.mdb"))));
assertTrue(
Arrays.equals(
Files.readAllBytes(tempFile.getLeft()),
Files.readAllBytes(destinationDir.resolve("file.mdb"))));
}

@Test
public void goodFilePreexistingSet() throws IOException, NoSuchAlgorithmException {
Pair<Path, String> tempFile = generateTempFile();
Pair<Path, String> existingFile = generateTempFile();

Path destinationDir = Files.createTempDirectory("dice-where");
Files.copy(existingFile.getLeft(), destinationDir.resolve("existingFile.mdb"));
IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource(
new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn(
aResponse().withStatus(HttpStatus.SC_OK)
.withHeader("Etag", tempFile.getRight())
.withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
.withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
wireMockRule.stubFor(WireMock.get(UrlPattern.ANY)
.willReturn(aResponse().withBody(
IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));

wireMockRule.stubFor(
WireMock.head(UrlPattern.ANY)
.willReturn(
aResponse()
.withStatus(HttpStatus.SC_OK)
.withHeader("Etag", tempFile.getRight())
.withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
.withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));

wireMockRule.stubFor(
WireMock.get(UrlPattern.ANY)
.willReturn(
aResponse()
.withBody(
IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));

IpInfoSiteSource ipInfoSiteSource =
new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
FileInfo fileInfo = ipInfoSiteSource.fileInfo();

ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false);
assertEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat());
assertEquals(2, Files.list(destinationDir).count());
assertTrue(Arrays.equals(Files.readAllBytes(tempFile.getLeft()),
Files.readAllBytes(destinationDir.resolve("file.mdb"))));
assertTrue(
Arrays.equals(
Files.readAllBytes(tempFile.getLeft()),
Files.readAllBytes(destinationDir.resolve("file.mdb"))));
}

private Pair<Path, String> generateTempFile() throws IOException, NoSuchAlgorithmException {
Expand All @@ -145,4 +178,4 @@ private Pair<Path, String> generateTempFile() throws IOException, NoSuchAlgorith
String hex = (new HexBinaryAdapter()).marshal(md.digest(contents));
return Pair.of(tempFile, hex);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package technology.dice.dicewhere.downloader.stream;

import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;

import com.github.tomakehurst.wiremock.client.WireMock;
import com.github.tomakehurst.wiremock.junit.WireMockRule;
import com.github.tomakehurst.wiremock.matching.UrlPattern;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.file.Path;
import java.security.NoSuchAlgorithmException;
import junit.framework.TestCase;
import org.apache.commons.io.IOUtils;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.internal.runners.JUnit4ClassRunner;
import org.junit.runner.RunWith;

@RunWith(JUnit4ClassRunner.class)
public class StreamWithMD5DecoratorTest extends TestCase {

private static final String PATH = "/maxmind/maxmind-city-1.zip";

@ClassRule static WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort());

@BeforeClass
public static void beforeClass() {
wireMockRule.start();
}

@Test
public void shouldSuccessfullyReadAndCalculateDigestOfStream()
throws IOException, NoSuchAlgorithmException, URISyntaxException {
Path path = Path.of(getClass().getResource(PATH).toURI());
StreamWithMD5Decorator is = StreamWithMD5Decorator.of(new FileInputStream(path.toFile()));

// Exhaust stream for the complete hash digest
byte[] buffer = new byte[8192];
while ((is.read(buffer)) != -1) {}

String first = is.md5().stringFormat();
IOUtils.toString(is, Charset.defaultCharset());

// Assert the Stream Hash before and after
assertEquals(first, is.md5().stringFormat());
assertEquals(first, "9c7dd68c8352f1c59a33efe0dca04f06");
}

@Test
public void shouldSuccessfullyReadAndCalculateDigestOfStreamFromHttp()
throws IOException, NoSuchAlgorithmException, URISyntaxException {
Path path = Path.of(getClass().getResource(PATH).toURI());

wireMockRule.stubFor(
WireMock.get(UrlPattern.ANY)
.willReturn(
aResponse().withBody(IOUtils.toByteArray(new FileInputStream(path.toFile())))));

URL url = new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb");
HttpURLConnection connection = (HttpURLConnection) url.openConnection();

StreamWithMD5Decorator is = StreamWithMD5Decorator.of(connection.getInputStream());
// Exhaust stream for the complete hash digest
byte[] buffer = new byte[8192];
while ((is.read(buffer)) != -1) {}

String first = is.md5().stringFormat();
// Read from the stream
IOUtils.toString(is, Charset.defaultCharset());

// Assert the Stream Hash before and after
assertEquals(first, is.md5().stringFormat());
assertEquals(first, "9c7dd68c8352f1c59a33efe0dca04f06");
}
}
Binary file not shown.

0 comments on commit 4c68df5

Please sign in to comment.