diff --git a/java/.DS_Store b/java/.DS_Store index e0f225a..462d3a7 100644 Binary files a/java/.DS_Store and b/java/.DS_Store differ diff --git a/java/src/.DS_Store b/java/src/.DS_Store index 1277e56..b5d90cb 100644 Binary files a/java/src/.DS_Store and b/java/src/.DS_Store differ diff --git a/java/src/test/.DS_Store b/java/src/test/.DS_Store index d32b332..f9cbbec 100644 Binary files a/java/src/test/.DS_Store and b/java/src/test/.DS_Store differ diff --git a/java/src/test/java/BloomFilterUnitTest.java b/java/src/test/java/BloomFilterUnitTest.java index 2142374..3d6ad06 100644 --- a/java/src/test/java/BloomFilterUnitTest.java +++ b/java/src/test/java/BloomFilterUnitTest.java @@ -22,27 +22,28 @@ import java.nio.charset.StandardCharsets; import java.security.NoSuchAlgorithmException; import java.util.*; +import java.util.concurrent.atomic.AtomicIntegerArray; import java.util.concurrent.atomic.AtomicLongArray; +import java.util.logging.Filter; public class BloomFilterUnitTest { - private static String JSON_TEST_FILE = "src/test/resources/testcase1.json"; + private static String JSON_TEST_FILE = "src/test/resources/filter-test.json"; private JSONArray testObjects = null; private BloomFilterImpl bloomFilter; private FilterTestData filterTestData = null; @Test - public void testBigInteger() throws FilterException,IOException,NoSuchAlgorithmException - { - BigInteger val = BloomFilterImpl.calcIndex(new byte[]{11}, 1, 100); - assert val.intValue() == 75; + public void testBigInteger() throws FilterException, IOException, NoSuchAlgorithmException { + BigInteger val = BloomFilterImpl.calcIndex(new byte[]{11}, 1, 100); + assert val.intValue() == 75; - val = BloomFilterImpl.calcIndex(new byte[]{1}, 1, 1); - assert val.intValue() == 0; + val = BloomFilterImpl.calcIndex(new byte[]{1}, 1, 1); + assert val.intValue() == 0; } @Test - public void runBasicBloom() throws FilterException,IOException,NoSuchAlgorithmException { + public void runBasicBloom() throws FilterException, IOException, NoSuchAlgorithmException { BloomFilterImpl impl = new BloomFilterImpl(1, (byte) 1, 1); impl.add(new byte[]{0, 5, 33, 44}); assert !impl.mightContain(new byte[]{0, 5, 88, 44}); @@ -52,7 +53,7 @@ public void runBasicBloom() throws FilterException,IOException,NoSuchAlgorithmEx } @Test - public void runDifferentByteSizeBlock() throws FilterException,IOException,NoSuchAlgorithmException { + public void runDifferentByteSizeBlock() throws FilterException, IOException, NoSuchAlgorithmException { BloomFilterImpl impl = new BloomFilterImpl(8, (byte) 1, 1); impl.add(new byte[]{0, 5, 33, 44}); @@ -63,7 +64,7 @@ public void runDifferentByteSizeBlock() throws FilterException,IOException,NoSuc int index = BloomFilterImpl.calcIndex(new byte[]{0, 5, 33, 44}, 0, numBits).intValue(); int bytepos = index / (Long.BYTES * 8); - long pattern = Long.MIN_VALUE >>> index ; + long pattern = Long.MIN_VALUE >>> index; longArray.set(bytepos, longArray.get(bytepos) | pattern); assert impl.getData().get(1) == longArray.get(0); @@ -80,7 +81,7 @@ public void testMinValues() throws FilterException { } @Test() - public void testNormalValues() throws FilterException,IOException,NoSuchAlgorithmException { + public void testNormalValues() throws FilterException, IOException, NoSuchAlgorithmException { BloomFilterImpl impl = new BloomFilterImpl(56049, (byte) 20, 1); impl.add(new byte[]{0, 9, 44}); assert impl.mightContain(new byte[]{0, 9, 44}); @@ -107,7 +108,7 @@ public void testNegativeElementSize() throws FilterException { } @Test() - public void testByteStream() throws FilterException,IOException,NoSuchAlgorithmException { + public void testByteStream() throws FilterException, IOException, NoSuchAlgorithmException { ByteArrayOutputStream output = new ByteArrayOutputStream(); BloomFilterImpl impl = new BloomFilterImpl(500, 0.000000001f); impl.add(new byte[]{5, 3, 2, 7}); @@ -132,7 +133,7 @@ public void testByteStream() throws FilterException,IOException,NoSuchAlgorithmE } @Test() - public void testByteOutputStream() throws FilterException,IOException,NoSuchAlgorithmException { + public void testByteOutputStream() throws FilterException, IOException, NoSuchAlgorithmException { ByteArrayOutputStream output = new ByteArrayOutputStream(); BloomFilterImpl impl = new BloomFilterImpl(1, 1); impl.writeTo(output); @@ -142,14 +143,14 @@ public void testByteOutputStream() throws FilterException,IOException,NoSuchAlgo @Test - public void compareSizes() throws FilterException,IOException,NoSuchAlgorithmException { + public void compareSizes() throws FilterException, IOException, NoSuchAlgorithmException { BloomFilterImpl impl = new BloomFilterImpl(1, 1); BloomFilterImpl impl2 = new BloomFilterImpl(1, 0.125f); assert impl.getData().length() == impl2.getData().length(); } @Test - public void compare() throws FilterException,IOException,NoSuchAlgorithmException { + public void compare() throws FilterException, IOException, NoSuchAlgorithmException { BloomFilterImpl impl = new BloomFilterImpl(1, 1); BloomFilterImpl impl2 = new BloomFilterImpl(1, 0.125f); assert impl.getData().length() == impl2.getData().length(); @@ -162,7 +163,7 @@ public void runTests() throws Exception { this.runBloomFilterTest(); } - private int doScans(BloomFilter filter, int scans) throws FilterException,IOException,NoSuchAlgorithmException { + private int doScans(BloomFilter filter, int scans) throws FilterException, IOException, NoSuchAlgorithmException { int falsePositives = 0; for (int x = 1; x < scans; x++) { if (filter.mightContain(ByteBuffer.allocate(4).putInt(Math.abs(x)).array())) { @@ -173,7 +174,7 @@ private int doScans(BloomFilter filter, int scans) throws FilterException,IOExce } @Test - public void testProbabilistcRate() throws FilterException,IOException,NoSuchAlgorithmException { + public void testProbabilistcRate() throws FilterException, IOException, NoSuchAlgorithmException { int scans = 10000000; float propScan = 0.1f; @@ -189,7 +190,7 @@ public void testProbabilistcRate() throws FilterException,IOException,NoSuchAlgo } @Test - public void testProbabilistcRate2() throws FilterException,IOException,NoSuchAlgorithmException { + public void testProbabilistcRate2() throws FilterException, IOException, NoSuchAlgorithmException { int scans = 10000000; BloomFilter filter = new BloomFilterImpl(100, (byte) 1, 4); double propScan = filter.getP(); @@ -203,7 +204,7 @@ public void testProbabilistcRate2() throws FilterException,IOException,NoSuchAlg } @Test - public void testProbabilistcRate3() throws FilterException,IOException,NoSuchAlgorithmException { + public void testProbabilistcRate3() throws FilterException, IOException, NoSuchAlgorithmException { int scans = 100000; float propScan = 0.00001f; int entries = 10000; @@ -224,26 +225,79 @@ public void testProbabilistcRate3() throws FilterException,IOException,NoSuchAlg } @Test - public void testRandom() throws FilterException,NoSuchAlgorithmException,IOException - { - BloomFilterImpl imp = new BloomFilterImpl(62,0.01f); + public void testRandom() throws FilterException, NoSuchAlgorithmException, IOException { + BloomFilterImpl imp = new BloomFilterImpl(62, 0.01f); - imp.add(new byte[]{16, 43, 72, -124, -99, 34, -113, -77, 78, -105, -113, 30, -90, -25, -38, 70, 76, 109, -92, -27, -15, 65, 36, -113, 3, -115, -4, -49, -81, -1, 69, -125, -22, 53, -49, 65, 31, 65, 18, 60, -56, -17, 16, 5, -11, 5, -3, -49, 4, -48, 122, 31, -37, -113, 54, -35, -83, -114, 62, 57, 125, 120, -26, 106}); + imp.add(new byte[]{16, 43, 72, -124, -99, 34, -113, -77, 78, -105, -113, 30, -90, -25, -38, 70, 76, 109, -92, + -27, -15, 65, 36, -113, 3, -115, -4, -49, -81, -1, 69, -125, -22, 53, -49, 65, 31, 65, 18, 60, -56, -17, + 16, 5, -11, 5, -3, -49, 4, -48, 122, 31, -37, -113, 54, -35, -83, -114, 62, 57, 125, 120, -26, 106}); } @Test - public void runSmokeTest() throws NoSuchAlgorithmException,IOException, FilterException - { + public void runSmokeTest() throws NoSuchAlgorithmException, IOException, FilterException { BloomFilterImpl imp = new BloomFilterImpl(1000000, 0.1f); - for (int x=0;x<100000;x++) - { + for (int x = 0; x < 100000; x++) { UUID guid = UUID.randomUUID(); - var hash = BloomFilterImpl.hash(guid.toString().getBytes(),'1'); + var hash = BloomFilterImpl.hash(guid.toString().getBytes(), '1'); - imp.add(hash);; + imp.add(hash); + ; } } + /// NEW + + @Test + public void runBase64Test() throws FilterException, NoSuchAlgorithmException, IOException { + // read the test objects from the json file + this.testObjects = this.readFromJson(); + assert this.testObjects != null; + for (int i = 0; i < this.testObjects.size(); i++) { + System.out.println("TEST (" + i + ") START"); + // for every element in the test data, do the tests + FilterTestData testData = this.extractTestData(i); + BloomFilterImpl filter = new BloomFilterImpl(testData.getDataSize(), (float) testData.getP()); + // now add all the test elements that should be added to the filter + for (int j = 0; j < testData.getWritten().length; j++) { + // if data has been written, add to filter + if (testData.getWritten()[j] == 1) { + filter.add(testData.getData().get(j).toString().getBytes(StandardCharsets.UTF_8)); + } + } + // all data has been written to the filter. Now get base64 of the filter + String filterAsBase64 = this.getFilterAsBase64(filter.getData()); + // write that back into the json file + JSONObject o = (JSONObject) this.testObjects.get(i); + o.put("filter", filterAsBase64); + this.writeToJson(o, i); + + // now try to read the data back that has been written to it + int[] exists = new int[testData.getDataSize()]; + for (int x = 0; x < testData.getWritten().length; x++) { + byte[] elem = dataToArr(testData.getData().get(x)); + int mightContain = filter.mightContain(elem) ? 1 : 0; + System.out.println(String.format("Filter reported element %s exists (%s) at index %s", testData.getData().get(x), mightContain, x)); + exists[x] = mightContain; + } + // write to json + o.put("exists", Arrays.toString(exists)); + this.writeToJson(o, i); + // now perform check if exists and written array are equal + String existsBase64 = Base64.getEncoder().encodeToString( + Arrays.toString(exists).getBytes(StandardCharsets.UTF_8)); + String writtenBase64 = Base64.getEncoder().encodeToString( + Arrays.toString(testData.getWritten()).getBytes(StandardCharsets.UTF_8)); + if (existsBase64.equals(writtenBase64)) { + System.out.println("BASE64TEST: Success"); + } else { + System.out.println("BASE64TEST: FAILURE!"); + } + System.out.println("TEST (" + i + ") END"); + } + } + + /// END NEW + @Test public void runBloomFilterTest() throws FilterException, IOException, NoSuchAlgorithmException { assert this.testObjects != null; @@ -262,7 +316,7 @@ public void runBloomFilterTest() throws FilterException, IOException, NoSuchAlgo } } - private BloomFilterImpl createFilterForData(FilterTestData data) throws FilterException, IOException, NoSuchAlgorithmException { + private BloomFilterImpl createFilterForData(FilterTestData data) { return new BloomFilterImpl(data.getDataSize(), (float) data.getP()); } @@ -280,36 +334,17 @@ public void calcBaseStringFromFilter(int i) { // this.storeBase64InFile(i, filterAsBase64); } - public void filterLookupTest(FilterTestData testData, int index) throws FilterException, NoSuchAlgorithmException, IOException { + public void filterLookupTest(FilterTestData testData, int index) throws FilterException, + NoSuchAlgorithmException, IOException { this.lookupFilter(testData, index); } -/* - @Test - public void runTSIBloomFilter() throws Exception { - // Contains all of the data from the test file - JSONArray jsonArray = this.readFromJson(); - // Iterate over all of the test-cases - assert jsonArray != null; - this.testObjects = jsonArray; - for (int i = 0; i < this.testObjects.size(); i++) { - System.out.printf("i: %s%n", i); - JSONObject object = (JSONObject) jsonArray.get(i); - FilterTestData testData = this.extractTestData(object); - this.bloomFilter = new BloomFilterImpl(testData.getDataSize(), testData.getK(), (float) testData.getP()); - this.addToTsiBloomFilter(testData); - this.storeFilterAsBase64(this.bloomFilter.getBits(), object, i); - this.printTsiFilterBits(); - this.lookupFilter(testData, object, i); - return; - } - - }*/ /** * Checks if all bits written in the testData.written array can be found in the filter. * Each element that actually exists will be set int he testData.exists array */ - private void lookupFilter(FilterTestData testData, int index) throws FilterException,IOException,NoSuchAlgorithmException { + private void lookupFilter(FilterTestData testData, int index) throws FilterException, IOException, + NoSuchAlgorithmException { int exists[] = new int[testData.getDataSize()]; for (int i = 0; i < testData.getDataSize(); i++) { // iterate over all testdata @@ -341,9 +376,9 @@ private byte[] dataToArr(Object obj) { return obj.toString().getBytes(StandardCharsets.UTF_8); } - private String getFilterAsBase64(AtomicLongArray filter) { + private String getFilterAsBase64(AtomicIntegerArray filter) { String base64Filter = getBase64FromFilter(filter); - System.out.println("TSI: " + base64Filter); + // System.out.println("TSI: " + base64Filter); return base64Filter; // objPointer.put("filter", base64Filter); // writeToJson(objPointer, index); @@ -378,11 +413,7 @@ private int[] toArray(JSONArray arr) { return intArr; } - private void printTsiFilterBits() { - // System.out.println(this.bloomFilter.getBytes().toString()); - } - - private void addToTsiBloomFilter(FilterTestData data) throws FilterException,IOException,NoSuchAlgorithmException { + private void addToTsiBloomFilter(FilterTestData data) throws IOException, NoSuchAlgorithmException { try { for (int i = 0; i < data.getDataSize(); i++) { // only add elements where written is set to 1 at given index i @@ -395,7 +426,7 @@ private void addToTsiBloomFilter(FilterTestData data) throws FilterException,IOE } } - private String getBase64FromFilter(AtomicLongArray bitArray) { + private String getBase64FromFilter(AtomicIntegerArray bitArray) { return Base64.getEncoder().encodeToString(bitArray.toString().getBytes(StandardCharsets.UTF_8)); } diff --git a/java/src/test/resources/filter-test.json b/java/src/test/resources/filter-test.json new file mode 100644 index 0000000..a5f9a15 --- /dev/null +++ b/java/src/test/resources/filter-test.json @@ -0,0 +1 @@ +[{"p":1.0E-4,"filter":"WzIxMDIwOTgwNjgsIDEyMDI2OTc2ODMsIDQzOTc1MjI2OF0=","data":["1"],"exists":"[1]","k":1,"written":[1]},{"p":1.0E-4,"filter":"WzIxMjk4NjE3NTYsIDE2ODA1NjM0MzJd","data":["1","2"],"exists":"[1, 1]","k":1,"written":[1,1]},{"p":1.0E-4,"filter":"WzcyNTY0NTA2LCA5MTcxOTI4OTZd","data":["1","2","3"],"exists":"[1, 0, 1]","k":1,"written":[1,0,1]},{"p":1.0E-4,"filter":"WzM3MzM2NzA2OCwgLTE0NTcyOTA3NDcsIC0xNTA0MzA1MzI2LCAzNzkyMTA5NDRd","data":["1","2","3","4"],"exists":"[1, 0, 1, 1]","k":1,"written":[1,0,1,1]}] \ No newline at end of file diff --git a/swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/dgca_bloomfilter_iosTests.swift b/swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/dgca_bloomfilter_iosTests.swift index 3287935..6b72000 100644 --- a/swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/dgca_bloomfilter_iosTests.swift +++ b/swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/dgca_bloomfilter_iosTests.swift @@ -17,12 +17,56 @@ final class dgca_bloomfilter_iosTests: XCTestCase { func testWithTestData() throws { let dataSets = try self.readTestData() - - for (index, data) in dataSets.enumerated() { - print("\(index): \(data.data)") + for (index, element) in dataSets.enumerated() { + print("TEST (\(index)) START") + // guard let data = element.data else { throw FilterError.unknownError } + let filter = try BloomFilter(numElems: UInt16(element.data.count), probRate: element.p) + for addIndex in 0.. Data? { + return try? JSONSerialization.data(withJSONObject: stringArray, options: []) + } + + func arrayToData(intArray: [Int32]) -> Data? { + return try? JSONSerialization.data(withJSONObject: intArray, options: []) } + func array2ToData(intArray: [Int]) -> Data? { + return try? JSONSerialization.data(withJSONObject: intArray, options: []) + } + func testRunBasicBloom() throws { let impl = try BloomFilter(size: 1, nHash: 1,numElems: 1); try impl.add(element: Data([0, 5, 33, 44])); @@ -44,8 +88,7 @@ final class dgca_bloomfilter_iosTests: XCTestCase { XCTAssert(hash.base64EncodedString() == "G2lkA9iYJ1bCNq+8WwnA9U3QaC7lNKddxLcKXV7Quo8=" ) } - func testRandom() throws - { + func testRandom() throws { let filter = try BloomFilter(numElems:62,probRate:0.01) diff --git a/swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/model/TestData.swift b/swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/model/TestData.swift index 01b38c1..00593ee 100644 --- a/swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/model/TestData.swift +++ b/swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/model/TestData.swift @@ -11,4 +11,7 @@ struct TestData: Codable { let p: Double; let filter: String; let data: [String]; + let exists: String; + let k: Int; + let written: [Int]; } diff --git a/swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/model/TestDataLoader.swift b/swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/model/TestDataLoader.swift index 4e5945a..d1e667f 100644 --- a/swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/model/TestDataLoader.swift +++ b/swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/model/TestDataLoader.swift @@ -43,9 +43,13 @@ class TestDataLoader { private func parseTestData(json: Data) -> [TestData]? { let decoder = JSONDecoder() - if let testData = try? decoder.decode([TestData].self, from: json) { - print(testData.count) - return testData + do { + let testData = try decoder.decode([TestData].self, from: json) + print(testData.count) + return testData + + } catch { + print(error) } return nil diff --git a/swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/resources/filter-test.json b/swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/resources/filter-test.json index 0354ab9..464f299 100644 --- a/swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/resources/filter-test.json +++ b/swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/resources/filter-test.json @@ -1 +1,62 @@ -[{"p":0.01,"filter":"WzAsIDAsIDAsIDAsIDAsIDAsIDAsIDEsIDAsIDBd","data":["a"],"exists":"[1]","k":1,"written":[1]},{"p":0.01,"filter":"WzAsIDAsIDAsIDAsIDAsIDAsIDAsIDEsIDAsIDEsIDAsIDAsIDAsIDAsIDAsIDAsIDAsIDAsIDAsIDBd","data":["a","b"],"exists":"[1, 1]","k":1,"written":[1,1]},{"p":0.01,"filter":"WzAsIDEsIDAsIDAsIDAsIDAsIDAsIDAsIDAsIDAsIDAsIDAsIDAsIDAsIDAsIDEsIDAsIDAsIDAsIDAsIDAsIDAsIDAsIDAsIDAsIDAsIDAsIDAsIDBd","data":["a","b","c"],"exists":"[1, 0, 1]","k":1,"written":[1,0,1]}] \ No newline at end of file +[ + { + "p": 1.0E-4, + "filter": "WzIxMDIwOTgwNjgsIDEyMDI2OTc2ODMsIDQzOTc1MjI2OF0=", + "data": [ + "1" + ], + "exists": "[1]", + "k": 1, + "written": [ + 1 + ] + }, + { + "p": 1.0E-4, + "filter": "WzIxMjk4NjE3NTYsIDE2ODA1NjM0MzJd", + "data": [ + "1", + "2" + ], + "exists": "[1, 1]", + "k": 1, + "written": [ + 1, + 1 + ] + }, + { + "p": 1.0E-4, + "filter": "WzcyNTY0NTA2LCA5MTcxOTI4OTZd", + "data": [ + "1", + "2", + "3" + ], + "exists": "[1, 0, 1]", + "k": 1, + "written": [ + 1, + 0, + 1 + ] + }, + { + "p": 1.0E-4, + "filter": "WzM3MzM2NzA2OCwgLTE0NTcyOTA3NDcsIC0xNTA0MzA1MzI2LCAzNzkyMTA5NDRd", + "data": [ + "1", + "2", + "3", + "4" + ], + "exists": "[1, 0, 1, 1]", + "k": 1, + "written": [ + 1, + 0, + 1, + 1 + ] + } +]