Skip to content

Commit

Permalink
feat: java and swift tests from testfile
Browse files Browse the repository at this point in the history
  • Loading branch information
@PaulBallmann
PaulBallmann committed Feb 4, 2022
1 parent 970a636 commit ef5eeca
Show file tree
Hide file tree
Showing 9 changed files with 213 additions and 70 deletions.
Binary file modified java/.DS_Store
View file
Binary file not shown.
Binary file modified java/src/.DS_Store
View file
Binary file not shown.
Binary file modified java/src/test/.DS_Store
View file
Binary file not shown.
153 changes: 92 additions & 61 deletions java/src/test/java/BloomFilterUnitTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,27 +22,28 @@
import java.nio.charset.StandardCharsets;
import java.security.NoSuchAlgorithmException;
import java.util.*;
import java.util.concurrent.atomic.AtomicIntegerArray;
import java.util.concurrent.atomic.AtomicLongArray;
import java.util.logging.Filter;

public class BloomFilterUnitTest {

private static String JSON_TEST_FILE = "src/test/resources/testcase1.json";
private static String JSON_TEST_FILE = "src/test/resources/filter-test.json";
private JSONArray testObjects = null;
private BloomFilterImpl bloomFilter;
private FilterTestData filterTestData = null;

@Test
public void testBigInteger() throws FilterException,IOException,NoSuchAlgorithmException
{
BigInteger val = BloomFilterImpl.calcIndex(new byte[]{11}, 1, 100);
assert val.intValue() == 75;
public void testBigInteger() throws FilterException, IOException, NoSuchAlgorithmException {
BigInteger val = BloomFilterImpl.calcIndex(new byte[]{11}, 1, 100);
assert val.intValue() == 75;

val = BloomFilterImpl.calcIndex(new byte[]{1}, 1, 1);
assert val.intValue() == 0;
val = BloomFilterImpl.calcIndex(new byte[]{1}, 1, 1);
assert val.intValue() == 0;
}

@Test
public void runBasicBloom() throws FilterException,IOException,NoSuchAlgorithmException {
public void runBasicBloom() throws FilterException, IOException, NoSuchAlgorithmException {
BloomFilterImpl impl = new BloomFilterImpl(1, (byte) 1, 1);
impl.add(new byte[]{0, 5, 33, 44});
assert !impl.mightContain(new byte[]{0, 5, 88, 44});
Expand All @@ -52,7 +53,7 @@ public void runBasicBloom() throws FilterException,IOException,NoSuchAlgorithmEx
}

@Test
public void runDifferentByteSizeBlock() throws FilterException,IOException,NoSuchAlgorithmException {
public void runDifferentByteSizeBlock() throws FilterException, IOException, NoSuchAlgorithmException {
BloomFilterImpl impl = new BloomFilterImpl(8, (byte) 1, 1);
impl.add(new byte[]{0, 5, 33, 44});

Expand All @@ -63,7 +64,7 @@ public void runDifferentByteSizeBlock() throws FilterException,IOException,NoSuc

int index = BloomFilterImpl.calcIndex(new byte[]{0, 5, 33, 44}, 0, numBits).intValue();
int bytepos = index / (Long.BYTES * 8);
long pattern = Long.MIN_VALUE >>> index ;
long pattern = Long.MIN_VALUE >>> index;
longArray.set(bytepos, longArray.get(bytepos) | pattern);

assert impl.getData().get(1) == longArray.get(0);
Expand All @@ -80,7 +81,7 @@ public void testMinValues() throws FilterException {
}

@Test()
public void testNormalValues() throws FilterException,IOException,NoSuchAlgorithmException {
public void testNormalValues() throws FilterException, IOException, NoSuchAlgorithmException {
BloomFilterImpl impl = new BloomFilterImpl(56049, (byte) 20, 1);
impl.add(new byte[]{0, 9, 44});
assert impl.mightContain(new byte[]{0, 9, 44});
Expand All @@ -107,7 +108,7 @@ public void testNegativeElementSize() throws FilterException {
}

@Test()
public void testByteStream() throws FilterException,IOException,NoSuchAlgorithmException {
public void testByteStream() throws FilterException, IOException, NoSuchAlgorithmException {
ByteArrayOutputStream output = new ByteArrayOutputStream();
BloomFilterImpl impl = new BloomFilterImpl(500, 0.000000001f);
impl.add(new byte[]{5, 3, 2, 7});
Expand All @@ -132,7 +133,7 @@ public void testByteStream() throws FilterException,IOException,NoSuchAlgorithmE
}

@Test()
public void testByteOutputStream() throws FilterException,IOException,NoSuchAlgorithmException {
public void testByteOutputStream() throws FilterException, IOException, NoSuchAlgorithmException {
ByteArrayOutputStream output = new ByteArrayOutputStream();
BloomFilterImpl impl = new BloomFilterImpl(1, 1);
impl.writeTo(output);
Expand All @@ -142,14 +143,14 @@ public void testByteOutputStream() throws FilterException,IOException,NoSuchAlgo


@Test
public void compareSizes() throws FilterException,IOException,NoSuchAlgorithmException {
public void compareSizes() throws FilterException, IOException, NoSuchAlgorithmException {
BloomFilterImpl impl = new BloomFilterImpl(1, 1);
BloomFilterImpl impl2 = new BloomFilterImpl(1, 0.125f);
assert impl.getData().length() == impl2.getData().length();
}

@Test
public void compare() throws FilterException,IOException,NoSuchAlgorithmException {
public void compare() throws FilterException, IOException, NoSuchAlgorithmException {
BloomFilterImpl impl = new BloomFilterImpl(1, 1);
BloomFilterImpl impl2 = new BloomFilterImpl(1, 0.125f);
assert impl.getData().length() == impl2.getData().length();
Expand All @@ -162,7 +163,7 @@ public void runTests() throws Exception {
this.runBloomFilterTest();
}

private int doScans(BloomFilter filter, int scans) throws FilterException,IOException,NoSuchAlgorithmException {
private int doScans(BloomFilter filter, int scans) throws FilterException, IOException, NoSuchAlgorithmException {
int falsePositives = 0;
for (int x = 1; x < scans; x++) {
if (filter.mightContain(ByteBuffer.allocate(4).putInt(Math.abs(x)).array())) {
Expand All @@ -173,7 +174,7 @@ private int doScans(BloomFilter filter, int scans) throws FilterException,IOExce
}

@Test
public void testProbabilistcRate() throws FilterException,IOException,NoSuchAlgorithmException {
public void testProbabilistcRate() throws FilterException, IOException, NoSuchAlgorithmException {
int scans = 10000000;
float propScan = 0.1f;

Expand All @@ -189,7 +190,7 @@ public void testProbabilistcRate() throws FilterException,IOException,NoSuchAlgo
}

@Test
public void testProbabilistcRate2() throws FilterException,IOException,NoSuchAlgorithmException {
public void testProbabilistcRate2() throws FilterException, IOException, NoSuchAlgorithmException {
int scans = 10000000;
BloomFilter filter = new BloomFilterImpl(100, (byte) 1, 4);
double propScan = filter.getP();
Expand All @@ -203,7 +204,7 @@ public void testProbabilistcRate2() throws FilterException,IOException,NoSuchAlg
}

@Test
public void testProbabilistcRate3() throws FilterException,IOException,NoSuchAlgorithmException {
public void testProbabilistcRate3() throws FilterException, IOException, NoSuchAlgorithmException {
int scans = 100000;
float propScan = 0.00001f;
int entries = 10000;
Expand All @@ -224,26 +225,79 @@ public void testProbabilistcRate3() throws FilterException,IOException,NoSuchAlg
}

@Test
public void testRandom() throws FilterException,NoSuchAlgorithmException,IOException
{
BloomFilterImpl imp = new BloomFilterImpl(62,0.01f);
public void testRandom() throws FilterException, NoSuchAlgorithmException, IOException {
BloomFilterImpl imp = new BloomFilterImpl(62, 0.01f);

imp.add(new byte[]{16, 43, 72, -124, -99, 34, -113, -77, 78, -105, -113, 30, -90, -25, -38, 70, 76, 109, -92, -27, -15, 65, 36, -113, 3, -115, -4, -49, -81, -1, 69, -125, -22, 53, -49, 65, 31, 65, 18, 60, -56, -17, 16, 5, -11, 5, -3, -49, 4, -48, 122, 31, -37, -113, 54, -35, -83, -114, 62, 57, 125, 120, -26, 106});
imp.add(new byte[]{16, 43, 72, -124, -99, 34, -113, -77, 78, -105, -113, 30, -90, -25, -38, 70, 76, 109, -92,
-27, -15, 65, 36, -113, 3, -115, -4, -49, -81, -1, 69, -125, -22, 53, -49, 65, 31, 65, 18, 60, -56, -17,
16, 5, -11, 5, -3, -49, 4, -48, 122, 31, -37, -113, 54, -35, -83, -114, 62, 57, 125, 120, -26, 106});
}

@Test
public void runSmokeTest() throws NoSuchAlgorithmException,IOException, FilterException
{
public void runSmokeTest() throws NoSuchAlgorithmException, IOException, FilterException {
BloomFilterImpl imp = new BloomFilterImpl(1000000, 0.1f);
for (int x=0;x<100000;x++)
{
for (int x = 0; x < 100000; x++) {
UUID guid = UUID.randomUUID();
var hash = BloomFilterImpl.hash(guid.toString().getBytes(),'1');
var hash = BloomFilterImpl.hash(guid.toString().getBytes(), '1');

imp.add(hash);;
imp.add(hash);
;
}
}

/// NEW

@Test
public void runBase64Test() throws FilterException, NoSuchAlgorithmException, IOException {
// read the test objects from the json file
this.testObjects = this.readFromJson();
assert this.testObjects != null;
for (int i = 0; i < this.testObjects.size(); i++) {
System.out.println("TEST (" + i + ") START");
// for every element in the test data, do the tests
FilterTestData testData = this.extractTestData(i);
BloomFilterImpl filter = new BloomFilterImpl(testData.getDataSize(), (float) testData.getP());
// now add all the test elements that should be added to the filter
for (int j = 0; j < testData.getWritten().length; j++) {
// if data has been written, add to filter
if (testData.getWritten()[j] == 1) {
filter.add(testData.getData().get(j).toString().getBytes(StandardCharsets.UTF_8));
}
}
// all data has been written to the filter. Now get base64 of the filter
String filterAsBase64 = this.getFilterAsBase64(filter.getData());
// write that back into the json file
JSONObject o = (JSONObject) this.testObjects.get(i);
o.put("filter", filterAsBase64);
this.writeToJson(o, i);

// now try to read the data back that has been written to it
int[] exists = new int[testData.getDataSize()];
for (int x = 0; x < testData.getWritten().length; x++) {
byte[] elem = dataToArr(testData.getData().get(x));
int mightContain = filter.mightContain(elem) ? 1 : 0;
System.out.println(String.format("Filter reported element %s exists (%s) at index %s", testData.getData().get(x), mightContain, x));
exists[x] = mightContain;
}
// write to json
o.put("exists", Arrays.toString(exists));
this.writeToJson(o, i);
// now perform check if exists and written array are equal
String existsBase64 = Base64.getEncoder().encodeToString(
Arrays.toString(exists).getBytes(StandardCharsets.UTF_8));
String writtenBase64 = Base64.getEncoder().encodeToString(
Arrays.toString(testData.getWritten()).getBytes(StandardCharsets.UTF_8));
if (existsBase64.equals(writtenBase64)) {
System.out.println("BASE64TEST: Success");
} else {
System.out.println("BASE64TEST: FAILURE!");
}
System.out.println("TEST (" + i + ") END");
}
}

/// END NEW

@Test
public void runBloomFilterTest() throws FilterException, IOException, NoSuchAlgorithmException {
assert this.testObjects != null;
Expand All @@ -262,7 +316,7 @@ public void runBloomFilterTest() throws FilterException, IOException, NoSuchAlgo
}
}

private BloomFilterImpl createFilterForData(FilterTestData data) throws FilterException, IOException, NoSuchAlgorithmException {
private BloomFilterImpl createFilterForData(FilterTestData data) {
return new BloomFilterImpl(data.getDataSize(), (float) data.getP());
}

Expand All @@ -280,36 +334,17 @@ public void calcBaseStringFromFilter(int i) {
// this.storeBase64InFile(i, filterAsBase64);
}

public void filterLookupTest(FilterTestData testData, int index) throws FilterException, NoSuchAlgorithmException, IOException {
public void filterLookupTest(FilterTestData testData, int index) throws FilterException,
NoSuchAlgorithmException, IOException {
this.lookupFilter(testData, index);
}
/*
@Test
public void runTSIBloomFilter() throws Exception {
// Contains all of the data from the test file
JSONArray jsonArray = this.readFromJson();
// Iterate over all of the test-cases
assert jsonArray != null;
this.testObjects = jsonArray;
for (int i = 0; i < this.testObjects.size(); i++) {
System.out.printf("i: %s%n", i);
JSONObject object = (JSONObject) jsonArray.get(i);
FilterTestData testData = this.extractTestData(object);
this.bloomFilter = new BloomFilterImpl(testData.getDataSize(), testData.getK(), (float) testData.getP());
this.addToTsiBloomFilter(testData);
this.storeFilterAsBase64(this.bloomFilter.getBits(), object, i);
this.printTsiFilterBits();
this.lookupFilter(testData, object, i);
return;
}
}*/

/**
* Checks if all bits written in the testData.written array can be found in the filter.
* Each element that actually exists will be set int he testData.exists array
*/
private void lookupFilter(FilterTestData testData, int index) throws FilterException,IOException,NoSuchAlgorithmException {
private void lookupFilter(FilterTestData testData, int index) throws FilterException, IOException,
NoSuchAlgorithmException {
int exists[] = new int[testData.getDataSize()];
for (int i = 0; i < testData.getDataSize(); i++) {
// iterate over all testdata
Expand Down Expand Up @@ -341,9 +376,9 @@ private byte[] dataToArr(Object obj) {
return obj.toString().getBytes(StandardCharsets.UTF_8);
}

private String getFilterAsBase64(AtomicLongArray filter) {
private String getFilterAsBase64(AtomicIntegerArray filter) {
String base64Filter = getBase64FromFilter(filter);
System.out.println("TSI: " + base64Filter);
// System.out.println("TSI: " + base64Filter);
return base64Filter;
// objPointer.put("filter", base64Filter);
// writeToJson(objPointer, index);
Expand Down Expand Up @@ -378,11 +413,7 @@ private int[] toArray(JSONArray arr) {
return intArr;
}

private void printTsiFilterBits() {
// System.out.println(this.bloomFilter.getBytes().toString());
}

private void addToTsiBloomFilter(FilterTestData data) throws FilterException,IOException,NoSuchAlgorithmException {
private void addToTsiBloomFilter(FilterTestData data) throws IOException, NoSuchAlgorithmException {
try {
for (int i = 0; i < data.getDataSize(); i++) {
// only add elements where written is set to 1 at given index i
Expand All @@ -395,7 +426,7 @@ private void addToTsiBloomFilter(FilterTestData data) throws FilterException,IOE
}
}

private String getBase64FromFilter(AtomicLongArray bitArray) {
private String getBase64FromFilter(AtomicIntegerArray bitArray) {
return Base64.getEncoder().encodeToString(bitArray.toString().getBytes(StandardCharsets.UTF_8));
}

Expand Down
1 change: 1 addition & 0 deletions java/src/test/resources/filter-test.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[{"p":1.0E-4,"filter":"WzIxMDIwOTgwNjgsIDEyMDI2OTc2ODMsIDQzOTc1MjI2OF0=","data":["1"],"exists":"[1]","k":1,"written":[1]},{"p":1.0E-4,"filter":"WzIxMjk4NjE3NTYsIDE2ODA1NjM0MzJd","data":["1","2"],"exists":"[1, 1]","k":1,"written":[1,1]},{"p":1.0E-4,"filter":"WzcyNTY0NTA2LCA5MTcxOTI4OTZd","data":["1","2","3"],"exists":"[1, 0, 1]","k":1,"written":[1,0,1]},{"p":1.0E-4,"filter":"WzM3MzM2NzA2OCwgLTE0NTcyOTA3NDcsIC0xNTA0MzA1MzI2LCAzNzkyMTA5NDRd","data":["1","2","3","4"],"exists":"[1, 0, 1, 1]","k":1,"written":[1,0,1,1]}]
53 changes: 48 additions & 5 deletions swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/dgca_bloomfilter_iosTests.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,56 @@ final class dgca_bloomfilter_iosTests: XCTestCase {

func testWithTestData() throws {
let dataSets = try self.readTestData()

for (index, data) in dataSets.enumerated() {
print("\(index): \(data.data)")
for (index, element) in dataSets.enumerated() {
print("TEST (\(index)) START")
// guard let data = element.data else { throw FilterError.unknownError }
let filter = try BloomFilter(numElems: UInt16(element.data.count), probRate: element.p)
for addIndex in 0..<element.data.count {
if element.written[addIndex] == 1 {
guard let elemToAdd = stringArrayToData(stringArray: element.data) else { throw FilterError.unknownError }
try filter.add(element: elemToAdd)
}
}
// get base64 string of filter after all elems have been added
guard let dataFromArray = arrayToData(intArray: filter.getData()) else { throw FilterError.unknownError }
let base64String = dataFromArray.base64EncodedString()
if base64String.elementsEqual(element.filter) {
print("BASE64TEST: SUCCESS!")
} else {
print("BASE64TEST: FAILURE!")
}
var exists: [Int32] = [Int32](repeating: 0, count: element.data.count)
//var exists: [Int32] = [Int32(element.data.count)]
for x in 0..<element.data.count {
let mightContain = try filter.mightContain(element: element.data[x].data(using: .utf8)!) ? 1 : 0
exists[x] = Int32(mightContain)
print("Filter reported element \(element.data[x]) \(mightContain == 1 ? "exists" : "not exist") at index \(x)")
}
// compare written and exist arrays
let base64Exist = arrayToData(intArray: exists)!.base64EncodedString()
let base64Written = array2ToData(intArray: element.written)!.base64EncodedString()
if base64Exist.elementsEqual(base64Written) {
print("BASE64TEST: SUCCESS!")
} else {
print("BASE64TEST: FAILURE!")
}
print("TEST (\(index) END")
}

}

func stringArrayToData(stringArray: [String]) -> Data? {
return try? JSONSerialization.data(withJSONObject: stringArray, options: [])
}

func arrayToData(intArray: [Int32]) -> Data? {
return try? JSONSerialization.data(withJSONObject: intArray, options: [])
}

func array2ToData(intArray: [Int]) -> Data? {
return try? JSONSerialization.data(withJSONObject: intArray, options: [])
}

func testRunBasicBloom() throws {
let impl = try BloomFilter(size: 1, nHash: 1,numElems: 1);
try impl.add(element: Data([0, 5, 33, 44]));
Expand All @@ -44,8 +88,7 @@ final class dgca_bloomfilter_iosTests: XCTestCase {
XCTAssert(hash.base64EncodedString() == "G2lkA9iYJ1bCNq+8WwnA9U3QaC7lNKddxLcKXV7Quo8=" )
}

func testRandom() throws
{
func testRandom() throws {
let filter = try BloomFilter(numElems:62,probRate:0.01)


Expand Down
3 changes: 3 additions & 0 deletions swift/dgca-bloomfilter-ios/Tests/dgca-bloomfilter-iosTests/model/TestData.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,7 @@ struct TestData: Codable {
let p: Double;
let filter: String;
let data: [String];
let exists: String;
let k: Int;
let written: [Int];
}
Loading

0 comments on commit ef5eeca

Please sign in to comment.