Skip to content

Commit

Permalink
Add tests for array indexes
Browse files Browse the repository at this point in the history
  • Loading branch information
@SasinduDilshara
SasinduDilshara committed Jul 12, 2024
1 parent 916eff1 commit fba132a
Show file tree
Hide file tree
Showing 4 changed files with 197 additions and 20 deletions.
4 changes: 2 additions & 2 deletions ballerina-tests/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ def testCommonTomlFilePlaceHolder = new File("${project.rootDir}/build-config/re
def ballerinaDist = "${project.rootDir}/target/ballerina-runtime"
def distributionBinPath = "${ballerinaDist}/bin"
def testCoverageParam = "--code-coverage --coverage-format=xml --includes=io.ballerina.stdlib.data.*:ballerina.*"
def testPackages = ["user-config-tests", "type-compatible-tests", "unicode-tests", "constraint-validation-tests", "parse-list-types-tests", "parse-record-types-tests",
"parse-string-array-types-tests", "parse-string-record-types-tests", "union-type-tests"]
def testPackages = ["parse-string-array-types-tests", "user-config-tests", "type-compatible-tests", "unicode-tests", "constraint-validation-tests",
"parse-list-types-tests", "parse-record-types-tests", "parse-string-record-types-tests", "union-type-tests"]
def testCommonPackage = "csv-commons"

def stripBallerinaExtensionVersion(String extVersion) {
Expand Down
125 changes: 125 additions & 0 deletions ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_array_test.bal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,3 +309,128 @@ function testFromCsvStringWithTypeForStringAndArrayAsExpectedType3() {
test:assertTrue(cv7daa is csv:Error);
test:assertEquals((<csv:Error>cv7daa).message(), common:generateErrorMessageForInvalidCast("true", "decimal"));
}

@test:Config
function testArrayIndexes() {
string csv = string `a, b
1, 2
3, 4
5, 6
7, 8`;

string csv2 = string `a, b
1, 2, 3
3, 4, 5
5, 6, 7
7, 8, 9`;

record {}[2]|error rec = csv:parseStringToRecord(csv);
test:assertEquals(rec, [
{a: 1, b: 2},
{a: 3, b: 4}
]);

map<int>[2]|error rec_2 = csv:parseStringToRecord(csv);
test:assertEquals(rec_2, [
{a: 1, b: 2},
{a: 3, b: 4}
]);

record {|int a;|}[2]|error rec2 = csv:parseStringToRecord(csv, {skipLines: [2]});
test:assertEquals(rec2, [
{a: 1},
{a: 5}
]);

record {|int a;|}[5]|error rec2_2 = csv:parseStringToRecord(csv, {skipLines: [2]});
test:assertTrue(rec2_2 is csv:Error);

int[][2]|error rec3 = csv:parseStringToList(csv2);
test:assertEquals(rec3, [
[1, 2],
[3, 4],
[5, 6],
[7, 8]
]);

[int, int][2]|error rec3_2 = csv:parseStringToList(csv2);
test:assertEquals(rec3_2, [
[1, 2],
[3, 4]
]);

[int...][2]|error rec3_3 = csv:parseStringToList(csv2);
test:assertEquals(rec3_3, [
[1, 2, 3],
[3, 4, 5]
]);

int[1][2]|error rec4 = csv:parseStringToList(csv2, {skipLines: [2]});
test:assertEquals(rec4, [
[1, 2]
]);

int[2][]|error rec5 = csv:parseStringToList(csv2);
test:assertEquals(rec5, [
[1, 2, 3],
[3, 4, 5]
]);
}

@test:Config
function testArrayIndexes2() {
map<int>[] csv = [{a: 1, b: 2}, {a: 3, b: 4}, {a: 5, b: 6}, {a: 7, b: 8}];
string[][] csv2 = [["1", "2", "3"], ["3", "4", "5"], ["5", "6", "7"], ["7", "8", "9"]];

record {}[2]|error rec = csv:parseRecordAsRecordType(csv);
test:assertEquals(rec, [
{a: 1, b: 2},
{a: 3, b: 4}
]);

map<int>[2]|error rec_2 = csv:parseListAsRecordType(csv2, ["a", "b", "c"]);
test:assertEquals(rec_2, [
{a: 1, b: 2, c: 3},
{a: 3, b: 4, c: 5}
]);

record {|int a;|}[2]|error rec2 = csv:parseRecordAsRecordType(csv, {skipLines: [2]});
test:assertEquals(rec2, [
{a: 1},
{a: 5}
]);

record {|int a;|}[5]|error rec2_2 = csv:parseRecordAsRecordType(csv, {skipLines: [2]});
test:assertTrue(rec2_2 is csv:Error);

int[][2]|error rec3 = csv:parseRecordAsListType(csv, ["a", "b"]);
test:assertEquals(rec3, [
[1, 2],
[3, 4],
[5, 6],
[7, 8]
]);

[int, int][2]|error rec3_2 = csv:parseListAsListType(csv2);
test:assertEquals(rec3_2, [
[1, 2],
[3, 4]
]);

[int...][2]|error rec3_3 = csv:parseRecordAsListType(csv, ["a", "b"], {skipLines: [1]});
test:assertEquals(rec3_3, [
[3, 4],
[5, 6]
]);

int[1][2]|error rec4 = csv:parseRecordAsListType(csv, ["a", "b"], {skipLines: [2]});
test:assertEquals(rec4, [
[1, 2]
]);

int[2][]|error rec5 = csv:parseListAsListType(csv2);
test:assertEquals(rec5, [
[1, 2, 3],
[3, 4, 5]
]);
}
7 changes: 6 additions & 1 deletion native/src/main/java/io/ballerina/stdlib/data/csvdata/csv/CsvParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ static class StateMachine {
private StringBuilder hexBuilder = new StringBuilder(4);
boolean isValueStart = false;
State prevState;
int arraySize = 0;
StateMachine() {
reset();
}
Expand Down Expand Up @@ -171,6 +172,7 @@ public void reset() {
isQuoteClosed = false;
isIntersectionElementType = false;
prevState = null;
arraySize = 0;
}

private static boolean isWhitespace(char ch, Object lineTerminator) {
Expand Down Expand Up @@ -646,9 +648,12 @@ private static void initiateNewRowType(StateMachine sm) {

private static void finalizeTheRow(StateMachine sm) {
int rootArraySize = sm.rootArrayType.getSize();
if (rootArraySize == -1 || sm.rowIndex < rootArraySize) {
if (rootArraySize == -1) {
sm.rootCsvNode.append(sm.currentCsvNode);
} else if (sm.arraySize < rootArraySize) {
sm.rootCsvNode.add(sm.arraySize, sm.currentCsvNode);
}
sm.arraySize++;
}

private static void addRowValue(StateMachine sm) throws CsvParserException {
Expand Down
81 changes: 64 additions & 17 deletions native/src/main/java/io/ballerina/stdlib/data/csvdata/csv/CsvTraversal.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ static class CsvTree {
Type sourceArrayElementType;
CsvConfig config;
String[] headers = null;
int arraySize = 0;

void reset() {
currentCsvNode = null;
Expand All @@ -127,6 +128,23 @@ void reset() {
sourceArrayElementType = null;
config = null;
headers = null;
arraySize = 0;
}


void resetForUnionTypes() {
currentCsvNode = null;
currentField = null;
fieldHierarchy.clear();
updatedRecordFieldNames.clear();
headerFieldHierarchy.clear();
fields.clear();;
restType = null;
fieldNames.clear();
rootCsvNode = null;
expectedArrayElementType = null;
headers = null;
arraySize = 0;
}

CsvTree() {
Expand Down Expand Up @@ -163,23 +181,24 @@ public Object traverseCsv(BArray csv, CsvConfig config, Type type) {
int expectedArraySize = ((ArrayType) referredType).getSize();
setRootCsvNodeForNonUnionArrays(referredType, type);
validateExpectedArraySize(expectedArraySize, sourceArraySize);
traverseCsvWithExpectedType(expectedArraySize, sourceArraySize, csv, type);
traverseCsvWithExpectedType(sourceArraySize, csv, type);
} else {
traverseCsvWithUnionExpectedType(referredType, type, sourceArraySize, csv, config);
traverseCsvWithUnionExpectedType(referredType, type, sourceArraySize, csv);
}
return rootCsvNode;
}

private void traverseCsvWithUnionExpectedType(Type referredType, Type type, int sourceArraySize,
BArray csv, CsvConfig config) {
BArray csv) {
for (Type memberType: ((UnionType) referredType).getMemberTypes()) {
Type mType = TypeUtils.getReferredType(memberType);
if (mType.getTag() == TypeTags.ARRAY_TAG) {
int expectedArraySize = ((ArrayType) mType).getSize();
resetForUnionTypes();
try {
setRootCsvNodeForNonUnionArrays(mType, mType);
validateExpectedArraySize(expectedArraySize, sourceArraySize);
traverseCsvWithExpectedType(expectedArraySize, sourceArraySize, csv, type);
traverseCsvWithExpectedType(sourceArraySize, csv, type);
return;
} catch (Exception ex) {
// ignore
Expand All @@ -189,7 +208,7 @@ private void traverseCsvWithUnionExpectedType(Type referredType, Type type, int
throw DiagnosticLog.error(DiagnosticErrorCode.SOURCE_CANNOT_CONVERT_INTO_EXP_TYPE, type);
}

private void traverseCsvWithExpectedType(int expectedArraySize, int sourceArraySize,
private void traverseCsvWithExpectedType(int sourceArraySize,
BArray csv, Type type) {
boolean isIntersection = false;
if (expectedArrayElementType.getTag() == TypeTags.INTERSECTION_TAG) {
Expand All @@ -203,18 +222,17 @@ private void traverseCsvWithExpectedType(int expectedArraySize, int sourceArrayS
switch (expectedArrayElementType.getTag()) {
case TypeTags.RECORD_TYPE_TAG:
case TypeTags.MAP_TAG:
case TypeTags.TABLE_TAG:
traverseCsvArrayMembersWithMapAsCsvElementType(expectedArraySize == -1 ?
sourceArraySize : expectedArraySize, csv, expectedArrayElementType, isIntersection);
traverseCsvArrayMembersWithMapAsCsvElementType(sourceArraySize, csv,
expectedArrayElementType, isIntersection);
break;
case TypeTags.ARRAY_TAG:
case TypeTags.TUPLE_TAG:
traverseCsvArrayMembersWithArrayAsCsvElementType(expectedArraySize == -1 ?
sourceArraySize : expectedArraySize, csv, expectedArrayElementType, isIntersection);
traverseCsvArrayMembersWithArrayAsCsvElementType(sourceArraySize, csv,
expectedArrayElementType, isIntersection);
break;
case TypeTags.UNION_TAG:
traverseCsvArrayMembersWithUnionAsCsvElementType(expectedArraySize == -1 ?
sourceArraySize : expectedArraySize, csv, (UnionType) expectedArrayElementType, type);
traverseCsvArrayMembersWithUnionAsCsvElementType(sourceArraySize, csv,
(UnionType) expectedArrayElementType, type);
break;
case TypeTags.INTERSECTION_TAG:
for (Type constituentType : ((IntersectionType) expectedArrayElementType).getConstituentTypes()) {
Expand All @@ -234,41 +252,61 @@ private void traverseCsvWithExpectedType(int expectedArraySize, int sourceArrayS
public void traverseCsvArrayMembersWithMapAsCsvElementType(long length, BArray csv, Type expectedArrayType,
boolean isIntersection) {
Object rowValue;
ArrayType arrayType = (ArrayType) rootCsvNode.getType();
for (int i = 0; i < length; i++) {
if (ignoreRow(i + 1, config.skipLines)) {
continue;
}
if (arrayType.getState() == ArrayType.ArrayState.CLOSED &&
arrayType.getSize() - 1 < this.arraySize) {
break;
}

rowValue = traverseCsvElementWithMapOrRecord(csv.get(i), expectedArrayType);
if (isIntersection) {
rowValue = CsvCreator.constructReadOnlyValue(rowValue);
}
rootCsvNode.append(rowValue);
rootCsvNode.add(this.arraySize, rowValue);
this.arraySize++;
}
}

public void traverseCsvArrayMembersWithArrayAsCsvElementType(long length, BArray csv, Type expectedArrayType,
boolean isIntersection) {
Object rowValue;
ArrayType arrayType = (ArrayType) rootCsvNode.getType();
for (int i = 0; i < length; i++) {
if (ignoreRow(i + 1, config.skipLines)) {
continue;
}
if (arrayType.getState() == ArrayType.ArrayState.CLOSED &&
arrayType.getSize() - 1 < this.arraySize) {
break;
}

rowValue = traverseCsvElementWithArray(csv.get(i), expectedArrayType);
if (isIntersection) {
rowValue = CsvCreator.constructReadOnlyValue(rowValue);
}
rootCsvNode.append(rowValue);
rootCsvNode.add(this.arraySize, rowValue);
this.arraySize++;
}
}

public void traverseCsvArrayMembersWithUnionAsCsvElementType(long length, BArray csv,
UnionType expectedArrayType, Type type) {
Object rowValue;
ArrayType arrayType = (ArrayType) rootCsvNode.getType();
for (int i = 0; i < length; i++) {
boolean isCompatible = false;
if (ignoreRow(i + 1, config.skipLines)) {
continue;
}
if (arrayType.getState() == ArrayType.ArrayState.CLOSED &&
arrayType.getSize() - 1 < this.arraySize) {
break;
}

Object csvData = csv.get(i);
for (Type memberType: expectedArrayType.getMemberTypes()) {
boolean isIntersection = false;
Expand All @@ -294,7 +332,8 @@ public void traverseCsvArrayMembersWithUnionAsCsvElementType(long length, BArray
if (isIntersection) {
rowValue = CsvCreator.constructReadOnlyValue(rowValue);
}
rootCsvNode.append(rowValue);
rootCsvNode.add(this.arraySize, rowValue);
this.arraySize++;
isCompatible = true;
break;
} catch (Exception e) {
Expand Down Expand Up @@ -484,7 +523,7 @@ private boolean checkExpectedTypeMatchWithHeaders(Type expectedType, BArray csvE
if (type instanceof TupleType) {
return checkExpectedTypeMatchWithHeadersForTuple(expectedType, (TupleType) type);
} else {
return checkExpectedTypeMatchWithHeadersForArray(expectedType, csvElement.getElementType(), arraySize);
return checkExpectedTypeMatchWithHeadersForArray(expectedType);
}
}

Expand Down Expand Up @@ -530,7 +569,7 @@ private boolean checkExpectedTypeMatchWithHeadersForTuple(Type expectedType, Tup
return false;
}

private boolean checkExpectedTypeMatchWithHeadersForArray(Type expectedType, Type arrayType, int arraySize) {
private boolean checkExpectedTypeMatchWithHeadersForArray(Type expectedType) {
if (expectedType instanceof RecordType) {
if (this.restType != null) {
return true;
Expand Down Expand Up @@ -723,7 +762,15 @@ private void addCurrentFieldValue(Type type, Object recValue, BString key, boole
public void addValuesToArrayType(Object arrayValue, Type type, int index,
Object currentCsvNode, CsvConfig config) {
Object value = getFieldValue(type, arrayValue, false);
boolean isArrayType = type instanceof ArrayType;
if (!(value instanceof UnMappedValue)) {
if (isArrayType) {
ArrayType arrayType = (ArrayType) TypeUtils.getType(type);
if (arrayType.getState() == ArrayType.ArrayState.CLOSED &&
arrayType.getSize() - 1 < index) {
return;
}
}
((BArray) currentCsvNode).add(index, value);
return;
}
Expand Down

0 comments on commit fba132a

Please sign in to comment.