Merge pull request #23 from SasinduDilshara/development-revamp

Allow empty strings to represent nil values using user configs

ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_array_test.bal

@@ -437,14 +437,12 @@ function testParseStringArrayAsExpectedTypeWithOutputHeaders() {
  ]);
 
  string[][]|csv:Error cv1baa_5 = csv:parseString(csvStringWithBooleanValues1, {outputWithHeaders: true, header: 2});
- test:assertEquals(cv1baa_5, [
- ["true", "false", "true", "false"],
- ["true", "false", "true", "false"]
- ]);
+ test:assertTrue(cv1baa_5 is csv:Error);
+ test:assertEquals((<csv:Error>cv1baa_5).message(), "Duplicate header found: 'true'");
 
- string[][]|csv:Error cv1baa_6 = csv:parseString(csvStringWithBooleanValues1, {outputWithHeaders: false, header: 2});
+ string[][]|csv:Error cv1baa_6 = csv:parseString(csvStringWithBooleanValues8, {outputWithHeaders: false, header: 2});
  test:assertEquals(cv1baa_6, [
- ["true", "false", "true", "false"]
+ ["true", "false", "true1", "false1"]
  ]);
 
  [string, string, string, string, string][]|csv:Error cv2baa_7 = csv:parseString(csvStringWithBooleanValues2, {outputWithHeaders: true});

ballerina-tests/parse-string-array-types-tests/tests/test_data_values.bal

@@ -60,3 +60,9 @@ string csvStringWithBooleanValues6 = string `b2,b3
 string csvStringWithBooleanValues7 = string `b1,b2,b3,b4
 ${b1},${b2},(),${b4}
 `;
+
+string csvStringWithBooleanValues8 = string `b1,b2,b3,b4
+true,false,true1,false1 
+true,false, true1,false1
+true,false,true1,false1
+`;

ballerina-tests/type-compatible-tests/tests/csv_content_2.txt

@@ -0,0 +1,4 @@
+a, b, c d, e
+"Hello World1", \"Hello World2\", Hello World3, 21
+"Hello World1", \"Hello World2\", Hello World3, 22
+"Hello World1", \"Hello World2\", Hello World3, 23

ballerina-tests/type-compatible-tests/tests/nill_type_test.bal

@@ -0,0 +1,58 @@
+import ballerina/data.csv;
+import ballerina/test;
+
+type Book1 record {
+ ()|string name;
+ ()|string author;
+ ()|string year;
+};
+
+string csvString1 = string `name,author,year
+ ,b,c
+ a,,c
+ a,b,`;
+
+@test:Config{}
+function testEmptyStringWithNilConfig() {
+ Book1[]|error books1 = csv:parseString(csvString1, {nilValue: ""});
+ test:assertEquals(books1, [
+ {name: null, author: "b", year: "c"},
+ {name: "a", author: null, year: "c"},
+ {name: "a", author: "b", year: null}
+ ]);
+
+ Book1[]|error books2 = csv:parseString(csvString1);
+ test:assertEquals(books2, [
+ {name: "", author: "b", year: "c"},
+ {name: "a", author: "", year: "c"},
+ {name: "a", author: "b", year: ""}
+ ]);
+
+ (boolean|()|string|int)[][]|error arrbooks1 = csv:parseString(csvString1, {nilValue: ""});
+ test:assertEquals(arrbooks1, [
+ [null, "b", "c"],
+ ["a", null, "c"],
+ ["a", "b", null]
+ ]);
+
+ (boolean|()|string|int)[][2]|error arrbooks2 = csv:parseString(csvString1, {nilValue: ""});
+ test:assertEquals(arrbooks2, [
+ [null, "b"],
+ ["a", null],
+ ["a", "b"]
+ ]);
+
+ (boolean|()|string|int)[][]|error arrbooks3 = csv:parseString(csvString1);
+ test:assertEquals(arrbooks3, [
+ ["", "b", "c"],
+ ["a", "", "c"],
+ ["a", "b", ""]
+ ]);
+
+ (boolean|()|string|int)[][2]|error arrbooks4 = csv:parseString(csvString1);
+ test:assertEquals(arrbooks4, [
+ ["", "b"],
+ ["a", ""],
+ ["a", "b"]
+ ]);
+}

ballerina-tests/type-compatible-tests/tests/parse_string_compatibality_test.bal

@@ -20,6 +20,7 @@ import ballerina/io;
 import ballerina/test;
 
 const string filepath = "tests/csv_content.txt";
+const string filepath2 = "tests/csv_content_2.txt";
 const string errorFilepath = "tests/csv_error_content.txt";
 
 @test:Config
@@ -215,6 +216,7 @@ function testSpaceBetweendData() {
 @test:Config
 function testParseBytes() returns error? {
  byte[] csvBytes = check io:fileReadBytes(filepath);
+ byte[] csvBytes2 = check io:fileReadBytes(filepath2);
 
  record{}[]|csv:Error rec = csv:parseBytes(csvBytes, {});
  test:assertEquals(rec, [
@@ -245,17 +247,17 @@ function testParseBytes() returns error? {
  ["Hello World", "\"Hello World\"", "Hello World", "2"]
  ]);
 
- rec2 = csv:parseBytes(csvBytes, {outputWithHeaders: true, header: 1});
+ rec2 = csv:parseBytes(csvBytes2, {outputWithHeaders: true, header: 1});
  test:assertEquals(rec2, [
- ["Hello World", "\"Hello World\"", "Hello World", "2"],
- ["Hello World", "\"Hello World\"", "Hello World", "2"],
- ["Hello World", "\"Hello World\"", "Hello World", "2"]
+ ["Hello World1", "\"Hello World2\"", "Hello World3", "21"],
+ ["Hello World1", "\"Hello World2\"", "Hello World3", "22"],
+ ["Hello World1", "\"Hello World2\"", "Hello World3", "23"]
  ]);
 
- rec2 = csv:parseBytes(csvBytes, { header: 1});
+ rec2 = csv:parseBytes(csvBytes2, { header: 1});
  test:assertEquals(rec2, [
- ["Hello World", "\"Hello World\"", "Hello World", "2"],
- ["Hello World", "\"Hello World\"", "Hello World", "2"]
+ ["Hello World1", "\"Hello World2\"", "Hello World3", "22"],
+ ["Hello World1", "\"Hello World2\"", "Hello World3", "23"]
  ]);
 
  int[][]|csv:Error rec3 = csv:parseBytes(csvBytes, {});
@@ -270,6 +272,8 @@ function testParseBytes() returns error? {
 @test:Config
 function testParseStream() returns error? {
  stream<byte[], io:Error?> csvByteStream = check io:fileReadBlocksAsStream(filepath);
+ stream<byte[], io:Error?> csvByteStream2 = check io:fileReadBlocksAsStream(filepath2);
+
  record{}[]|csv:Error rec = csv:parseStream(csvByteStream, {});
  test:assertEquals(rec, [
  {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2},
@@ -293,12 +297,11 @@ function testParseStream() returns error? {
  ["Hello World", "\"Hello World\"", "Hello World", "2"]
  ]);
 
- csvByteStream = check io:fileReadBlocksAsStream(filepath);
- rec2 = csv:parseStream(csvByteStream, {header: 1, outputWithHeaders: true});
+ rec2 = csv:parseStream(csvByteStream2, {header: 1, outputWithHeaders: true});
  test:assertEquals(rec2, [
- ["Hello World", "\"Hello World\"", "Hello World", "2"],
- ["Hello World", "\"Hello World\"", "Hello World", "2"],
- ["Hello World", "\"Hello World\"", "Hello World", "2"]
+ ["Hello World1", "\"Hello World2\"", "Hello World3", "21"],
+ ["Hello World1", "\"Hello World2\"", "Hello World3", "22"],
+ ["Hello World1", "\"Hello World2\"", "Hello World3", "23"]
  ]);
 
  csvByteStream = check io:fileReadBlocksAsStream(filepath);
@@ -310,11 +313,11 @@ function testParseStream() returns error? {
  ["Hello World", "\"Hello World\"", "Hello World", "2"]
  ]);
 
- csvByteStream = check io:fileReadBlocksAsStream(filepath);
+ csvByteStream = check io:fileReadBlocksAsStream(filepath2);
  rec2 = csv:parseStream(csvByteStream, {header: 1});
  test:assertEquals(rec2, [
- ["Hello World", "\"Hello World\"", "Hello World", "2"],
- ["Hello World", "\"Hello World\"", "Hello World", "2"]
+ ["Hello World1", "\"Hello World2\"", "Hello World3", "22"],
+ ["Hello World1", "\"Hello World2\"", "Hello World3", "23"]
  ]);
 
  csvByteStream = check io:fileReadBlocksAsStream(filepath);

ballerina-tests/user-config-tests/tests/test_data_values.bal

@@ -121,3 +121,28 @@ string csvStringData9 = string `
  4@ string@ true@ 2.234@ ()@-3.21
 
  5@ string@ true@ 2.234@ -3.21@ null`; 
+
+string csvStringData10 = string `
+ hello, hello, (), 12, true, 12.34
+ // comment
+ 
+ a, b, c, d, e, f
+ 
+
+ 1, string1, true, 2.234, 2.234, ()
+ 2, string2, false, 0, 0, null
+ 3, string3, false, 1.23, 1.23, ()
+ 4, string4, true, -6.51, -6.52, ()
+ 5, string5, true, 3, 31, ()`;
+
+string csvStringData11 = string `
+ a, b, c, d, e, f
+
+
+
+ 1, string1, true, 2.234, 2.234, ()
+ 2, string2, false, 0, 0, null
+ 3, string3, false, 1.23, 1.23, ()
+
+ 4, string4, true, -6.51, -6.52, ()
+ 5, string5, true, 3, 3, ()`;

ballerina-tests/user-config-tests/tests/user_config_with_parser_options_test.bal

@@ -125,9 +125,9 @@ function testFromCsvStringWithParserOptions() {
  {a: 5, b: "string5", c: true, d: 3, e: 3, f: "()"}
  ]);
 
- record {}[]|csv:Error csv3op3_4 = csv:parseString(csvStringData3, {header: 9, skipLines: "2-10"});
+ record {}[]|csv:Error csv3op3_4 = csv:parseString(csvStringData11, {header: 9, skipLines: "2-10"});
  test:assertEquals(csv3op3_4, [
- {'4: 5, string4: "string5", "true": true, "-6.51": 3, "()": null}
+ {'4: 5, string4: "string5", "true": true, "-6.51": 3,  "-6.52": 3, "()": null}
  ]);
 }
 
@@ -203,15 +203,19 @@ function testHeaderOption() {
  {a: 5, b: "string5", c: true, d: 3, e: 3, f: ()}
  ]);
 
- record {}[]|csv:Error csv2cop2 = csv:parseString(csvStringData2, {header: 100});
+ record {}[]|csv:Error csv2cop2 = csv:parseString(csvStringData10, {header: 100});
  test:assertTrue(csv2cop2 is csv:Error);
  test:assertEquals((<error>csv2cop2).message(), "The provided header row is empty");
 
- record {}[]|csv:Error csv2cop3 = csv:parseString(csvStringData2, {header: 11});
+ record {}[]|csv:Error csv2cop3 = csv:parseString(csvStringData10, {header: 11});
  test:assertEquals(csv2cop3, []);
 
- record {}[]|csv:Error csv2cop4 = csv:parseString(csvStringData2, {header: 10});
- test:assertEquals(csv2cop4, [{'4: 5, string4: "string5", "true": true, "-6.51": 3, "()": ()}]);
+ record {}[]|csv:Error csv2cop3_2 = csv:parseString(csvStringData10, {header: 9});
+ test:assertTrue(csv2cop3_2 is csv:Error);
+ test:assertEquals((<error>csv2cop3_2).message(), "Duplicate header found: '1.23'");
+
+ record {}[]|csv:Error csv2cop4 = csv:parseString(csvStringData10, {header: 10});
+ test:assertEquals(csv2cop4, [{'4: 5, string4: "string5", "true": true, "-6.51": 3, "-6.52": 31, "()": ()}]);
 
  record {}[]|csv:Error csv1cop5 = csv:parseString(csvStringData1, {});
  test:assertTrue(csv1cop5 is csv:Error);
@@ -313,6 +317,9 @@ function testCommentConfigOption() {
  string csvValue9 = string `a,# b
  1 ,#2 # comment
  # comment`;
+ string csvValue10 = string `a# b
+ 1 ,#2 # comment
+ # comment`;
 
  record {int a;}[]|csv:Error cn;
 
@@ -345,8 +352,13 @@ function testCommentConfigOption() {
  // TODO:Fix the error message
  // test:assertEquals((<error> cn).message(), common:generateErrorMessageForInvalidCast("1, 2", "int"));
 
+ cn = csv:parseString(csvValue10);
+ test:assertTrue(cn is csv:Error);
+ test:assertEquals((<error> cn).message(), "Invalid length for the headers");
+
  cn = csv:parseString(csvValue9);
- test:assertEquals(cn, [{a: 1}]);
+ test:assertTrue(cn is csv:Error);
+ test:assertEquals((<error> cn).message(), "The provided header row is empty");
 }
 
 @test:Config {dependsOn: [testCSVLocale]}
@@ -375,6 +387,20 @@ function testCommentConfigOption2() {
 
  & comment`;
 
+ string csvValue7 = string `
+
+ a& b
+ 1 ,&2 & comment
+
+ & comment`;
+
+ string csvValue8 = string `
+
+ a,e& b
+ 1 ,&2 & comment
+
+ & comment`;
+
  record {int a; int b;}[]|csv:Error cn;
  record {int c;}[]|csv:Error cn2;
 
@@ -401,8 +427,16 @@ function testCommentConfigOption2() {
 
  cn = csv:parseString(csvValue6, {comment: "&", header: 2});
  test:assertTrue(cn is csv:Error);
+ test:assertEquals((<error>cn).message(), "The provided header row is empty");
+
+ cn = csv:parseString(csvValue8, {comment: "&", header: 2});
+ test:assertTrue(cn is csv:Error);
  test:assertEquals((<error>cn).message(), common:generateErrorMessageForMissingRequiredField("b"));
 
+ cn = csv:parseString(csvValue7, {comment: "&", header: 2});
+ test:assertTrue(cn is csv:Error);
+ test:assertEquals((<error>cn).message(), "Invalid length for the headers");
+
  cn2 = csv:parseString(csvValue1, {comment: "&"});
  test:assertTrue(cn2 is csv:Error);
  test:assertEquals((<error>cn2).message(), common:generateErrorMessageForMissingRequiredField("c"));
@@ -426,7 +460,7 @@ function testCommentConfigOption2() {
 
  cn2 = csv:parseString(csvValue6, {header: 2, comment: "&"});
  test:assertTrue(cn2 is csv:Error);
- test:assertEquals((<error>cn2).message(), common:generateErrorMessageForMissingRequiredField("c"));
+ test:assertEquals((<error>cn2).message(), "The provided header row is empty");
 }
 
 @test:Config {dependsOn: [testCSVLocale]}

native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvCreator.java

@@ -66,9 +66,9 @@ static Object initRowValue(Type expectedType) {
  };
  }
 
- static void convertAndUpdateCurrentJsonNode(CsvParser.StateMachine sm,
-  String value, Type type, CsvConfig config, Type exptype,
-  Field currentField) {
+ static void convertAndUpdateCurrentCsvNode(CsvParser.StateMachine sm,
+ String value, Type type, CsvConfig config, Type exptype,
+ Field currentField) {
  Object currentCsv = sm.currentCsvNode;
  Object nilValue = config.nilValue;
  if (sm.config.nilAsOptionalField && !type.isNilable()
@@ -91,6 +91,7 @@ static void convertAndUpdateCurrentJsonNode(CsvParser.StateMachine sm,
  case TypeTags.RECORD_TYPE_TAG:
  ((BMap<BString, Object>) currentCsv).put(StringUtils.fromString(getHeaderValueForColumnIndex(sm)),
  convertedValue);
+ sm.currentCsvNodeLength++;
  return;
  case TypeTags.ARRAY_TAG:
  ArrayType arrayType = (ArrayType) currentCsvNodeType;
@@ -99,9 +100,11 @@ static void convertAndUpdateCurrentJsonNode(CsvParser.StateMachine sm,
  return;
  }
  ((BArray) currentCsv).add(sm.columnIndex, convertedValue);
+ sm.currentCsvNodeLength++;
  return;
  case TypeTags.TUPLE_TAG:
  ((BArray) currentCsv).add(sm.columnIndex, convertedValue);
+ sm.currentCsvNodeLength++;
  return;
  default:
  }