Add tests for byte and byte streams

.github/workflows/build-with-bal-test-graalvm.yml

@@ -35,3 +35,5 @@ jobs:
  lang_tag: ${{ inputs.lang_tag }}
  lang_version: ${{ inputs.lang_version }}
  native_image_options: '-J-Xmx7G ${{ inputs.native_image_options }}'
+ # TODO : Enable after fixing this issue : https://github.com/ballerina-platform/ballerina-lang/issues/38882
+ additional_windows_build_flags: '-x test'

ballerina-tests/build.gradle

@@ -32,9 +32,8 @@ def testCommonTomlFilePlaceHolder = new File("${project.rootDir}/build-config/re
 def ballerinaDist = "${project.rootDir}/target/ballerina-runtime"
 def distributionBinPath = "${ballerinaDist}/bin"
 def testCoverageParam = "--code-coverage --coverage-format=xml --includes=io.ballerina.stdlib.data.*:ballerina.*"
-def testPackages = ["constraint-validation-tests", "parse-list-types-tests", "parse-record-types-tests",
- "parse-string-array-types-tests", "parse-string-record-types-tests", "type-compatible-tests",
- "union-type-tests", "user-config-tests"]
+def testPackages = ["user-config-tests", "type-compatible-tests", "unicode-tests", "constraint-validation-tests", "parse-list-types-tests", "parse-record-types-tests",
+ "parse-string-array-types-tests", "parse-string-record-types-tests", "union-type-tests"]
 def testCommonPackage = "csv-commons"
 
 def stripBallerinaExtensionVersion(String extVersion) {

ballerina-tests/type-compatible-tests/tests/csv_content.txt

@@ -0,0 +1,4 @@
+a, b, c d, e
+"Hello World", \"Hello World\", Hello World, 2
+"Hello World", \"Hello World\", Hello World, 2
+"Hello World", \"Hello World\", Hello World, 2

ballerina-tests/type-compatible-tests/tests/parse_string_compatibality_test.bal

@@ -1,7 +1,10 @@
 import ballerina/csv_commons as common;
 import ballerina/data.csv as csv;
+import ballerina/io;
 import ballerina/test;
 
+const string filepath = "tests/csv_content.txt";
+
 @test:Config
 function testFromCsvStringWithTypeCompatibility() {
  string value = string `i1,i2,s1,s2, b1,b2,n1,n2,f1,f2, d1,d2,j1,a1,j2,a2
@@ -180,4 +183,52 @@ function testFromCsvStringWithTypeCompatibility() {
  1.2, abc, true,1.0`);
  test:assertTrue(m3rra is csv:Error);
  test:assertEquals((<csv:Error>m3rra).message(), common:generateErrorMessageForInvalidCast("1.0", "int"));
-}
+}
+
+@test:Config
+function testSpaceBetweendData() {
+ string csv = string `a b, b d e, f
+ "Hello world", " Hi I am ", \" Hi I am \"`;
+
+ record{|string...;|}[]|csv:Error rec = csv:parseStringToRecord(csv);
+ test:assertEquals(rec, [
+ {"a b":"Hello world","b d e":" Hi I am ","f":"\"Hi I am \""}]);
+}
+
+@test:Config
+function testParseBytes() returns error? {
+ byte[] csvBytes = check io:fileReadBytes(filepath);
+
+ record{}[]|csv:Error rec = csv:parseBytesToRecord(csvBytes, {});
+ test:assertEquals(rec, [
+ {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2},
+ {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2},
+ {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2}]
+ );
+
+ string[][]|csv:Error rec2 = csv:parseBytesToList(csvBytes, {});
+ test:assertEquals(rec2, [
+ ["Hello World", "\"Hello World\"", "Hello World", "2"],
+ ["Hello World", "\"Hello World\"", "Hello World", "2"],
+ ["Hello World", "\"Hello World\"", "Hello World", "2"]
+ ]);
+}
+
+@test:Config
+function testParseStream() returns error? {
+ stream<byte[], io:Error?> csvByteStream = check io:fileReadBlocksAsStream(filepath);
+ record{}[]|csv:Error rec = csv:parseStreamToRecord(csvByteStream, {});
+ test:assertEquals(rec, [
+ {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2},
+ {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2},
+ {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2}]
+ );
+
+ csvByteStream = check io:fileReadBlocksAsStream(filepath);
+ string[][]|csv:Error rec2 = csv:parseStreamToList(csvByteStream, {});
+ test:assertEquals(rec2, [
+ ["Hello World", "\"Hello World\"", "Hello World", "2"],
+ ["Hello World", "\"Hello World\"", "Hello World", "2"],
+ ["Hello World", "\"Hello World\"", "Hello World", "2"]
+ ]);
+}

ballerina-tests/type-compatible-tests/tests/parse_type_compatibility_test.bal

@@ -5,8 +5,6 @@ import ballerina/test;
 @test:Config
 function testFromCsvWithTypeFunctionWithTypeCompatibility() {
  var value = {i1, i2, s1, s2, b1, b2, n1, n2, f1, f2, d1, d2, j1: b1, a1: d1, j2: b2, a2: d2};
- var value2 = {i1, s1, b1, n1, f1, d1, j1: b1, a1: d1, s2, s3, j2: b2, a2: d2};
- var value3 = {i1, s1, b1, n1, f1, d1, j1: b1, a1: d1, s2, s3};
 
  CustomRecord27Array|csv:Error vcr27a = csv:parseRecordAsRecordType([value, value, value], {}, CustomRecord27Array);
  test:assertEquals(vcr27a , [

ballerina-tests/unicode-tests/.gitignore

@@ -0,0 +1,11 @@
+# Ballerina generates this directory during the compilation of a package. 
+# It contains compiler-generated artifacts and the final executable if this is an application package.
+target/
+
+# Ballerina maintains the compiler-generated source code here.
+# Remove this if you want to commit generated sources.
+generated/
+
+# Contains configuration values used during development time.
+# See https://ballerina.io/learn/provide-values-to-configurable-variables/ for more details.
+Config.toml

ballerina-tests/unicode-tests/Ballerina.toml

@@ -0,0 +1,13 @@
+[package]
+org = "ballerina"
+name = "unicode_tests"
+version = "0.1.0"
+
+[[dependency]]
+org = "ballerina"
+name = "csv_commons"
+repository = "local"
+version = "0.1.0"
+
+[platform.java17]
+graalvmCompatible = true

ballerina-tests/unicode-tests/tests/escape_character_test.bal

@@ -0,0 +1,55 @@
+import ballerina/data.csv as csv;
+import ballerina/test;
+
+@test:Config
+function testEscapedCharactres() returns error? {
+ string csvString = string `a, b
+ quote\"\"quoted\"quote, 1
+ backslash\\backslash, 2
+ newline\nnewline, 3
+ tab\ttab, 5
+ unicode\u0061unicode, 6
+ slash\/slash, 9
+ quoted string \\'abc\\', 10`;
+
+ record{string a; int b;}[]|csv:Error rec = csv:parseStringToRecord(csvString);
+ test:assertEquals(rec, [
+ {a: string `quote""quoted"quote`, b: 1},
+ {a: string `backslash${"\\"}backslash`, b: 2},
+ {a: string `newline${"\n"}newline`, b: 3},
+ {a: string `tab${"\t"}tab`, b: 5},
+ {a: string `unicodeaunicode`, b: 6},
+ {a: string `slash/slash`, b: 9},
+ {a: string `quoted string \'abc\'`, b: 10}
+ ]);
+}
+
+@test:Config
+function testEscapedCharactres2() returns error? {
+ string csvString = string `a, b
+ backspace\bbackspace, 7`;
+
+ record{string a; int b;}[]|csv:Error rec = csv:parseStringToRecord(csvString);
+ test:assertTrue(rec is record{string a; int b;}[]);
+}
+
+@test:Config
+function testEscapedCharactres3() returns error? {
+ string csvString = string ` a c, b
+ carriage return\r carriage return, 4`;
+
+ record{}[]|csv:Error rec = csv:parseStringToRecord(csvString);
+ test:assertEquals(rec, [
+ {"a c": string `carriage return${"\r"} carriage return`, b: 4}
+ ]);
+}
+
+@test:Config
+function testEscapedCharactres4() returns error? {
+ string csvString = string `a, b
+ form feed\f form feed, 8`;
+
+ record{string a; int b;}[]|csv:Error rec = csv:parseStringToRecord(csvString);
+ test:assertTrue(rec is record {string a; int b;}[]);
+ // TODO: Add tests after supports \f by Ballerina
+}

ballerina-tests/union-type-tests/tests/test_with_singleton_test.bal

@@ -122,12 +122,10 @@ type SubtypeTuple3 [SubType...];
 function testSubtypeExpectedTypes() returns error? {
  var value1 = [{a: 1, c: 1, d: 1, e: 1, f: "a", g: 1, h: 1, i: 1},
  {a: 1, c: 1, d: 1, e: 1, f: "a", g: 1, h: 1, i: 1}];
- var value2 = [{a: 1, c: int:MAX_VALUE, d: 1, e: 1, f: "a", g: 1, h: 1, i: 1},
- {a: 1, c: 1, d: 1, e: 1, f: "a", g: 1, h: 1, i: 1}]; 
+ var value2 = [["1", "1", "1", "1", "a", "1", "1", "1"],
+ ["1", "1", "1", "1", "a", "1", "1", "1"]];
  var value3 = [[1, 1, 1, 1, "a", 1, 1, 1],
  [1, 1, 1, 1, "a", 1, 1, 1]];
- var value4 = [["1", "1", "1", "1", "a", "1", "1", "1"],
- ["1", "1", "1", "1", "a", "1", "1", "1"]];
 
  SubtypeRecord[]|csv:Error a = csv:parseStringToRecord(string `a, c, d, e, f, g, h, i
  1, 1, 1, 1, a, 1, 1, 1
@@ -182,24 +180,24 @@ function testSubtypeExpectedTypes() returns error? {
  ["a", "c", "d", "e", "f", "g", "h", "i"], {});
  test:assertEquals(a12, value3); 
 
- SubtypeRecord[]|csv:Error a13 = csv:parseListAsRecordType(value4, 
+ SubtypeRecord[]|csv:Error a13 = csv:parseListAsRecordType(value2, 
  ["a", "c", "d", "e", "f", "g", "h", "i"], {});
  test:assertEquals(a13, value1);
 
- SubtypeRecord2[]|csv:Error a14 = csv:parseListAsRecordType(value4, 
+ SubtypeRecord2[]|csv:Error a14 = csv:parseListAsRecordType(value2, 
  ["a", "c", "d", "e", "f", "g", "h", "i"], {});
  test:assertEquals(a14, [{a: 1, c: 1}, {a: 1, c: 1}]);
 
- SubtypeRecord3[]|csv:Error a15 = csv:parseListAsRecordType(value4, 
+ SubtypeRecord3[]|csv:Error a15 = csv:parseListAsRecordType(value2, 
  ["a", "c", "d", "e", "f", "g", "h", "i"], {});
  test:assertEquals(a15, value1);
 
- SubtypeTuple[]|csv:Error a16 = csv:parseListAsListType(value4, {});
+ SubtypeTuple[]|csv:Error a16 = csv:parseListAsListType(value2, {});
  test:assertEquals(a16, value3);
 
- SubtypeTuple2[]|csv:Error a17 = csv:parseListAsListType(value4, {});
+ SubtypeTuple2[]|csv:Error a17 = csv:parseListAsListType(value2, {});
  test:assertEquals(a17, [[1, 1], [1, 1]]);
 
- SubtypeTuple3[]|csv:Error a18 = csv:parseListAsListType(value4, {});
+ SubtypeTuple3[]|csv:Error a18 = csv:parseListAsListType(value2, {});
  test:assertEquals(a18, value3); 
 }

ballerina-tests/user-config-tests/tests/user_config_with_parser_options_test.bal

@@ -116,7 +116,7 @@ function testFromCsvStringWithHeaderLessParserOptions() {
  record {}[]|csv:Error csv2op6_2 = csv:parseStringToRecord(csvStringData2, {header: false, skipLines: [5, 7]});
  test:assertEquals(csv2op6_2, [
  {'1: "hello", '2: "hello", '3: (), '4: 12, '5: true, '6: 12.34},
- {'1: "//comment"},
+ {'1: "// comment"},
  {'1: "a", '2: "b", '3: "c", '4: "d", '5: "e", '6: "f"},
  {'1: 1, '2: "string1", '3: true, '4: 2.234, '5: 2.234, '6: ()},
  {'1: 3, '2: "string3", '3: false, '4: 1.23, '5: 1.23, '6: ()},
@@ -136,7 +136,8 @@ function testHeaderOption() {
  ]);
 
  record {}[]|csv:Error csv2cop2 = csv:parseStringToRecord(csvStringData2, {header: 100});
- test:assertEquals(csv2cop2, []);
+ test:assertTrue(csv2cop2 is csv:Error);
+ test:assertEquals((<error> csv2cop2).message(), "The provided header row is empty");
 
  record {}[]|csv:Error csv2cop3 = csv:parseStringToRecord(csvStringData2, {header: 11});
  test:assertEquals(csv2cop3, []);