From 78aabf2a9961d6e029f51da32fbf353d918b5a80 Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Wed, 14 Aug 2024 14:04:53 +0530 Subject: [PATCH 1/2] Update the order of union types --- .../tests/parse_list_type_as_list_test.bal | 2 +- .../tests/parse_list_type_as_record_test.bal | 10 +- .../tests/parse_string_to_array_test.bal | 11 +- .../tests/parse_string_to_record_tests.bal | 7 + .../tests/types.bal | 6 + .../tests/parse_type_compatibility_test.bal | 45 ++- .../tests/test_with_intersection_types.bal | 23 +- .../tests/test_with_singleton_test.bal | 26 +- .../tests/test_with_union_types.bal | 316 ++++++++++++++++-- .../sample_package_11/main.bal | 16 +- .../compiler/CsvDataTypeValidator.java | 3 + .../lib/data/csvdata/FromString.java | 21 +- .../lib/data/csvdata/csv/CsvCreator.java | 1 + .../lib/data/csvdata/csv/CsvParser.java | 10 +- .../lib/data/csvdata/csv/CsvTraversal.java | 109 ++---- .../csvdata/utils/DiagnosticErrorCode.java | 3 +- .../src/main/resources/csv_error.properties | 3 + 17 files changed, 464 insertions(+), 148 deletions(-) diff --git a/ballerina-tests/parse-list-types-tests/tests/parse_list_type_as_list_test.bal b/ballerina-tests/parse-list-types-tests/tests/parse_list_type_as_list_test.bal index 086e760..834f854 100644 --- a/ballerina-tests/parse-list-types-tests/tests/parse_list_type_as_list_test.bal +++ b/ballerina-tests/parse-list-types-tests/tests/parse_list_type_as_list_test.bal @@ -472,7 +472,7 @@ function testParseListsWithOutputHeaders() { ct1bt1 = csv:parseList([["a", "true", "1"], ["a", "true", "1"]], {headersRows: 21}); test:assertEquals(ct1bt1, []); - (string|boolean|int)[][]|csv:Error ct1bt1_2 = csv:parseList([["a", "b", "c"], ["a", "true", "1"]], {headersRows: 1}); + (boolean|int|string)[][]|csv:Error ct1bt1_2 = csv:parseList([["a", "b", "c"], ["a", "true", "1"]], {headersRows: 1}); test:assertEquals(ct1bt1_2, [ ["a", true, 1] ]); diff --git a/ballerina-tests/parse-list-types-tests/tests/parse_list_type_as_record_test.bal b/ballerina-tests/parse-list-types-tests/tests/parse_list_type_as_record_test.bal index 848e76e..28bd953 100644 --- a/ballerina-tests/parse-list-types-tests/tests/parse_list_type_as_record_test.bal +++ b/ballerina-tests/parse-list-types-tests/tests/parse_list_type_as_record_test.bal @@ -544,6 +544,14 @@ function testFromCsvWithTypeForTupleAndRecordAsExpectedType7() { [["a", "1", "true", "0", "2.23", "null"], ["a", "1", "true", "2.23", "0", "()"]], {customHeaders: ["f", "e", "d", "c", "b", "a"]}); test:assertEquals(ct1br9_2, [ + {a: (), b: 2.23, c: 0, d: "true", e: 1, f: "a"}, + {a: (), b: 0, c: 2.23, d: "true", e: 1, f: "a"} + ]); + + record{|int|() a; float b; decimal? c; boolean|string d; int|string e; string f; string...;|}[]|csv:Error ct1br9_3 = csv:parseList( + [["a", "1", "true", "0", "2.23", "null"], ["a", "1", "true", "2.23", "0", "()"]], + {customHeaders: ["f", "e", "d", "c", "b", "a"]}); + test:assertEquals(ct1br9_3, [ {a: (), b: 2.23, c: 0, d: true, e: 1, f: "a"}, {a: (), b: 0, c: 2.23, d: true, e: 1, f: "a"} ]); @@ -593,7 +601,7 @@ function testFromCsvWithTypeForTupleAndRecordAsExpectedTypeWithHeaders() { test:assertTrue(ct1br4_5_2 is csv:Error); test:assertEquals((ct1br4_5).message(), "Custom headers should be provided"); - map[]|csv:Error ct2br4_3 = csv:parseList([["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"]], {headersRows: 1, customHeaders: ["a", "c", "b"], outputWithHeaders: true}); + map[]|csv:Error ct2br4_3 = csv:parseList([["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"]], {headersRows: 1, customHeaders: ["a", "c", "b"], outputWithHeaders: true}); test:assertEquals(ct2br4_3, [ {a: "a", b: true, c: 1}, {a: "a", b: true, c: 1} diff --git a/ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_array_test.bal b/ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_array_test.bal index dfeaf58..e6b3565 100644 --- a/ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_array_test.bal +++ b/ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_array_test.bal @@ -407,20 +407,27 @@ function testParseStringArrayAsExpectedTypeWithOutputHeaders() { ["true", "false", "true", "false"] ]); - (string|boolean)[][]|csv:Error cv2baa = csv:parseString(csvStringWithBooleanValues2, {outputWithHeaders: true}); + (boolean|string)[][]|csv:Error cv2baa = csv:parseString(csvStringWithBooleanValues2, {outputWithHeaders: true}); test:assertEquals(cv2baa, [ ["b1", "b2", "b3", "b4", "b5"], [true, false, true, false, true], [true, false, true, false, true] ]); - [string...][]|csv:Error cv2baa_2 = csv:parseString(csvStringWithBooleanValues2, {outputWithHeaders: true}); + (string|boolean)[][]|csv:Error cv2baa_2 = csv:parseString(csvStringWithBooleanValues2, {outputWithHeaders: true}); test:assertEquals(cv2baa_2, [ ["b1", "b2", "b3", "b4", "b5"], ["true", "false", "true", "false", "true"], ["true", "false", "true", "false", "true"] ]); + [string...][]|csv:Error cv2baa_2_2 = csv:parseString(csvStringWithBooleanValues2, {outputWithHeaders: true}); + test:assertEquals(cv2baa_2_2, [ + ["b1", "b2", "b3", "b4", "b5"], + ["true", "false", "true", "false", "true"], + ["true", "false", "true", "false", "true"] + ]); + [boolean|string...][]|csv:Error cv2baa_3 = csv:parseString(csvStringWithBooleanValues2, {outputWithHeaders: true}); test:assertEquals(cv2baa_3, [ ["b1", "b2", "b3", "b4", "b5"], diff --git a/ballerina-tests/parse-string-record-types-tests/tests/parse_string_to_record_tests.bal b/ballerina-tests/parse-string-record-types-tests/tests/parse_string_to_record_tests.bal index 3a212dc..1d18803 100644 --- a/ballerina-tests/parse-string-record-types-tests/tests/parse_string_to_record_tests.bal +++ b/ballerina-tests/parse-string-record-types-tests/tests/parse_string_to_record_tests.bal @@ -381,6 +381,13 @@ function testFromCsvStringWithTypeForStringAndRecordAsExpectedType4() { test:assertTrue(csvb1br12 is csv:Error); test:assertEquals((csvb1br12).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + BooleanRecord13_2[]|csv:Error csvb1br13_2 = csv:parseString(csvStringWithBooleanValues1, {}); + test:assertEquals(csvb1br13_2, [ + {b1: "true", b2: "false", b3: "true", b4: "false", defaultableField: "", nillableField: ()}, + {b1: "true", b2: "false", b3: "true", b4: "false", defaultableField: "", nillableField: ()}, + {b1: "true", b2: "false", b3: "true", b4: "false", defaultableField: "", nillableField: ()} + ]); + BooleanRecord13Array|csv:Error csvb1br13 = csv:parseString(csvStringWithBooleanValues1, {}); test:assertEquals(csvb1br13, [ {b1: true, b2: false, b3: true, b4: false, defaultableField: "", nillableField: ()}, diff --git a/ballerina-tests/parse-string-record-types-tests/tests/types.bal b/ballerina-tests/parse-string-record-types-tests/tests/types.bal index 6c50a58..0a189a3 100644 --- a/ballerina-tests/parse-string-record-types-tests/tests/types.bal +++ b/ballerina-tests/parse-string-record-types-tests/tests/types.bal @@ -94,6 +94,12 @@ type BooleanRecord12 record {| |}; type BooleanRecord13 record {| + string defaultableField = ""; + string? nillableField = (); + boolean|string...; +|}; + +type BooleanRecord13_2 record {| string defaultableField = ""; string? nillableField = (); string|boolean...; diff --git a/ballerina-tests/type-compatible-tests/tests/parse_type_compatibility_test.bal b/ballerina-tests/type-compatible-tests/tests/parse_type_compatibility_test.bal index cedf454..325109a 100644 --- a/ballerina-tests/type-compatible-tests/tests/parse_type_compatibility_test.bal +++ b/ballerina-tests/type-compatible-tests/tests/parse_type_compatibility_test.bal @@ -132,9 +132,9 @@ function testFromCsvWithIntersectionTypeCompatibility2() { 3,string3,true`); test:assertEquals(r2a, [ - {a: 1, b: "string", c: true}, - {a: 2, b: "string2", c: false}, - {a: 3, b: "string3", c: true} + {a: 1d, b: "string", c: "true"}, + {a: 2d, b: "string2", c: "false"}, + {a: 3d, b: "string3", c: "true"} ]); record {A2 a; B2 b; C2 c;}[]|csv:Error r3a = csv:parseString(string `a,b,c @@ -190,9 +190,9 @@ function testFromCsvWithIntersectionTypeCompatibility2() { 3,string3,true`); test:assertEquals(r17a, [ - [1, "string", true], - [2, "string2", false], - [3, "string3", true] + [1d, "string", "true"], + [2d, "string2", "false"], + [3d, "string3", "true"] ]); [A2, B2, C2][]|csv:Error r18a = csv:parseString( @@ -299,9 +299,9 @@ function testFromCsvWithIntersectionTypeCompatibility2() { , {headersOrder: ["a", "b", "c"]}); test:assertEquals(rt7a, [ - [1, "string", true], - [2, "string2", false], - [3, "string3", true] + [1d, "string", true], + [2d, "string2", false], + [3d, "string3", true] ]); [A2, B2, C2][]|csv:Error rt8a = csv:transform( @@ -348,18 +348,27 @@ function testFromCsvWithIntersectionTypeCompatibility2() { [["1", "string", "true"], ["2", "string2", "false"], ["3", "string3", "true"]], {customHeaders: ["a", "b", "c"]}); test:assertEquals(rt12a, [ - {a: 1, b: "string", c: true}, - {a: 2, b: "string2", c: false}, - {a: 3, b: "string3", c: true} + {a: 1d, b: "string", c: "true"}, + {a: 2d, b: "string2", c: "false"}, + {a: 3d, b: "string3", c: "true"} ]); record {string|decimal a; B b; C c;}[]|csv:Error rt12a_3 = csv:parseList( [["1", "string", "true"], ["2", "string2", "false"], ["3", "string3", "true"]], {customHeaders: ["a", "b", "c"]}); test:assertEquals(rt12a_3, [ - {a: 1, b: "string", c: true}, - {a: 2, b: "string2", c: false}, - {a: 3, b: "string3", c: true} + {a: "1", b: "string", c: "true"}, + {a: "2", b: "string2", c: "false"}, + {a: "3", b: "string3", c: "true"} + ]); + + record {decimal|string a; B b; C c;}[]|csv:Error rt12a_4 = csv:parseList( + [["1", "string", "true"], ["2", "string2", "false"], ["3", "string3", "true"]], {customHeaders: ["a", "b", "c"]}); + + test:assertEquals(rt12a_4, [ + {a: 1, b: "string", c: "true"}, + {a: 2, b: "string2", c: "false"}, + {a: 3, b: "string3", c: "true"} ]); record {A2 a; B2 b; C2 c;}[]|csv:Error rt13a = csv:parseList( @@ -409,9 +418,9 @@ function testFromCsvWithIntersectionTypeCompatibility2() { [["1", "string", "true"], ["2", "string2", "false"], ["3", "string3", "true"]]); test:assertEquals(rt17a, [ - [1, "string", true], - [2, "string2", false], - [3, "string3", true] + [1d, "string", "true"], + [2d, "string2", "false"], + [3d, "string3", "true"] ]); [A2, B2, C2][]|csv:Error rt18a = csv:parseList( diff --git a/ballerina-tests/union-type-tests/tests/test_with_intersection_types.bal b/ballerina-tests/union-type-tests/tests/test_with_intersection_types.bal index 5d37b8b..726ef47 100644 --- a/ballerina-tests/union-type-tests/tests/test_with_intersection_types.bal +++ b/ballerina-tests/union-type-tests/tests/test_with_intersection_types.bal @@ -70,7 +70,7 @@ function testIntersectionExpectedTypes() returns error? { 1,2 a,a`); test:assertTrue(a9 is (((int[] & readonly)|([string, string] & readonly)) & readonly)[]); - test:assertEquals(a9, [[1, 2], ["a", "a"]]); + test:assertEquals(a9, [["1", "2"], ["a", "a"]]); ((record {string a; string b;} & readonly)|(record {int a; int b;} & readonly))[] & readonly|csv:Error a10 = csv:parseString(string `a,b @@ -115,13 +115,22 @@ function testIntersectionExpectedTypes2() returns error? { test:assertEquals(a8, [{a: "a", b: "a"}, {a: "c", b: "c"}]); (((int[] & readonly)|([string, string] & readonly)) & readonly)[]|csv:Error a9 = csv:transform([{"a": 1, "b": 2}, {"a": "a", "b": "b"}], {headersOrder: ["a", "b"]}); - test:assertTrue(a9 is (((int[] & readonly)|([string, string] & readonly)) & readonly)[]); - test:assertEquals(a9, [[1, 2], ["a", "b"]]); + test:assertTrue(a9 is error); + test:assertEquals(( a9).message(), "The CSV cannot be converted into any of the uniform union types in '((int[] & readonly)|([string,string] & readonly))[]'"); + + (((int[] & readonly)|([string, string] & readonly)) & readonly)[]|csv:Error a9_2 = csv:transform([{"a": "1", "b": "2"}, {"a": "a", "b": "b"}], {headersOrder: ["a", "b"]}); + test:assertTrue(a9_2 is (((int[] & readonly)|([string, string] & readonly)) & readonly)[]); + test:assertEquals(a9_2, [["1", "2"], ["a", "b"]]); ((record {string a; string b;} & readonly)|(record {int a; int b;} & readonly))[] - & readonly|csv:Error a10 = csv:transform([{"a": "a", "b": "a"}, {"a": 1, "b": 2}], {}); + & readonly|csv:Error a10 = csv:transform([{"a": "a", "b": "a"}, {"a": "1", "b": "2"}], {}); test:assertTrue(a10 is ((record {string a; string b;} & readonly)|(record {int a; int b;} & readonly))[] & readonly); - test:assertEquals(a10, [{a: "a", b: "a"}, {a: 1, b: 2}]); + test:assertEquals(a10, [{a: "a", b: "a"}, {a: "1", b: "2"}]); + + ((record {string a; string b;} & readonly)|(record {int a; int b;} & readonly))[] + & readonly|csv:Error a10_2 = csv:transform([{"a": "a", "b": "a"}, {"a": 1, "b": 2}], {}); + test:assertTrue(a10_2 is error); + test:assertEquals(( a10_2).message(), "The CSV cannot be converted into any of the uniform union types in '((union_type_tests:record {| string a; string b; anydata...; |} & readonly)|(union_type_tests:record {| int a; int b; anydata...; |} & readonly))[]'"); } @test:Config @@ -160,7 +169,7 @@ function testIntersectionExpectedTypes3() returns error? { (((int[] & readonly)|([string, string] & readonly)) & readonly)[]|csv:Error a9 = csv:parseList([["1", "2"], ["a", "b"]], {}); test:assertTrue(a9 is (((int[] & readonly)|([string, string] & readonly)) & readonly)[]); - test:assertEquals(a9, [[1, 2], ["a", "b"]]); + test:assertEquals(a9, [["1", "2"], ["a", "b"]]); ((record {string a; string b;} & readonly)|(record {int a; int b;} & readonly))[] & readonly|csv:Error a10 = csv:parseList([["a", "a"], ["1", "2"]], {customHeaders: ["a", "b"]}); @@ -170,5 +179,5 @@ function testIntersectionExpectedTypes3() returns error? { ((record {int a; int b;} & readonly)|(record {string a; string b;} & readonly))[] & readonly|csv:Error a11 = csv:parseList([["a", "a"], ["1", "2"]], {customHeaders: ["a", "b"]}); test:assertTrue(a11 is ((record {string a; string b;} & readonly)|(record {int a; int b;} & readonly))[] & readonly); - test:assertEquals(a11, [{a: "a", b: "a"}, {a: 1, b: 2}]); + test:assertEquals(a11, [{a: "a", b: "a"}, {a: "1", b: "2"}]); } diff --git a/ballerina-tests/union-type-tests/tests/test_with_singleton_test.bal b/ballerina-tests/union-type-tests/tests/test_with_singleton_test.bal index cf6a55a..4bf4c7f 100644 --- a/ballerina-tests/union-type-tests/tests/test_with_singleton_test.bal +++ b/ballerina-tests/union-type-tests/tests/test_with_singleton_test.bal @@ -110,7 +110,7 @@ function testSingletonExpectedTypes2() returns error? { test:assertEquals((a8).message(), common:generateErrorMessageForInvalidCast("c", "(\"a\"|\"d\")")); } -type SubType byte|int:Signed8|int:Signed16|int:Signed32|string:Char|int:Unsigned8|int:Unsigned16|int:Unsigned32; +type SubType byte|int:Signed8|int:Signed16|int:Signed32|int:Unsigned8|int:Unsigned16|int:Unsigned32|string:Char; type SubtypeRecord record { byte a; int:Signed8 c; int:Signed16 d; int:Signed32 e; @@ -142,6 +142,14 @@ function testSubtypeExpectedTypes() returns error? { ["1", "1", "1", "1", "a", "1", "1", "1"]]; var value3 = [[1, 1, 1, 1, "a", 1, 1, 1], [1, 1, 1, 1, "a", 1, 1, 1]]; + var value4 = [[1, 1, 1, 1, "a", 1, 1, 1], + [1, 1, 1, 1, "a", 1, 1, 1]]; + var value5 = [{a: 1, c: 1, d: 1, e: 1, f: "a", g: 1, h: 1, i: 1}, + {a: 1, c: 1, d: 1, e: 1, f: "a", g: 1, h: 1, i: 1}]; + var value6 = [[1, 1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 1]]; + var value7 = [{a: 1, c: 1, d: 1, e: 1, g: 1, h: 1, i: 1}, + {a: 1, c: 1, d: 1, e: 1, g: 1, h: 1, i: 1}]; SubtypeRecord[]|csv:Error a = csv:parseString(string `a, c, d, e, f, g, h, i 1, 1, 1, 1, a, 1, 1, 1 @@ -154,10 +162,15 @@ function testSubtypeExpectedTypes() returns error? { test:assertEquals(a2, [{a: 1, c: 1}, {a: 1, c: 1}]); - SubtypeRecord3[]|csv:Error a3 = csv:parseString(string `a, c, d, e, f, g, h, i + SubtypeRecord3[]|csv:Error a3 = csv:parseString(string `a, c, d, e, g, h, i + 1, 1, 1, 1, 1, 1, 1 + 1, 1, 1, 1, 1, 1, 1 `); + test:assertEquals(a3, value7); + + SubtypeRecord3[]|csv:Error a3_2 = csv:parseString(string `a, c, d, e, f, g, h, i 1, 1, 1, 1, a, 1, 1, 1 1, 1, 1, 1, a, 1, 1, 1 `); - test:assertEquals(a3, value1); + test:assertEquals(a3_2, value5); SubtypeTuple[]|csv:Error a4 = csv:parseString(string `a, c, d, e, f, g, h, i 1, 1, 1, 1, a, 1, 1, 1 @@ -172,7 +185,12 @@ function testSubtypeExpectedTypes() returns error? { SubtypeTuple3[]|csv:Error a6 = csv:parseString(string `a, c, d, e, f, g, h, i 1, 1, 1, 1, a, 1, 1, 1 1, 1, 1, 1, a, 1, 1, 1 `); - test:assertEquals(a6, value3); + test:assertEquals(a6, value4); + + SubtypeTuple3[]|csv:Error a6_2 = csv:parseString(string `a, c, d, e, g, h, i + 1, 1, 1, 1, 1, 1, 1 + 1, 1, 1, 1, 1, 1, 1 `); + test:assertEquals(a6_2, value6); SubtypeRecord[]|csv:Error a7 = csv:transform(value1, {}); test:assertEquals(a7, value1); diff --git a/ballerina-tests/union-type-tests/tests/test_with_union_types.bal b/ballerina-tests/union-type-tests/tests/test_with_union_types.bal index f7dda6b..f9246ce 100644 --- a/ballerina-tests/union-type-tests/tests/test_with_union_types.bal +++ b/ballerina-tests/union-type-tests/tests/test_with_union_types.bal @@ -119,30 +119,30 @@ function testParseToStringWithUnionExpectedTypes() returns error? { 1, 2 a, b`, {header: 1}); test:assertEquals(csv1op12, [ - {a: 1, b: 2}, + {a: "1", b: "2"}, {a: "a", b: "b"} ]); - ([int, int]|[string, string])[]|csv:Error csv1op13 = csv:parseString(string ` + ([int, int]|[string|int, string|int])[]|csv:Error csv1op13 = csv:parseString(string ` a,b 1, 2 a, b`, {header: 1}); test:assertEquals(csv1op13, [ - [1, 2], + ["1", "2"], ["a", "b"] ]); } +record{int a; string b; boolean c; decimal d; float e; () f;}[] value = [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 2, b: "string2", c: false, d: 0, e: 0, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 4, b: "string4", c: true, d: -6.51, e: -6.51, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} +]; + @test:Config function testParseToStringWithUnionExpectedTypes2() returns error? { - record{int a; string b; boolean c; decimal d; float e; () f;}[] value = [ - {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, - {a: 2, b: "string2", c: false, d: 0, e: 0, f: ()}, - {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, - {a: 4, b: "string4", c: true, d: -6.51, e: -6.51, f: ()}, - {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} - ]; - (RecA|RecC)[]|csv:Error csv1op1 = csv:transform(value, {}); test:assertEquals(csv1op1, [ {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, @@ -233,13 +233,17 @@ function testParseToStringWithUnionExpectedTypes2() returns error? { (record{|string a; int...;|}|record{|string a; string...;|})[]|csv:Error csv1op12 = csv:transform(value, {}); test:assertTrue(csv1op12 is csv:Error); - test:assertEquals((csv1op12).message(), "The source value cannot convert in to the '(union_type_tests:record {| string a; int...; |}|union_type_tests:record {| string a; string...; |})[]'"); + test:assertEquals((csv1op12).message(), "The CSV cannot be converted into any of the uniform union types in '(union_type_tests:record {| string a; int...; |}|union_type_tests:record {| string a; string...; |})[]'"); - (record{|int a; int...;|}|record{|string a; string...;|})[]|csv:Error csv1op13 = csv:transform([{"a": 1, "b": 2}, {"a": "a", "b": "b"}], {}); + (record{|int a; int...;|}|record{|int|string a; int|string...;|})[]|csv:Error csv1op13 = csv:transform([{"a": 1, "b": 2}, {"a": "a", "b": "b"}], {}); test:assertEquals(csv1op13, [ {a: 1, b: 2}, {a: "a", b: "b"} ]); + + (record{|int a; int...;|}|record{|string a; string...;|})[]|csv:Error csv1op14 = csv:transform([{"a": 1, "b": 2}, {"a": "a", "b": "b"}], {}); + test:assertTrue(csv1op14 is csv:Error); + test:assertEquals((csv1op14).message(), "The CSV cannot be converted into any of the uniform union types in '(union_type_tests:record {| int a; int...; |}|union_type_tests:record {| string a; string...; |})[]'"); } @test:Config @@ -338,7 +342,7 @@ function testParseToStringWithUnionExpectedTypes3() returns error? { (record{|int a; int...;|}|record{|string a; string...;|})[]|csv:Error csv1op13 = csv:parseList([["1", "2"], ["a", "b"]], {customHeaders: ["a", "b"]}); test:assertEquals(csv1op13, [ - {a: 1, b: 2}, + {a: "1", b: "2"}, {a: "a", b: "b"} ]); } @@ -387,7 +391,7 @@ function testParseToStringWithUnionExpectedTypes4() returns error? { (TupB|[boolean])[]|csv:Error csv1op4_2 = csv:transform(value, {headersOrder: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); test:assertTrue(csv1op4_2 is csv:Error); - test:assertEquals((csv1op4_2).message(), "The source value cannot convert in to the '(union_type_tests:TupB|[boolean])[]'"); + test:assertEquals((csv1op4_2).message(), "The CSV cannot be converted into any of the uniform union types in '(union_type_tests:TupB|[boolean])[]'"); (TupA|TupB)[]|csv:Error csv1op5 = csv:transform(value, {headersOrder: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); test:assertEquals(csv1op5, [ @@ -412,29 +416,33 @@ function testParseToStringWithUnionExpectedTypes4() returns error? { ([string...]|[int...])[]|csv:Error csv1op8 = csv:transform(value, {headersOrder: ["a", "b", "c", "d", "e", "f"]}); test:assertTrue(csv1op8 is csv:Error); - test:assertEquals((csv1op8).message(), "The source value cannot convert in to the '([string...]|[int...])[]'"); + test:assertEquals((csv1op8).message(), "The CSV cannot be converted into any of the uniform union types in '([string...]|[int...])[]'"); ([int...]|[string...])[]|csv:Error csv1op9 = csv:transform(value, {headersOrder: ["a", "b", "c", "d", "e", "f"]}); test:assertTrue(csv1op9 is csv:Error); - test:assertEquals((csv1op9).message(), "The source value cannot convert in to the '([int...]|[string...])[]'"); + test:assertEquals((csv1op9).message(), "The CSV cannot be converted into any of the uniform union types in '([int...]|[string...])[]'"); ([int, string...]|[string, int...])[]|csv:Error csv1op10 = csv:transform(value, {headersOrder: ["a", "b", "c", "d", "e", "f"]}); test:assertTrue(csv1op10 is csv:Error); - test:assertEquals((csv1op10).message(), "The source value cannot convert in to the '([int,string...]|[string,int...])[]'"); + test:assertEquals((csv1op10).message(), "The CSV cannot be converted into any of the uniform union types in '([int,string...]|[string,int...])[]'"); ([string, int...]|[int, string...])[]|csv:Error csv1op11 = csv:transform(value, {headersOrder: ["a", "b", "c", "d", "e", "f"]}); test:assertTrue(csv1op11 is csv:Error); - test:assertEquals((csv1op11).message(), "The source value cannot convert in to the '([string,int...]|[int,string...])[]'"); + test:assertEquals((csv1op11).message(), "The CSV cannot be converted into any of the uniform union types in '([string,int...]|[int,string...])[]'"); ([string, int...]|[string, string...])[]|csv:Error csv1op12 = csv:transform(value, {headersOrder: ["a", "b", "c", "d", "e", "f"]}); test:assertTrue(csv1op12 is csv:Error); - test:assertEquals((csv1op12).message(), "The source value cannot convert in to the '([string,int...]|[string,string...])[]'"); + test:assertEquals((csv1op12).message(), "The CSV cannot be converted into any of the uniform union types in '([string,int...]|[string,string...])[]'"); - ([int, int...]|[string, string...])[]|csv:Error csv1op13 = csv:transform([{"a": 1, "b": 2}, {"a": "a", "b": "b"}], {headersOrder: ["a", "b"]}); + ([string, string...]|[int|string, int|string...])[]|csv:Error csv1op13 = csv:transform([{"a": 1, "b": 2}, {"a": "a", "b": "b"}], {headersOrder: ["a", "b"]}); test:assertEquals(csv1op13, [ [1, 2], ["a", "b"] ]); + + ([int, int...]|[string, string...])[]|csv:Error csv1op14 = csv:transform([{"a": 1, "b": 2}, {"a": "a", "b": "b"}], {headersOrder: ["a", "b"]}); + test:assertTrue(csv1op14 is csv:Error); + test:assertEquals((csv1op14).message(), "The CSV cannot be converted into any of the uniform union types in '([int,int...]|[string,string...])[]'"); } @test:Config @@ -480,7 +488,7 @@ function testParseToStringWithUnionExpectedTypes5() returns error? { (TupB|[boolean])[]|csv:Error csv1op4_2 = csv:parseList(value, {skipLines: [2, 4]}); test:assertTrue(csv1op4_2 is csv:Error); - test:assertEquals((csv1op4_2).message(), "The source value cannot convert in to the '(union_type_tests:TupB|[boolean])[]'"); + test:assertEquals((csv1op4_2).message(), "The CSV cannot be converted into any of the uniform union types in '(union_type_tests:TupB|[boolean])[]'"); (TupA|TupB)[]|csv:Error csv1op5 = csv:parseList(value, {skipLines: [2, 4]}); test:assertEquals(csv1op5, [ @@ -532,7 +540,7 @@ function testParseToStringWithUnionExpectedTypes5() returns error? { ([int, int...]|[string, string...])[]|csv:Error csv1op13 = csv:parseList([["1", "2"], ["a", "b"]], {}); test:assertEquals(csv1op13, [ - [1, 2], + ["1", "2"], ["a", "b"] ]); } @@ -1022,3 +1030,265 @@ function testParseToStringWithUnionExpectedTypes10() returns error? { [string, int...][]|[string, string...][]|csv:Error csv1op12 = csv:parseList(value, {}); test:assertEquals(csv1op12, value); } + +@test:Config +function testUnionTypeWithOrdering() returns error? { + string[][] value1 = [["1", "1.0", "true", "a"], ["2", "2.0", "false", "b"]]; + string value2 = string `a,b, c, d + 1, 1.0, true, a + 2, 2.0, false, b`; + record{}[] value3 = [{"a": 1, "b": 1.0, "c": true, "d": "a"}, {"a": 2, "b": 2.0, "c": false, "d":"b"}]; + + string[][]|anydata[][]|error csv1op1 = csv:parseList(value1); + test:assertEquals(csv1op1, [ + ["1", "1.0", "true", "a"], + ["2", "2.0", "false", "b"] + ]); + + anydata[][]|string[][]|error csv1op2 = csv:parseList(value1); + test:assertEquals(csv1op2, [ + [1, 1.0, true, "a"], + [2, 2.0, false, "b"] + ]); + + string[][2]|float[][2]|error csv1op1_2 = csv:parseList(value1); + test:assertEquals(csv1op1_2, [ + ["1", "1.0"], + ["2", "2.0"] + ]); + + float[][2]|string[][2]|error csv1op2_2 = csv:parseList(value1); + test:assertEquals(csv1op2_2, [ + [1f, 1.0], + [2f, 2.0] + ]); + + (string|anydata)[][]|error csv1op1_3 = csv:parseList(value1); + test:assertEquals(csv1op1_3, [ + ["1", "1.0", "true", "a"], + ["2", "2.0", "false", "b"] + ]); + + (anydata|string)[][]|error csv1op2_3 = csv:parseList(value1); + test:assertEquals(csv1op2_3, [ + [1, 1.0, true, "a"], + [2, 2.0, false, "b"] + ]); + + (string|float)[][2]|error csv1op1_4 = csv:parseList(value1); + test:assertEquals(csv1op1_4, [ + ["1", "1.0"], + ["2", "2.0"] + ]); + + (float|string)[][2]|error csv1op2_4 = csv:parseList(value1); + test:assertEquals(csv1op2_4, [ + [1f, 1.0], + [2f, 2.0] + ]); + + string[][]|anydata[][]|error recCsv1op1 = csv:transform(value3); + test:assertEquals(recCsv1op1, [ + [1, 1.0, true, "a"], + [2, 2.0, false, "b"] + ]); + + anydata[][]|string[][]|error recCsv1op2 = csv:transform(value3); + test:assertEquals(recCsv1op2, [ + [1, 1.0, true, "a"], + [2, 2.0, false, "b"] + ]); + + string[][2]|float[][2]|error recCsv1op1_2 = csv:transform(value3); + test:assertEquals(recCsv1op1_2, [ + [1f, 1.0], + [2f, 2.0] + ]); + + float[][2]|string[][2]|error recCsv1op2_2 = csv:transform(value3); + test:assertEquals(recCsv1op2_2, [ + [1f, 1.0], + [2f, 2.0] + ]); + + (string|anydata)[][]|error recCsv1op1_3 = csv:transform(value3); + test:assertEquals(recCsv1op1_3, [ + [1, 1.0, true, "a"], + [2, 2.0, false, "b"] + ]); + + (anydata|string)[][]|error recCsv1op2_3 = csv:transform(value3); + test:assertEquals(recCsv1op2_3, [ + [1, 1.0, true, "a"], + [2, 2.0, false, "b"] + ]); + + (string|float)[][2]|error recCsv1op1_4 = csv:transform(value3); + test:assertEquals(recCsv1op1_4, [ + [1f, 1.0], + [2f, 2.0] + ]); + + (float|string)[][2]|error recCsv1op2_4 = csv:transform(value3); + test:assertEquals(recCsv1op2_4, [ + [1f, 1.0], + [2f, 2.0] + ]); + + string[][]|anydata[][]|error parseStrCsv1op1 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op1, [ + ["1", "1.0", "true", "a"], + ["2", "2.0", "false", "b"] + ]); + + anydata[][]|string[][]|error parseStrCsv1op2 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op2, [ + [1, 1.0, true, "a"], + [2, 2.0, false, "b"] + ]); + + string[][2]|float[][2]|error parseStrCsv1op1_2 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op1_2, [ + ["1", "1.0"], + ["2", "2.0"] + ]); + + float[][2]|string[][2]|error parseStrCsv1op2_2 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op2_2, [ + [1f, 1.0], + [2f, 2.0] + ]); + + (string|anydata)[][]|error parseStrCsv1op1_3 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op1_3, [ + ["1", "1.0", "true", "a"], + ["2", "2.0", "false", "b"] + ]); + + (anydata|string)[][]|error parseStrCsv1op2_3 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op2_3, [ + [1, 1.0, true, "a"], + [2, 2.0, false, "b"] + ]); + + (string|float)[][2]|error parseStrCsv1op1_4 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op1_4, [ + ["1", "1.0"], + ["2", "2.0"] + ]); + + (float|string)[][2]|error parseStrCsv1op2_4 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op2_4, [ + [1f, 1.0], + [2f, 2.0] + ]); +} + +@test:Config +function testUnionTypeWithNull() returns error? { + string[][] value1 = [["1", "()", "true", "a"], ["2", "()", "false", "b"]]; + string value2 = string `a,b, c, d + 1, (), true, a + 2, (), false, b`; + record{}[] value3 = [{"a": 1, "b": (), "c": true, "d": "a"}, {"a": 2, "b": (), "c": false, "d":"b"}]; + string value4 = string `a,b, c, d + 1, , true, a + 2, , false, b`; + string value5 = string `a,b, c, d + 1, null, true, a + 2, null, false, b`; + string value6 = string `a,b, c, d + 1, N/A, true, a + 2, N/A, false, b`; + string[][] value7 = [["1", "null", "true", "a"], ["2", "null", "false", "b"]]; + + string?[][2]|float[][2]|error csv1op1_2 = csv:parseList(value1); + test:assertEquals(csv1op1_2, [ + ["1", ()], + ["2", ()] + ]); + + csv1op1_2 = csv:parseList(value7); + test:assertEquals(csv1op1_2, [ + ["1", ()], + ["2", ()] + ]); + + float[][2]|string?[][2]|error csv1op2_2 = csv:parseList(value1); + test:assertEquals(csv1op2_2, [ + ["1", ()], + ["2", ()] + ]); + + csv1op2_2 = csv:parseList(value7); + test:assertEquals(csv1op2_2, [ + ["1", ()], + ["2", ()] + ]); + + (string|float?)[][2]|error csv1op1_4 = csv:parseList(value1); + test:assertEquals(csv1op1_4, [ + ["1", ()], + ["2", ()] + ]); + + csv1op1_4 = csv:parseList(value7); + test:assertEquals(csv1op1_4, [ + ["1", ()], + ["2", ()] + ]); + + (float|string?)[][2]|error csv1op2_4 = csv:parseList(value1); + test:assertEquals(csv1op2_4, [ + [1f, ()], + [2f, ()] + ]); + + csv1op2_4 = csv:parseList(value7); + test:assertEquals(csv1op2_4, [ + [1f, ()], + [2f, ()] + ]); + + int?[][2]|float[][2]|error recCsv1op1_2 = csv:transform(value3); + test:assertEquals(recCsv1op1_2, [ + [1, ()], + [2, ()] + ]); + + float?[][2]|int?[][2]|error recCsv1op2_2 = csv:transform(value3); + test:assertEquals(recCsv1op2_2, [ + [1f, ()], + [2f, ()] + ]); + + (float|string?)[][2]|error recCsv1op2_4 = csv:transform(value3); + test:assertEquals(recCsv1op2_4, [ + [1f, ()], + [2f, ()] + ]); + + string?[][2]|float[][2]|error parseStrCsv1op1_2 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op1_2, [ + ["1", ()], + ["2", ()] + ]); + + float[][2]|string?[][2]|error parseStrCsv1op2_2 = csv:parseString(value4, {nilValue: ""}); + test:assertEquals(parseStrCsv1op2_2, [ + ["1", ()], + ["2", ()] + ]); + + (string|float?)[][2]|error parseStrCsv1op1_4 = csv:parseString(value5, {nilValue: "null"}); + test:assertEquals(parseStrCsv1op1_4, [ + ["1", ()], + ["2", ()] + ]); + + (float|string)?[][2]|error parseStrCsv1op2_4 = csv:parseString(value6, {nilValue: "N/A"}); + test:assertEquals(parseStrCsv1op2_4, [ + [1f, ()], + [2f, ()] + ]); +} diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_11/main.bal b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_11/main.bal index d25595b..b548c58 100644 --- a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_11/main.bal +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_11/main.bal @@ -7,10 +7,20 @@ type A record { }; public function main() returns error? { - record {}[] a = check csv:parseString(string `a,b`, {}); - record {}[] b = test({headerRows: 2, outputWithHeaders: false}); + record {}[] _ = check csv:parseString(string `a,b`, {}); + record {}[] _ = test1({headerRows: 2, outputWithHeaders: false}); + [int...][] _ = test2({headerRows: 2, outputWithHeaders: false}); + record {} _ = test3({headerRows: 2, outputWithHeaders: false}); } -function test(A a) returns record{}[] { +function test1(A a) returns record{}[] { return [{}]; } + +function test2(A a) returns [int...][] { + return []; +} + +function test3(A a) returns record{} { + return {}; +} diff --git a/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/CsvDataTypeValidator.java b/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/CsvDataTypeValidator.java index 3b666ea..9b42862 100644 --- a/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/CsvDataTypeValidator.java +++ b/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/CsvDataTypeValidator.java @@ -408,6 +408,9 @@ private boolean isSupportedArrayMemberType(SyntaxNodeAnalysisContext ctx, if (typeSymbol.typeKind() == TypeDescKind.TYPE_REFERENCE) { typeSymbol = ((TypeReferenceTypeSymbol) typeSymbol).typeDescriptor(); } + if (typeSymbol.typeKind() == TypeDescKind.INTERSECTION) { + typeSymbol = getRawType(typeSymbol); + } TypeDescKind kind = typeSymbol.typeKind(); if (kind == TypeDescKind.TYPE_REFERENCE) { kind = ((TypeReferenceTypeSymbol) typeSymbol).typeDescriptor().typeKind(); diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/FromString.java b/native/src/main/java/io/ballerina/lib/data/csvdata/FromString.java index bbea231..8b22db5 100644 --- a/native/src/main/java/io/ballerina/lib/data/csvdata/FromString.java +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/FromString.java @@ -109,9 +109,9 @@ public static Object fromStringWithType(BString string, Type expType, CsvConfig case TypeTags.BOOLEAN_TAG -> stringToBoolean(value); case TypeTags.NULL_TAG -> stringToNull(value, config); case TypeTags.FINITE_TYPE_TAG -> stringToFiniteType(value, (FiniteType) expType, config); - case TypeTags.UNION_TAG -> stringToUnion(string, (UnionType) expType, config); + case TypeTags.UNION_TAG -> stringToUnion(string, (UnionType) expType, config, false); case TypeTags.JSON_TAG, TypeTags.ANYDATA_TAG -> - stringToUnion(string, JSON_TYPE_WITH_BASIC_TYPES, config); + stringToUnion(string, JSON_TYPE_WITH_BASIC_TYPES, config, true); case TypeTags.TYPE_REFERENCED_TYPE_TAG -> fromStringWithType(string, ((ReferenceType) expType).getReferredType(), config); case TypeTags.INTERSECTION_TAG -> @@ -275,13 +275,20 @@ private static Object stringToNull(String value, CsvConfig config) throws Number return returnError(value, nullValue == null ? Constants.Values.BALLERINA_NULL : nullValue.toString()); } - private static Object stringToUnion(BString string, UnionType expType, CsvConfig config) + private static Object stringToUnion(BString string, UnionType expType, CsvConfig config, boolean isJsonOrAnydata) throws NumberFormatException { List memberTypes = new ArrayList<>(expType.getMemberTypes()); - memberTypes.sort(Comparator.comparingInt(t -> { - int index = TYPE_PRIORITY_ORDER.indexOf(TypeUtils.getReferredType(t).getTag()); - return index == -1 ? Integer.MAX_VALUE : index; - })); + if (isJsonOrAnydata) { + memberTypes.sort(Comparator.comparingInt(t -> { + int index = TYPE_PRIORITY_ORDER.indexOf(TypeUtils.getReferredType(t).getTag()); + return index == -1 ? Integer.MAX_VALUE : index; + })); + } else { + memberTypes.sort(Comparator.comparingInt(t -> { + int tag = TypeUtils.getReferredType(t).getTag(); + return tag == TypeTags.NULL_TAG ? Integer.MIN_VALUE : memberTypes.indexOf(t); + })); + } for (Type memberType : memberTypes) { try { Object result = fromStringWithType(string, memberType, config); diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvCreator.java b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvCreator.java index 06741f4..081155d 100644 --- a/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvCreator.java +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvCreator.java @@ -97,6 +97,7 @@ static void convertAndUpdateCurrentCsvNode(CsvParser.StateMachine sm, ArrayType arrayType = (ArrayType) currentCsvNodeType; if (arrayType.getState() == ArrayType.ArrayState.CLOSED && arrayType.getSize() - 1 < sm.columnIndex) { + sm.earlyReturn = true; return; } ((BArray) currentCsv).add(sm.columnIndex, convertedValue); diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvParser.java b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvParser.java index 7cb9684..0c22193 100644 --- a/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvParser.java +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvParser.java @@ -146,6 +146,7 @@ static class StateMachine { int arraySize = 0; boolean addHeadersForOutput = false; int currentCsvNodeLength = 0; + boolean earlyReturn = false; StateMachine() { reset(); @@ -182,6 +183,7 @@ public void reset() { arraySize = 0; addHeadersForOutput = false; currentCsvNodeLength = 0; + earlyReturn = false; } private static boolean isWhitespace(char ch, Object lineTerminator) { @@ -637,7 +639,7 @@ private static void handleCsvRow(StateMachine sm, boolean trim) { if (trim) { value = value.trim(); } - if (!(value.isBlank() && sm.currentCsvNodeLength == 0)) { + if (!(value.isBlank() && sm.currentCsvNodeLength == 0) && !sm.earlyReturn) { addRowValue(sm, trim); } if (!sm.isCurrentCsvNodeEmpty) { @@ -658,6 +660,7 @@ private static void updateLineAndColumnIndexesWithoutRowIndexes(StateMachine sm) sm.currentCsvNode = null; sm.isCurrentCsvNodeEmpty = true; sm.columnIndex = 0; + sm.earlyReturn = false; sm.clear(); } @@ -708,6 +711,9 @@ private static void addRowValue(StateMachine sm) { } private static void addRowValue(StateMachine sm, boolean trim) { + if (sm.earlyReturn) { + return; + } Field currentField = null; sm.isValueStart = false; Type exptype = sm.expectedArrayElementType; @@ -769,6 +775,7 @@ private static Type getExpectedRowTypeOfTuple(StateMachine sm, TupleType tupleTy } else { sm.charBuffIndex = 0; if (sm.config.allowDataProjection) { + sm.earlyReturn = true; return null; } throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_EXPECTED_TUPLE_SIZE, tupleTypes.size()); @@ -780,6 +787,7 @@ private static Type getExpectedRowTypeOfArray(StateMachine sm, ArrayType arrayTy if (arrayType.getSize() != -1 && arrayType.getSize() <= sm.columnIndex) { sm.charBuffIndex = 0; if (sm.config.allowDataProjection) { + sm.earlyReturn = true; return null; } throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_EXPECTED_ARRAY_SIZE, arrayType.getSize()); diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvTraversal.java b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvTraversal.java index 211ee64..46176dc 100644 --- a/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvTraversal.java +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvTraversal.java @@ -253,7 +253,7 @@ private void traverseCsvWithExpectedType(int sourceArraySize, break; case TypeTags.UNION_TAG: traverseCsvWithUnionExpectedType(sourceArraySize, csv, - (UnionType) expectedArrayElementType, type); + (UnionType) expectedArrayElementType, type, isIntersection); break; default: throw DiagnosticLog.error(DiagnosticErrorCode.SOURCE_CANNOT_CONVERT_INTO_EXP_TYPE, type); @@ -336,83 +336,20 @@ public void traverseCsvWithListAsExpectedType(long length, BArray csv, Type expe } public void traverseCsvWithUnionExpectedType(long length, BArray csv, - UnionType expectedArrayType, Type type) { - Object rowValue; - ArrayType arrayType = (ArrayType) rootCsvNode.getType(); - int rowNumber = 0; - - outerLoop: - for (int i = 0; i < length; i++) { - boolean isCompatible = false; - this.isFirstRowIsHeader = false; - if (arrayType.getState() == ArrayType.ArrayState.CLOSED && - arrayType.getSize() - 1 < this.arraySize) { - break; - } - - Object o = csv.get(i); - - for (Type memberType: expectedArrayType.getMemberTypes()) { - boolean isIntersection = false; - try { - memberType = TypeUtils.getReferredType(memberType); - if (memberType.getTag() == TypeTags.INTERSECTION_TAG) { - Optional mutableType = CsvUtils.getMutableType((IntersectionType) memberType); - if (mutableType.isPresent()) { - isIntersection = true; - memberType = mutableType.get(); - } - } - - if (CsvUtils.isExpectedTypeIsMap(memberType)) { - if (i < config.headersRows && i != config.headersRows - 1) { - continue outerLoop; - } - - if (i >= config.headersRows && ignoreRow(rowNumber + 1, config.skipLines)) { - rowNumber++; - continue outerLoop; - } - rowValue = initStatesForCsvRowWithMappingAsExpectedType(o, memberType); - } else if (CsvUtils.isExpectedTypeIsArray(memberType)) { - if (!addHeadersForOutput && config.outputWithHeaders - && (o instanceof BMap || (config.customHeaders != null - || i == config.headersRows - 1))) { - // Headers will add to the list only in the first iteration - insertHeaderValuesForTheCsvIfApplicable(o, memberType); - } - if (i < config.headersRows) { - continue outerLoop; - } + UnionType expectedArrayType, Type type, boolean isIntersection) { - if (ignoreRow(rowNumber + 1, config.skipLines)) { - rowNumber++; - continue outerLoop; - } - rowValue = initStatesForCsvRowWithListAsExpectedType(o, memberType); - } else { - continue; - } - - if (isIntersection) { - rowValue = CsvCreator.constructReadOnlyValue(rowValue); - } - - if (!this.isFirstRowIsHeader) { - rootCsvNode.add(this.arraySize, rowValue); - this.arraySize++; - } - isCompatible = true; - break; - } catch (Exception e) { - resetForUnionMemberTypes(); + for (Type memberType: expectedArrayType.getMemberTypes()) { + try { + memberType = TypeCreator.createArrayType(TypeUtils.getReferredType(memberType)); + if (CsvUtils.isExpectedTypeIsMap(memberType) || CsvUtils.isExpectedTypeIsArray(memberType)) { + traverseCsv(csv, config, memberType); + return; } + } catch (Exception ex) { + resetForUnionTypes(); } - if (!isCompatible) { - throw DiagnosticLog.error(DiagnosticErrorCode.SOURCE_CANNOT_CONVERT_INTO_EXP_TYPE, type); - } - rowNumber++; } + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_UNION_CONVERSION, type); } private static boolean ignoreRow(int index, Object skipLinesConfig) { @@ -492,7 +429,10 @@ private void constructCsvArrayFromMapping(BMap map, Type type, } Type memberType = getTheElementTypeFromList(type, index); if (memberType != null) { - insertValuesIntoList(v, memberType, index, currentCsvNode); + boolean isArrayActive = insertToListAndReturnFalseIfListEnds(v, memberType, index, currentCsvNode); + if (!isArrayActive) { + return; + } } index++; } @@ -506,7 +446,11 @@ private void constructCsvArrayFromNonMapping(BArray csvElement, Type type, int e } Type memberType = getTheElementTypeFromList(type, index); if (memberType != null) { - insertValuesIntoList(csvElement.get(i), memberType, index, currentCsvNode); + boolean isArrayActive = insertToListAndReturnFalseIfListEnds( + csvElement.get(i), memberType, index, currentCsvNode); + if (!isArrayActive) { + return; + } } index++; } @@ -591,7 +535,11 @@ private void insertHeaderValuesForTheCsvIfApplicable(Object obj, Type type) { for (int i = 0; i < this.headers.length; i++) { Type memberType = getTheElementTypeFromList(type, i); if (memberType != null) { - insertValuesIntoList(StringUtils.fromString(headers[i]), memberType, i, headersArray); + boolean isArrayActive = insertToListAndReturnFalseIfListEnds(StringUtils.fromString( + headers[i]), memberType, i, headersArray); + if (!isArrayActive) { + break; + } } } @@ -865,7 +813,8 @@ private void insertCurrentFieldMemberIntoMapping(Type type, Object recValue, BSt throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_TYPE_FOR_FIELD, recValue, key); } - public void insertValuesIntoList(Object arrayValue, Type type, int index, Object currentCsvNode) { + public boolean insertToListAndReturnFalseIfListEnds(Object arrayValue, + Type type, int index, Object currentCsvNode) { Object value = convertCsvValueIntoExpectedType(type, arrayValue, false); boolean isArrayType = type instanceof ArrayType; if (!(value instanceof CsvUtils.UnMappedValue)) { @@ -873,11 +822,11 @@ public void insertValuesIntoList(Object arrayValue, Type type, int index, Object ArrayType arrayType = (ArrayType) TypeUtils.getType(type); if (arrayType.getState() == ArrayType.ArrayState.CLOSED && arrayType.getSize() - 1 < index) { - return; + return false; } } ((BArray) currentCsvNode).add(index, value); - return; + return true; } throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_TYPE_FOR_ARRAY, arrayValue, index, type); } diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/utils/DiagnosticErrorCode.java b/native/src/main/java/io/ballerina/lib/data/csvdata/utils/DiagnosticErrorCode.java index 82df3a5..f29bced 100644 --- a/native/src/main/java/io/ballerina/lib/data/csvdata/utils/DiagnosticErrorCode.java +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/utils/DiagnosticErrorCode.java @@ -55,7 +55,8 @@ public enum DiagnosticErrorCode { HEADERS_WITH_VARYING_LENGTH_NOT_SUPPORTED("BDE_0027", "headers.with.varying.length.not.supported"), HEADER_VALUE_CANNOT_BE_EMPTY("BDE_0028", "header.value.cannot.be.empty"), - DUPLICATE_HEADER("BDE_0029", "duplicate.header"); + DUPLICATE_HEADER("BDE_0029", "duplicate.header"), + INVALID_UNION_CONVERSION("BDE_0029", "invalid.union.conversion"); String diagnosticId; String messageKey; diff --git a/native/src/main/resources/csv_error.properties b/native/src/main/resources/csv_error.properties index ff9a22e..6fa91e0 100644 --- a/native/src/main/resources/csv_error.properties +++ b/native/src/main/resources/csv_error.properties @@ -110,3 +110,6 @@ error.header.value.cannot.be.empty=\ error.duplicate.header=\ Duplicate header found: ''{0}'' + +error.invalid.union.conversion=\ + The CSV cannot be converted into any of the uniform union types in ''{0}'' From 145ca8affc3f87d77316b2417ddf6e86e3890e1e Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Wed, 14 Aug 2024 14:58:44 +0530 Subject: [PATCH 2/2] Add tests for slice operations --- .../tests/parse_string_to_array_test.bal | 2 +- .../tests/parse_type_compatibility_test.bal | 58 +++++++++++++++++++ .../user_config_with_parser_options_test.bal | 22 +++---- .../lib/data/csvdata/csv/CsvCreator.java | 2 +- .../lib/data/csvdata/csv/CsvParser.java | 27 ++++++--- .../lib/data/csvdata/csv/CsvTraversal.java | 2 +- .../src/main/resources/csv_error.properties | 2 +- 7 files changed, 93 insertions(+), 22 deletions(-) diff --git a/ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_array_test.bal b/ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_array_test.bal index e6b3565..e764c8e 100644 --- a/ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_array_test.bal +++ b/ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_array_test.bal @@ -340,7 +340,7 @@ function testArrayIndexes() { 5, 6, 7 7, 8, 9`; - record {}[2]|csv:Error rec = csv:parseString(csv); + record{}[2]|csv:Error rec = csv:parseString(csv); test:assertEquals(rec, [ {a: 1, b: 2}, {a: 3, b: 4} diff --git a/ballerina-tests/type-compatible-tests/tests/parse_type_compatibility_test.bal b/ballerina-tests/type-compatible-tests/tests/parse_type_compatibility_test.bal index 325109a..06b36a7 100644 --- a/ballerina-tests/type-compatible-tests/tests/parse_type_compatibility_test.bal +++ b/ballerina-tests/type-compatible-tests/tests/parse_type_compatibility_test.bal @@ -450,3 +450,61 @@ function testFromCsvWithIntersectionTypeCompatibility2() { ["3", "string3", true, "string3"] ]); } + +@test:Config +function testSliceOperation() { + string[][] v = [["1", "2"], ["3", "4"], ["a", "b"]]; + var v2 = [{a: 1, b: 2}, {a: 3, b: 4}, {a: "a", b: "b"}]; + string v3 = string `a,b + 1,2 + 3,4 + a,b`; + + int[2][]|error c = csv:parseList(v); + test:assertEquals(c, [[1, 2], [3, 4]]); + + record{|int...;|}[2]|error c2 = csv:parseList(v, {customHeaders: ["a", "b"]}); + test:assertEquals(c2, [{a: 1, b: 2}, {a: 3, b: 4}]); + + int[2][]|error c3 = csv:transform(v2, {headersOrder: ["a", "b"]}); + test:assertEquals(c3, [[1, 2], [3, 4]]); + + record{|int...;|}[2]|error c4 = csv:transform(v2); + test:assertEquals(c4, [{a: 1, b: 2}, {a: 3, b: 4}]); + + int[2][]|error c5 = csv:parseString(v3); + test:assertEquals(c5, [[1, 2], [3, 4]]); + + record{|int...;|}[2]|error c6 = csv:parseString(v3); + test:assertEquals(c6, [{a: 1, b: 2}, {a: 3, b: 4}]); +} + +@test:Config +function testSliceOperation2() { + string[][] v = [["c", "c", "c"], ["1", "2", "a"], ["c", "c", "c"], ["3", "4", "a"], ["a", "b", "a"]]; + var v2 = [{a: "c", b: "c", c: "c"}, {a: 1, b: 2, c: "c"}, {a: "c", b: "c", c: "c"}, {a: 3, b: 4, c: "c"}, {a: "a", b: "b", c: "c"}]; + string v3 = string `a,b, c + c,c,c + 1,2,c + c,c,c + 3,4,c + a,b,c`; + + int[2][2]|error c = csv:parseList(v, {skipLines: [1, 3]}); + test:assertEquals(c, [[1, 2], [3, 4]]); + + record{|int...;|}[2]|error c2 = csv:parseList(v, {customHeaders: ["a", "b", "c"], skipLines: [1, 3]}); + test:assertEquals(c2, [{a: 1, b: 2}, {a: 3, b: 4}]); + + int[2][2]|error c3 = csv:transform(v2, {headersOrder: ["a", "b", "c"], skipLines: [1, 3]}); + test:assertEquals(c3, [[1, 2], [3, 4]]); + + record{|int...;|}[2]|error c4 = csv:transform(v2, {skipLines: [1, 3]}); + test:assertEquals(c4, [{a: 1, b: 2}, {a: 3, b: 4}]); + + int[2][2]|error c5 = csv:parseString(v3, {skipLines: [1, 3]}); + test:assertEquals(c5, [[1, 2], [3, 4]]); + + record{|int...;|}[2]|error c6 = csv:parseString(v3, {skipLines: [1, 3]}); + test:assertEquals(c6, [{a: 1, b: 2}, {a: 3, b: 4}]); +} \ No newline at end of file diff --git a/ballerina-tests/user-config-tests/tests/user_config_with_parser_options_test.bal b/ballerina-tests/user-config-tests/tests/user_config_with_parser_options_test.bal index 91d84f2..b510840 100644 --- a/ballerina-tests/user-config-tests/tests/user_config_with_parser_options_test.bal +++ b/ballerina-tests/user-config-tests/tests/user_config_with_parser_options_test.bal @@ -354,7 +354,7 @@ function testCommentConfigOption() { cn = csv:parseString(csvValue10); test:assertTrue(cn is csv:Error); - test:assertEquals(( cn).message(), "Invalid length for the headers"); + test:assertEquals(( cn).message(), "Invalid number of headers"); cn = csv:parseString(csvValue9); test:assertTrue(cn is csv:Error); @@ -435,7 +435,7 @@ function testCommentConfigOption2() { cn = csv:parseString(csvValue7, {comment: "&", header: 2}); test:assertTrue(cn is csv:Error); - test:assertEquals((cn).message(), "Invalid length for the headers"); + test:assertEquals((cn).message(), "Invalid number of headers"); cn2 = csv:parseString(csvValue1, {comment: "&"}); test:assertTrue(cn2 is csv:Error); @@ -600,19 +600,19 @@ function testCustomHeaderOption() { record {}[]|csv:Error ct1br = csv:parseList([["a", "1", "true"], ["a", "1", "true"]], {customHeaders: ["a", "b"]}); test:assertTrue(ct1br is csv:Error); - test:assertEquals((ct1br).message(), "Invalid length for the headers"); + test:assertEquals((ct1br).message(), "Invalid number of headers"); record {}[]|csv:Error ct1br2 = csv:parseList([["a", "1", "true"], ["a", "1", "true"]], {customHeaders: ["a", "b", "c", "d"]}); test:assertTrue(ct1br2 is csv:Error); - test:assertEquals((ct1br2).message(), "Invalid length for the headers"); + test:assertEquals((ct1br2).message(), "Invalid number of headers"); record {}[]|csv:Error ct1br2_2 = csv:parseList([["a", "1", "true"], ["a", "1", "true"]], {customHeaders: ["a", "c", "b", "d"]}); test:assertTrue(ct1br2_2 is csv:Error); - test:assertEquals((ct1br2_2).message(), "Invalid length for the headers"); + test:assertEquals((ct1br2_2).message(), "Invalid number of headers"); record {}[]|csv:Error ct1br3 = csv:parseList([["a", "1", "true"], ["a", "1", "true"]], {customHeaders: []}); test:assertTrue(ct1br3 is csv:Error); - test:assertEquals((ct1br3).message(), "Invalid length for the headers"); + test:assertEquals((ct1br3).message(), "Invalid number of headers"); record {|string a; string b; string c;|}[]|csv:Error ct1br5 = csv:parseList([["a", "1", "true"], ["a", "1", "true"]], {customHeaders: ["a", "e", "b"]}); test:assertTrue(ct1br5 is csv:Error); @@ -686,11 +686,11 @@ function testCustomHeaderParserOption2() { record {}[]|csv:Error ct1br2 = csv:parseString(csvStringData4, {header: false, customHeadersIfHeadersAbsent: []}); test:assertTrue(ct1br2 is csv:Error); - test:assertEquals((ct1br2).message(), "Invalid length for the headers"); + test:assertEquals((ct1br2).message(), "Invalid number of headers"); record {int a; string b; boolean c; decimal d; float e; () f;}[]|csv:Error ct1br3 = csv:parseString(csvStringData4, {header: false, customHeadersIfHeadersAbsent: ["a", "b"]}); test:assertTrue(ct1br3 is csv:Error); - test:assertEquals((ct1br3).message(), "Invalid length for the headers"); + test:assertEquals((ct1br3).message(), "Invalid number of headers"); record {int a; string b; boolean c; decimal d; float e; () f;}[]|csv:Error ct1br4 = csv:parseString(csvStringData1, {header: 1, customHeadersIfHeadersAbsent: ["a", "b", "c", "d", "e", "f"]}); test:assertEquals(ct1br4, [ @@ -761,7 +761,7 @@ function testCustomHeaderParserOption2() { record {|boolean d1; string e1;|}[]|csv:Error ct1br11 = csv:parseString(csvStringData1, {header: false, customHeadersIfHeadersAbsent: ["f1", "e1"]}); test:assertTrue(ct1br11 is csv:Error); - test:assertEquals((ct1br11).message(), "Invalid length for the headers"); + test:assertEquals((ct1br11).message(), "Invalid number of headers"); record {|string d1; string e1;|}[]|csv:Error ct1br12 = csv:parseString(csvStringData4, {header: false, customHeadersIfHeadersAbsent: ["f1", "e1", "d1", "c1", "b1", "a1"]}); test:assertEquals(ct1br12, [ @@ -837,12 +837,12 @@ function testTextQuotesWithParserOptions() { record {}[]|csv:Error cn6 = csv:parseString(csvValue6, {}); test:assertTrue(cn6 is csv:Error); - test:assertEquals((cn6).message(), "Invalid length for the headers"); + test:assertEquals((cn6).message(), "Invalid number of headers"); cn6 = csv:parseString(string `a,b,c,d 1,1,1,1,1`, {}); test:assertTrue(cn6 is csv:Error); - test:assertEquals((cn6).message(), "Invalid length for the headers"); + test:assertEquals((cn6).message(), "Invalid number of headers"); } @test:Config diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvCreator.java b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvCreator.java index 081155d..05f3ecf 100644 --- a/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvCreator.java +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvCreator.java @@ -97,7 +97,7 @@ static void convertAndUpdateCurrentCsvNode(CsvParser.StateMachine sm, ArrayType arrayType = (ArrayType) currentCsvNodeType; if (arrayType.getState() == ArrayType.ArrayState.CLOSED && arrayType.getSize() - 1 < sm.columnIndex) { - sm.earlyReturn = true; + sm.isColumnMaxSizeReached = true; return; } ((BArray) currentCsv).add(sm.columnIndex, convertedValue); diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvParser.java b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvParser.java index 0c22193..239a024 100644 --- a/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvParser.java +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvParser.java @@ -146,7 +146,8 @@ static class StateMachine { int arraySize = 0; boolean addHeadersForOutput = false; int currentCsvNodeLength = 0; - boolean earlyReturn = false; + boolean isColumnMaxSizeReached = false; + boolean isRowMaxSizeReached = false; StateMachine() { reset(); @@ -183,7 +184,8 @@ public void reset() { arraySize = 0; addHeadersForOutput = false; currentCsvNodeLength = 0; - earlyReturn = false; + isColumnMaxSizeReached = false; + isRowMaxSizeReached = false; } private static boolean isWhitespace(char ch, Object lineTerminator) { @@ -544,6 +546,13 @@ public State transition(StateMachine sm, char[] buff, int i, int count) { for (; i < count; i++) { ch = buff[i]; sm.processLocation(ch); + if (sm.isRowMaxSizeReached) { + if (ch == EOF) { + state = ROW_END_STATE; + break; + } + continue; + } if (ch == Constants.LineTerminator.CR) { CsvUtils.setCarriageTokenPresent(true); continue; @@ -639,7 +648,8 @@ private static void handleCsvRow(StateMachine sm, boolean trim) { if (trim) { value = value.trim(); } - if (!(value.isBlank() && sm.currentCsvNodeLength == 0) && !sm.earlyReturn) { + if (!(value.isBlank() && sm.currentCsvNodeLength == 0) + && !sm.isColumnMaxSizeReached && !sm.isRowMaxSizeReached) { addRowValue(sm, trim); } if (!sm.isCurrentCsvNodeEmpty) { @@ -660,7 +670,7 @@ private static void updateLineAndColumnIndexesWithoutRowIndexes(StateMachine sm) sm.currentCsvNode = null; sm.isCurrentCsvNodeEmpty = true; sm.columnIndex = 0; - sm.earlyReturn = false; + sm.isColumnMaxSizeReached = false; sm.clear(); } @@ -704,6 +714,9 @@ private static void finalizeTheRow(StateMachine sm) { } sm.arraySize++; sm.currentCsvNodeLength = 0; + if (sm.arraySize == rootArraySize) { + sm.isRowMaxSizeReached = true; + } } private static void addRowValue(StateMachine sm) { @@ -711,7 +724,7 @@ private static void addRowValue(StateMachine sm) { } private static void addRowValue(StateMachine sm, boolean trim) { - if (sm.earlyReturn) { + if (sm.isColumnMaxSizeReached || sm.isRowMaxSizeReached) { return; } Field currentField = null; @@ -775,7 +788,7 @@ private static Type getExpectedRowTypeOfTuple(StateMachine sm, TupleType tupleTy } else { sm.charBuffIndex = 0; if (sm.config.allowDataProjection) { - sm.earlyReturn = true; + sm.isColumnMaxSizeReached = true; return null; } throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_EXPECTED_TUPLE_SIZE, tupleTypes.size()); @@ -787,7 +800,7 @@ private static Type getExpectedRowTypeOfArray(StateMachine sm, ArrayType arrayTy if (arrayType.getSize() != -1 && arrayType.getSize() <= sm.columnIndex) { sm.charBuffIndex = 0; if (sm.config.allowDataProjection) { - sm.earlyReturn = true; + sm.isColumnMaxSizeReached = true; return null; } throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_EXPECTED_ARRAY_SIZE, arrayType.getSize()); diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvTraversal.java b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvTraversal.java index 46176dc..e551ac5 100644 --- a/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvTraversal.java +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvTraversal.java @@ -475,7 +475,7 @@ private void constructCsvMapFromNonMapping(BArray csvElement, if (this.headers == null) { this.headers = CsvUtils.createHeadersForParseLists(csvElement, headers, config); if (!this.isFirstRowInserted && config.headersRows >= 1) { - // To skip the row at the position [config.headersRows - 1] from being added to the result. + // To skip the row at the position [config.headersRows - 1] from being aded to the result. this.isFirstRowIsHeader = true; this.isFirstRowInserted = true; return; diff --git a/native/src/main/resources/csv_error.properties b/native/src/main/resources/csv_error.properties index 6fa91e0..2afb2e2 100644 --- a/native/src/main/resources/csv_error.properties +++ b/native/src/main/resources/csv_error.properties @@ -82,7 +82,7 @@ error.inconsistent.header=\ Header ''{0}'' cannot be find in data rows error.invalid.custom.header.length=\ - Invalid length for the headers + Invalid number of headers error.invalid.header.names.length=\ Invalid number of headers