Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow empty strings to represent nil values using user configs #23

Merged
merged 1 commit into from
Aug 12, 2024
Merged

Allow empty strings to represent nil values using user configs #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_array_test.bal
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -437,14 +437,12 @@ function testParseStringArrayAsExpectedTypeWithOutputHeaders() {
]);

string[][]|csv:Error cv1baa_5 = csv:parseString(csvStringWithBooleanValues1, {outputWithHeaders: true, header: 2});
test:assertEquals(cv1baa_5, [
["true", "false", "true", "false"],
["true", "false", "true", "false"]
]);
test:assertTrue(cv1baa_5 is csv:Error);
test:assertEquals((<csv:Error>cv1baa_5).message(), "Duplicate header found: 'true'");

string[][]|csv:Error cv1baa_6 = csv:parseString(csvStringWithBooleanValues1, {outputWithHeaders: false, header: 2});
string[][]|csv:Error cv1baa_6 = csv:parseString(csvStringWithBooleanValues8, {outputWithHeaders: false, header: 2});
test:assertEquals(cv1baa_6, [
["true", "false", "true", "false"]
["true", "false", "true1", "false1"]
]);

[string, string, string, string, string][]|csv:Error cv2baa_7 = csv:parseString(csvStringWithBooleanValues2, {outputWithHeaders: true});
Expand Down
6 changes: 6 additions & 0 deletions ballerina-tests/parse-string-array-types-tests/tests/test_data_values.bal
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,9 @@ string csvStringWithBooleanValues6 = string `b2,b3
string csvStringWithBooleanValues7 = string `b1,b2,b3,b4
${b1},${b2},(),${b4}
`;

string csvStringWithBooleanValues8 = string `b1,b2,b3,b4
true,false,true1,false1
true,false, true1,false1
true,false,true1,false1
`;
4 changes: 4 additions & 0 deletions ballerina-tests/type-compatible-tests/tests/csv_content_2.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
a, b, c d, e
"Hello World1", \"Hello World2\", Hello World3, 21
"Hello World1", \"Hello World2\", Hello World3, 22
"Hello World1", \"Hello World2\", Hello World3, 23
58 changes: 58 additions & 0 deletions ballerina-tests/type-compatible-tests/tests/nill_type_test.bal
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import ballerina/data.csv;
import ballerina/test;

type Book1 record {
()|string name;
()|string author;
()|string year;
};

string csvString1 = string `name,author,year
,b,c
a,,c
a,b,`;

@test:Config{}
function testEmptyStringWithNilConfig() {
Book1[]|error books1 = csv:parseString(csvString1, {nilValue: ""});
test:assertEquals(books1, [
{name: null, author: "b", year: "c"},
{name: "a", author: null, year: "c"},
{name: "a", author: "b", year: null}
]);

Book1[]|error books2 = csv:parseString(csvString1);
test:assertEquals(books2, [
{name: "", author: "b", year: "c"},
{name: "a", author: "", year: "c"},
{name: "a", author: "b", year: ""}
]);

(boolean|()|string|int)[][]|error arrbooks1 = csv:parseString(csvString1, {nilValue: ""});
test:assertEquals(arrbooks1, [
[null, "b", "c"],
["a", null, "c"],
["a", "b", null]
]);

(boolean|()|string|int)[][2]|error arrbooks2 = csv:parseString(csvString1, {nilValue: ""});
test:assertEquals(arrbooks2, [
[null, "b"],
["a", null],
["a", "b"]
]);

(boolean|()|string|int)[][]|error arrbooks3 = csv:parseString(csvString1);
test:assertEquals(arrbooks3, [
["", "b", "c"],
["a", "", "c"],
["a", "b", ""]
]);

(boolean|()|string|int)[][2]|error arrbooks4 = csv:parseString(csvString1);
test:assertEquals(arrbooks4, [
["", "b"],
["a", ""],
["a", "b"]
]);
}
33 changes: 18 additions & 15 deletions ballerina-tests/type-compatible-tests/tests/parse_string_compatibality_test.bal
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import ballerina/io;
import ballerina/test;

const string filepath = "tests/csv_content.txt";
const string filepath2 = "tests/csv_content_2.txt";
const string errorFilepath = "tests/csv_error_content.txt";

@test:Config
Expand Down Expand Up @@ -215,6 +216,7 @@ function testSpaceBetweendData() {
@test:Config
function testParseBytes() returns error? {
byte[] csvBytes = check io:fileReadBytes(filepath);
byte[] csvBytes2 = check io:fileReadBytes(filepath2);

record{}[]|csv:Error rec = csv:parseBytes(csvBytes, {});
test:assertEquals(rec, [
Expand Down Expand Up @@ -245,17 +247,17 @@ function testParseBytes() returns error? {
["Hello World", "\"Hello World\"", "Hello World", "2"]
]);

rec2 = csv:parseBytes(csvBytes, {outputWithHeaders: true, header: 1});
rec2 = csv:parseBytes(csvBytes2, {outputWithHeaders: true, header: 1});
test:assertEquals(rec2, [
["Hello World", "\"Hello World\"", "Hello World", "2"],
["Hello World", "\"Hello World\"", "Hello World", "2"],
["Hello World", "\"Hello World\"", "Hello World", "2"]
["Hello World1", "\"Hello World2\"", "Hello World3", "21"],
["Hello World1", "\"Hello World2\"", "Hello World3", "22"],
["Hello World1", "\"Hello World2\"", "Hello World3", "23"]
]);

rec2 = csv:parseBytes(csvBytes, { header: 1});
rec2 = csv:parseBytes(csvBytes2, { header: 1});
test:assertEquals(rec2, [
["Hello World", "\"Hello World\"", "Hello World", "2"],
["Hello World", "\"Hello World\"", "Hello World", "2"]
["Hello World1", "\"Hello World2\"", "Hello World3", "22"],
["Hello World1", "\"Hello World2\"", "Hello World3", "23"]
]);

int[][]|csv:Error rec3 = csv:parseBytes(csvBytes, {});
Expand All @@ -270,6 +272,8 @@ function testParseBytes() returns error? {
@test:Config
function testParseStream() returns error? {
stream<byte[], io:Error?> csvByteStream = check io:fileReadBlocksAsStream(filepath);
stream<byte[], io:Error?> csvByteStream2 = check io:fileReadBlocksAsStream(filepath2);

record{}[]|csv:Error rec = csv:parseStream(csvByteStream, {});
test:assertEquals(rec, [
{"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2},
Expand All @@ -293,12 +297,11 @@ function testParseStream() returns error? {
["Hello World", "\"Hello World\"", "Hello World", "2"]
]);

csvByteStream = check io:fileReadBlocksAsStream(filepath);
rec2 = csv:parseStream(csvByteStream, {header: 1, outputWithHeaders: true});
rec2 = csv:parseStream(csvByteStream2, {header: 1, outputWithHeaders: true});
test:assertEquals(rec2, [
["Hello World", "\"Hello World\"", "Hello World", "2"],
["Hello World", "\"Hello World\"", "Hello World", "2"],
["Hello World", "\"Hello World\"", "Hello World", "2"]
["Hello World1", "\"Hello World2\"", "Hello World3", "21"],
["Hello World1", "\"Hello World2\"", "Hello World3", "22"],
["Hello World1", "\"Hello World2\"", "Hello World3", "23"]
]);

csvByteStream = check io:fileReadBlocksAsStream(filepath);
Expand All @@ -310,11 +313,11 @@ function testParseStream() returns error? {
["Hello World", "\"Hello World\"", "Hello World", "2"]
]);

csvByteStream = check io:fileReadBlocksAsStream(filepath);
csvByteStream = check io:fileReadBlocksAsStream(filepath2);
rec2 = csv:parseStream(csvByteStream, {header: 1});
test:assertEquals(rec2, [
["Hello World", "\"Hello World\"", "Hello World", "2"],
["Hello World", "\"Hello World\"", "Hello World", "2"]
["Hello World1", "\"Hello World2\"", "Hello World3", "22"],
["Hello World1", "\"Hello World2\"", "Hello World3", "23"]
]);

csvByteStream = check io:fileReadBlocksAsStream(filepath);
Expand Down
25 changes: 25 additions & 0 deletions ballerina-tests/user-config-tests/tests/test_data_values.bal
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,3 +121,28 @@ string csvStringData9 = string `
4@ string@ true@ 2.234@ ()@-3.21

5@ string@ true@ 2.234@ -3.21@ null`;

string csvStringData10 = string `
hello, hello, (), 12, true, 12.34
// comment

a, b, c, d, e, f


1, string1, true, 2.234, 2.234, ()
2, string2, false, 0, 0, null
3, string3, false, 1.23, 1.23, ()
4, string4, true, -6.51, -6.52, ()
5, string5, true, 3, 31, ()`;

string csvStringData11 = string `
a, b, c, d, e, f



1, string1, true, 2.234, 2.234, ()
2, string2, false, 0, 0, null
3, string3, false, 1.23, 1.23, ()

4, string4, true, -6.51, -6.52, ()
5, string5, true, 3, 3, ()`;
50 changes: 42 additions & 8 deletions ballerina-tests/user-config-tests/tests/user_config_with_parser_options_test.bal
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,9 @@ function testFromCsvStringWithParserOptions() {
{a: 5, b: "string5", c: true, d: 3, e: 3, f: "()"}
]);

record {}[]|csv:Error csv3op3_4 = csv:parseString(csvStringData3, {header: 9, skipLines: "2-10"});
record {}[]|csv:Error csv3op3_4 = csv:parseString(csvStringData11, {header: 9, skipLines: "2-10"});
test:assertEquals(csv3op3_4, [
{'4: 5, string4: "string5", "true": true, "-6.51": 3, "()": null}
{'4: 5, string4: "string5", "true": true, "-6.51": 3, "-6.52": 3, "()": null}
]);
}

Expand Down Expand Up @@ -203,15 +203,19 @@ function testHeaderOption() {
{a: 5, b: "string5", c: true, d: 3, e: 3, f: ()}
]);

record {}[]|csv:Error csv2cop2 = csv:parseString(csvStringData2, {header: 100});
record {}[]|csv:Error csv2cop2 = csv:parseString(csvStringData10, {header: 100});
test:assertTrue(csv2cop2 is csv:Error);
test:assertEquals((<error>csv2cop2).message(), "The provided header row is empty");

record {}[]|csv:Error csv2cop3 = csv:parseString(csvStringData2, {header: 11});
record {}[]|csv:Error csv2cop3 = csv:parseString(csvStringData10, {header: 11});
test:assertEquals(csv2cop3, []);

record {}[]|csv:Error csv2cop4 = csv:parseString(csvStringData2, {header: 10});
test:assertEquals(csv2cop4, [{'4: 5, string4: "string5", "true": true, "-6.51": 3, "()": ()}]);
record {}[]|csv:Error csv2cop3_2 = csv:parseString(csvStringData10, {header: 9});
test:assertTrue(csv2cop3_2 is csv:Error);
test:assertEquals((<error>csv2cop3_2).message(), "Duplicate header found: '1.23'");

record {}[]|csv:Error csv2cop4 = csv:parseString(csvStringData10, {header: 10});
test:assertEquals(csv2cop4, [{'4: 5, string4: "string5", "true": true, "-6.51": 3, "-6.52": 31, "()": ()}]);

record {}[]|csv:Error csv1cop5 = csv:parseString(csvStringData1, {});
test:assertTrue(csv1cop5 is csv:Error);
Expand Down Expand Up @@ -313,6 +317,9 @@ function testCommentConfigOption() {
string csvValue9 = string `a,# b
1 ,#2 # comment
# comment`;
string csvValue10 = string `a# b
1 ,#2 # comment
# comment`;

record {int a;}[]|csv:Error cn;

Expand Down Expand Up @@ -345,8 +352,13 @@ function testCommentConfigOption() {
// TODO:Fix the error message
// test:assertEquals((<error> cn).message(), common:generateErrorMessageForInvalidCast("1, 2", "int"));

cn = csv:parseString(csvValue10);
test:assertTrue(cn is csv:Error);
test:assertEquals((<error> cn).message(), "Invalid length for the headers");

cn = csv:parseString(csvValue9);
test:assertEquals(cn, [{a: 1}]);
test:assertTrue(cn is csv:Error);
test:assertEquals((<error> cn).message(), "The provided header row is empty");
}

@test:Config {dependsOn: [testCSVLocale]}
Expand Down Expand Up @@ -375,6 +387,20 @@ function testCommentConfigOption2() {

& comment`;

string csvValue7 = string `

a& b
1 ,&2 & comment

& comment`;

string csvValue8 = string `

a,e& b
1 ,&2 & comment

& comment`;

record {int a; int b;}[]|csv:Error cn;
record {int c;}[]|csv:Error cn2;

Expand All @@ -401,8 +427,16 @@ function testCommentConfigOption2() {

cn = csv:parseString(csvValue6, {comment: "&", header: 2});
test:assertTrue(cn is csv:Error);
test:assertEquals((<error>cn).message(), "The provided header row is empty");

cn = csv:parseString(csvValue8, {comment: "&", header: 2});
test:assertTrue(cn is csv:Error);
test:assertEquals((<error>cn).message(), common:generateErrorMessageForMissingRequiredField("b"));

cn = csv:parseString(csvValue7, {comment: "&", header: 2});
test:assertTrue(cn is csv:Error);
test:assertEquals((<error>cn).message(), "Invalid length for the headers");

cn2 = csv:parseString(csvValue1, {comment: "&"});
test:assertTrue(cn2 is csv:Error);
test:assertEquals((<error>cn2).message(), common:generateErrorMessageForMissingRequiredField("c"));
Expand All @@ -426,7 +460,7 @@ function testCommentConfigOption2() {

cn2 = csv:parseString(csvValue6, {header: 2, comment: "&"});
test:assertTrue(cn2 is csv:Error);
test:assertEquals((<error>cn2).message(), common:generateErrorMessageForMissingRequiredField("c"));
test:assertEquals((<error>cn2).message(), "The provided header row is empty");
}

@test:Config {dependsOn: [testCSVLocale]}
Expand Down
9 changes: 6 additions & 3 deletions native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvCreator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,9 @@ static Object initRowValue(Type expectedType) {
};
}

static void convertAndUpdateCurrentJsonNode(CsvParser.StateMachine sm,
String value, Type type, CsvConfig config, Type exptype,
Field currentField) {
static void convertAndUpdateCurrentCsvNode(CsvParser.StateMachine sm,
String value, Type type, CsvConfig config, Type exptype,
Field currentField) {
Object currentCsv = sm.currentCsvNode;
Object nilValue = config.nilValue;
if (sm.config.nilAsOptionalField && !type.isNilable()
Expand All @@ -91,6 +91,7 @@ static void convertAndUpdateCurrentJsonNode(CsvParser.StateMachine sm,
case TypeTags.RECORD_TYPE_TAG:
((BMap<BString, Object>) currentCsv).put(StringUtils.fromString(getHeaderValueForColumnIndex(sm)),
convertedValue);
sm.currentCsvNodeLength++;
return;
case TypeTags.ARRAY_TAG:
ArrayType arrayType = (ArrayType) currentCsvNodeType;
Expand All @@ -99,9 +100,11 @@ static void convertAndUpdateCurrentJsonNode(CsvParser.StateMachine sm,
return;
}
((BArray) currentCsv).add(sm.columnIndex, convertedValue);
sm.currentCsvNodeLength++;
return;
case TypeTags.TUPLE_TAG:
((BArray) currentCsv).add(sm.columnIndex, convertedValue);
sm.currentCsvNodeLength++;
return;
default:
}
Expand Down
Loading
Loading