From c6b605730a8eef2fd86bfdd3fbb791da9a40ca25 Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Tue, 24 Sep 2024 12:27:14 +0530 Subject: [PATCH 01/23] Add bbe for convert csv string into records --- .../csv_string_to_record_array.bal | 29 +++++++++++++++++++ .../csv_string_to_record_array.md | 7 +++++ .../csv_string_to_record_array.metatags | 2 ++ .../csv_string_to_record_array.out | 3 ++ examples/index.json | 14 +++++++++ 5 files changed, 55 insertions(+) create mode 100644 examples/csv-string-to-record-array/csv_string_to_record_array.bal create mode 100644 examples/csv-string-to-record-array/csv_string_to_record_array.md create mode 100644 examples/csv-string-to-record-array/csv_string_to_record_array.metatags create mode 100644 examples/csv-string-to-record-array/csv_string_to_record_array.out diff --git a/examples/csv-string-to-record-array/csv_string_to_record_array.bal b/examples/csv-string-to-record-array/csv_string_to_record_array.bal new file mode 100644 index 0000000000..55132dfdb1 --- /dev/null +++ b/examples/csv-string-to-record-array/csv_string_to_record_array.bal @@ -0,0 +1,29 @@ +import ballerina/data.csv; +import ballerina/io; + +type Book record { + string name; + string author; + decimal price; + string publishedDate; +}; + +type BriefBookInfo record {| + string name; + string author; +|}; + +public function main() { + string csvString = string `name,author,price,publishedDate + Effective Java,Joshua Bloch,45.99,2018-01-01 + Clean Code,Robert C. Martin,37.50,2008-08-01`; + + // Convert the CSV string to a record array + Book[]|csv:Error bookRecords = csv:parseString(csvString); + io:println(bookRecords); + + // Convert the CSV string to a record array with data projection. + // In here only the name and author fields are selected. + BriefBookInfo[]|csv:Error briefBookRecords = csv:parseString(csvString); + io:println(briefBookRecords); +} diff --git a/examples/csv-string-to-record-array/csv_string_to_record_array.md b/examples/csv-string-to-record-array/csv_string_to_record_array.md new file mode 100644 index 0000000000..80b1332125 --- /dev/null +++ b/examples/csv-string-to-record-array/csv_string_to_record_array.md @@ -0,0 +1,7 @@ +# Convert CSV string into Ballerina record array + +The ballerina `data.csv` library offers a range of functions for converting CSV strings into Ballerina record arrays, enabling developers to parse CSV string data into structured records while allowing for the selection of specific fields within the expected record array. + +::: code csv_string_to_record_array.bal ::: + +::: out csv_string_to_record_array.out ::: diff --git a/examples/csv-string-to-record-array/csv_string_to_record_array.metatags b/examples/csv-string-to-record-array/csv_string_to_record_array.metatags new file mode 100644 index 0000000000..d9264c6965 --- /dev/null +++ b/examples/csv-string-to-record-array/csv_string_to_record_array.metatags @@ -0,0 +1,2 @@ +description: This BBE demonstrates conversion of CSV string into a Ballerina record array with and without data projection. +keywords: ballerina, ballerina by example, bbe, csv, csv string, record, record array, parseString, csv data module, data.csv, data projection diff --git a/examples/csv-string-to-record-array/csv_string_to_record_array.out b/examples/csv-string-to-record-array/csv_string_to_record_array.out new file mode 100644 index 0000000000..1422cf803f --- /dev/null +++ b/examples/csv-string-to-record-array/csv_string_to_record_array.out @@ -0,0 +1,3 @@ +$ bal run csv_string_to_record_array.bal +[{"name":"Effective Java","author":"Joshua Bloch","price":45.99,"publishedDate":"2018-01-01"},{"name":"Clean Code","author":"Robert C. Martin","price":37.5,"publishedDate":"2008-08-01"}] +[{"name":"Effective Java","author":"Joshua Bloch"},{"name":"Clean Code","author":"Robert C. Martin"}] diff --git a/examples/index.json b/examples/index.json index 6a45c8a9f1..58254f438b 100644 --- a/examples/index.json +++ b/examples/index.json @@ -4700,6 +4700,20 @@ } ] }, + { + "title": "CSV data", + "column": 3, + "category": "Common libraries", + "samples": [ + { + "name": "Convert CSV string into a record array", + "url": "csv-string-to-record-array", + "verifyBuild": true, + "verifyOutput": true, + "isLearnByExample": true + } + ] + }, { "title": "Constraint", "column": 3, From 28a28cff94aeaa48162aa2c01503c93eef8d91cf Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Tue, 24 Sep 2024 13:47:14 +0530 Subject: [PATCH 02/23] Update csv string to record bbe --- .../csv-string-to-record-array/csv_string_to_record_array.md | 4 ++-- .../csv_string_to_record_array.metatags | 2 +- examples/index.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/csv-string-to-record-array/csv_string_to_record_array.md b/examples/csv-string-to-record-array/csv_string_to_record_array.md index 80b1332125..09cc4b5dcf 100644 --- a/examples/csv-string-to-record-array/csv_string_to_record_array.md +++ b/examples/csv-string-to-record-array/csv_string_to_record_array.md @@ -1,6 +1,6 @@ -# Convert CSV string into Ballerina record array +# Convert CSV string to Ballerina record array -The ballerina `data.csv` library offers a range of functions for converting CSV strings into Ballerina record arrays, enabling developers to parse CSV string data into structured records while allowing for the selection of specific fields within the expected record array. +The ballerina `data.csv` library offers a range of functions for converting CSV strings to Ballerina record arrays, enabling developers to parse CSV string data to structured records while allowing for the selection of specific fields within the expected record array. ::: code csv_string_to_record_array.bal ::: diff --git a/examples/csv-string-to-record-array/csv_string_to_record_array.metatags b/examples/csv-string-to-record-array/csv_string_to_record_array.metatags index d9264c6965..6d2d4f10bb 100644 --- a/examples/csv-string-to-record-array/csv_string_to_record_array.metatags +++ b/examples/csv-string-to-record-array/csv_string_to_record_array.metatags @@ -1,2 +1,2 @@ -description: This BBE demonstrates conversion of CSV string into a Ballerina record array with and without data projection. +description: This BBE demonstrates conversion of CSV string to Ballerina record array with and without data projection. keywords: ballerina, ballerina by example, bbe, csv, csv string, record, record array, parseString, csv data module, data.csv, data projection diff --git a/examples/index.json b/examples/index.json index 58254f438b..e4e0535dbe 100644 --- a/examples/index.json +++ b/examples/index.json @@ -4706,7 +4706,7 @@ "category": "Common libraries", "samples": [ { - "name": "Convert CSV string into a record array", + "name": "Convert CSV string to Ballerina record array", "url": "csv-string-to-record-array", "verifyBuild": true, "verifyOutput": true, From 378d595c65f6d6faf8f15ac6cf6ee805c9af2b00 Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Tue, 24 Sep 2024 14:21:00 +0530 Subject: [PATCH 03/23] Add bbe for converting csv string to anydata array --- .../csv_string_to_anydata_array.bal | 21 +++++++++++++++++++ .../csv_string_to_anydata_array.md | 7 +++++++ .../csv_string_to_anydata_array.metatags | 2 ++ .../csv_string_to_anydata_array.out | 4 ++++ examples/index.json | 14 +++++++++++++ 5 files changed, 48 insertions(+) create mode 100644 examples/csv-string-to-anydata-array/csv_string_to_anydata_array.bal create mode 100644 examples/csv-string-to-anydata-array/csv_string_to_anydata_array.md create mode 100644 examples/csv-string-to-anydata-array/csv_string_to_anydata_array.metatags create mode 100644 examples/csv-string-to-anydata-array/csv_string_to_anydata_array.out diff --git a/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.bal b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.bal new file mode 100644 index 0000000000..ea86dce3c3 --- /dev/null +++ b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.bal @@ -0,0 +1,21 @@ +import ballerina/data.csv; +import ballerina/io; + +public function main() { + string csvString = string `name,author,price,publishedDate + Effective Java,Joshua Bloch,45.99,2018-01-01 + Clean Code,Robert C. Martin,37.50,2008-08-01`; + + // Parse the CSV string to a array of string arrays. + string[][]|csv:Error bookArray = csv:parseString(csvString); + io:println(bookArray); + + // Parse the CSV string to a array of anydata arrays. + anydata[][]|csv:Error bookArray2 = csv:parseString(csvString); + io:println(bookArray2); + + // Parse the CSV string to a array of string arrays with data projection. + // Here only the first two headers are extracted to the resulting array. + string[][2]|csv:Error bookArray3 = csv:parseString(csvString); + io:println(bookArray3); +} diff --git a/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.md b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.md new file mode 100644 index 0000000000..b88f050834 --- /dev/null +++ b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.md @@ -0,0 +1,7 @@ +# Convert CSV string to array of anydata arrays + +The Ballerina `data.csv` library offers a variety of functions for converting CSV strings to arrays of `anydata[]`, enabling developers to effectively parse CSV string data to flexible data structures. Additionally, it provides the option to select specific fields for inclusion in the resulting anydata arrays, ensuring that developers can tailor the output to meet their application's needs + +::: code csv_string_to_anydata_array.bal ::: + +::: out csv_string_to_anydata_array.out ::: diff --git a/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.metatags b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.metatags new file mode 100644 index 0000000000..82651de3b9 --- /dev/null +++ b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.metatags @@ -0,0 +1,2 @@ +description: This BBE demonstrates conversion of CSV string to a Ballerina array of anydata arrays with and without data projection. +keywords: ballerina, ballerina by example, bbe, csv, csv string, anydata, anydata array, parseString, csv data module, data.csv, data projection diff --git a/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.out b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.out new file mode 100644 index 0000000000..0be6074850 --- /dev/null +++ b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.out @@ -0,0 +1,4 @@ +$ bal run csv_string_to_anydata_array.bal +[["Effective Java","Joshua Bloch","45.99","2018-01-01"],["Clean Code","Robert C. Martin","37.50","2008-08-01"]] +[["Effective Java","Joshua Bloch",45.99,"2018-01-01"],["Clean Code","Robert C. Martin",37.5,"2008-08-01"]] +[["Effective Java","Joshua Bloch"],["Clean Code","Robert C. Martin"]] diff --git a/examples/index.json b/examples/index.json index 6a45c8a9f1..7334f700b1 100644 --- a/examples/index.json +++ b/examples/index.json @@ -4700,6 +4700,20 @@ } ] }, + { + "title": "CSV data", + "column": 3, + "category": "Common libraries", + "samples": [ + { + "name": "Convert CSV string to array of anydata arrays", + "url": "csv-string-to-anydata-array", + "verifyBuild": true, + "verifyOutput": true, + "isLearnByExample": true + } + ] + }, { "title": "Constraint", "column": 3, From 0c40eaf5360439f37826c10b75a69a6beb3b2c19 Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Tue, 24 Sep 2024 14:29:08 +0530 Subject: [PATCH 04/23] Update csv string to array of record bbe --- .../csv_string_to_record_array.bal | 11 ++++++----- .../csv_string_to_record_array.md | 4 ++-- .../csv_string_to_record_array.metatags | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/examples/csv-string-to-record-array/csv_string_to_record_array.bal b/examples/csv-string-to-record-array/csv_string_to_record_array.bal index 55132dfdb1..06bd316dac 100644 --- a/examples/csv-string-to-record-array/csv_string_to_record_array.bal +++ b/examples/csv-string-to-record-array/csv_string_to_record_array.bal @@ -8,7 +8,7 @@ type Book record { string publishedDate; }; -type BriefBookInfo record {| +type BookMinimal record {| string name; string author; |}; @@ -18,12 +18,13 @@ public function main() { Effective Java,Joshua Bloch,45.99,2018-01-01 Clean Code,Robert C. Martin,37.50,2008-08-01`; - // Convert the CSV string to a record array + // Parse CSV string to a array of records. Book[]|csv:Error bookRecords = csv:parseString(csvString); io:println(bookRecords); - // Convert the CSV string to a record array with data projection. - // In here only the name and author fields are selected. - BriefBookInfo[]|csv:Error briefBookRecords = csv:parseString(csvString); + // Parse CSV string to a array of records with data projection. + // Here only the fields specified in the target record type (`name` and `author` fields) + // are included in the constructed value. + BookMinimal[]|csv:Error briefBookRecords = csv:parseString(csvString); io:println(briefBookRecords); } diff --git a/examples/csv-string-to-record-array/csv_string_to_record_array.md b/examples/csv-string-to-record-array/csv_string_to_record_array.md index 09cc4b5dcf..d924d30b31 100644 --- a/examples/csv-string-to-record-array/csv_string_to_record_array.md +++ b/examples/csv-string-to-record-array/csv_string_to_record_array.md @@ -1,6 +1,6 @@ -# Convert CSV string to Ballerina record array +# Convert CSV string to array of records -The ballerina `data.csv` library offers a range of functions for converting CSV strings to Ballerina record arrays, enabling developers to parse CSV string data to structured records while allowing for the selection of specific fields within the expected record array. +The ballerina `data.csv` library offers a range of functions for converting CSV strings to array of records, enabling developers to parse CSV string data to structured records while allowing for the selection of specific fields within the expected record array. ::: code csv_string_to_record_array.bal ::: diff --git a/examples/csv-string-to-record-array/csv_string_to_record_array.metatags b/examples/csv-string-to-record-array/csv_string_to_record_array.metatags index 6d2d4f10bb..65cb588fe3 100644 --- a/examples/csv-string-to-record-array/csv_string_to_record_array.metatags +++ b/examples/csv-string-to-record-array/csv_string_to_record_array.metatags @@ -1,2 +1,2 @@ -description: This BBE demonstrates conversion of CSV string to Ballerina record array with and without data projection. +description: This BBE demonstrates conversion of CSV string to array of records with and without data projection. keywords: ballerina, ballerina by example, bbe, csv, csv string, record, record array, parseString, csv data module, data.csv, data projection From 8ffd81ac2fc93c84754071350af4c4793b176d0e Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Tue, 24 Sep 2024 14:33:35 +0530 Subject: [PATCH 05/23] Update the csv string to anydata[][] bbe --- .../csv-string-to-anydata-array/csv_string_to_anydata_array.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.md b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.md index b88f050834..b2795de5e9 100644 --- a/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.md +++ b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.md @@ -1,6 +1,6 @@ # Convert CSV string to array of anydata arrays -The Ballerina `data.csv` library offers a variety of functions for converting CSV strings to arrays of `anydata[]`, enabling developers to effectively parse CSV string data to flexible data structures. Additionally, it provides the option to select specific fields for inclusion in the resulting anydata arrays, ensuring that developers can tailor the output to meet their application's needs +The Ballerina `data.csv` library offers a variety of functions for converting CSV strings to sub types of `anydata[][]`, enabling developers to effectively parse CSV string data to flexible data structures. Additionally, it provides the option to select specific fields for inclusion in the resulting array of anydata arrays, ensuring that developers can tailor the output to meet their application's needs ::: code csv_string_to_anydata_array.bal ::: From 4e0e84c9f4f93944debc95ce7fd5cb464f095bbb Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Tue, 24 Sep 2024 15:30:57 +0530 Subject: [PATCH 06/23] Add bbe for handling csv byte streams --- .../csv_streams_to_record_array.bal | 38 +++++++++++++++++++ .../csv_streams_to_record_array.md | 7 ++++ .../csv_streams_to_record_array.metatags | 2 + .../csv_streams_to_record_array.out | 3 ++ examples/index.json | 14 +++++++ 5 files changed, 64 insertions(+) create mode 100644 examples/csv-streams-to-record-arrays/csv_streams_to_record_array.bal create mode 100644 examples/csv-streams-to-record-arrays/csv_streams_to_record_array.md create mode 100644 examples/csv-streams-to-record-arrays/csv_streams_to_record_array.metatags create mode 100644 examples/csv-streams-to-record-arrays/csv_streams_to_record_array.out diff --git a/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.bal b/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.bal new file mode 100644 index 0000000000..325ce0177d --- /dev/null +++ b/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.bal @@ -0,0 +1,38 @@ +import ballerina/data.csv; +import ballerina/io; + +type Book record { + string name; + string author; + decimal price; + string publishedDate; +}; + +type BookMinimal record {| + string name; + string author; +|}; + +const csvFilePath = "./files/csv_file.csv"; + +public function main() returns io:Error? { + // Write a CSV file. + check io:fileWriteCsv(csvFilePath, [["name", "author", "price", "publishedDate"], + ["Effective Java", "Joshua Bloch", "45.99", "2018-01-01"], + ["Clean Code", "Robert C. Martin", "37.5", "2008-08-01"]]); + + // Read the CSV file as a `byte stream`. + stream csvStream = check io:fileReadBlocksAsStream(csvFilePath); + + // Parse CSV `byte stream` to an array of records. + Book[]|csv:Error bookStream = csv:parseStream(csvStream); + io:println(bookStream); + + stream csvStream2 = check io:fileReadBlocksAsStream(csvFilePath); + + // Parse CSV `byte stream` to an array of records with data projection. + // Here only the fields specified in the target record type (`name` and `author` fields) + // are included in the constructed value. + BookMinimal[]|csv:Error briefBookStream = csv:parseStream(csvStream2); + io:println(briefBookStream); +} diff --git a/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.md b/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.md new file mode 100644 index 0000000000..2dd7b9fb6b --- /dev/null +++ b/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.md @@ -0,0 +1,7 @@ +# Parse CSV byte stream to Ballerina record array + +The Ballerina `data.csv` library allows parsing CSV byte streams into arrays of Ballerina records, making it easier to process CSV data directly from `streams`. This functionality supports data projection, enabling developers to extract and map only the required fields to the target `record` type. + +::: code csv_streams_to_record_array.bal ::: + +::: out csv_streams_to_record_array.out ::: diff --git a/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.metatags b/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.metatags new file mode 100644 index 0000000000..1ff76aa976 --- /dev/null +++ b/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.metatags @@ -0,0 +1,2 @@ +description: This BBE demonstrates the parsing of CSV byte streams into Ballerina record arrays, both with and without data projection. +keywords: ballerina, ballerina by example, bbe, csv, csv byte streams, csv streams, record, record array, parseStream, csv data module, data.csv, data projection diff --git a/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.out b/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.out new file mode 100644 index 0000000000..379d7c5416 --- /dev/null +++ b/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.out @@ -0,0 +1,3 @@ +$ bal run csv_streams_to_record_array.bal +[{"name":"Effective Java","author":"Joshua Bloch","price":45.99,"publishedDate":"2018-01-01"},{"name":"Clean Code","author":"Robert C. Martin","price":37.5,"publishedDate":"2008-08-01"}] +[{"name":"Effective Java","author":"Joshua Bloch"},{"name":"Clean Code","author":"Robert C. Martin"}] diff --git a/examples/index.json b/examples/index.json index 6a45c8a9f1..d9cbb62862 100644 --- a/examples/index.json +++ b/examples/index.json @@ -4700,6 +4700,20 @@ } ] }, + { + "title": "CSV data", + "column": 3, + "category": "Common libraries", + "samples": [ + { + "name": "Parse CSV byte streams to array of records", + "url": "csv-streams-to-record-array", + "verifyBuild": true, + "verifyOutput": true, + "isLearnByExample": true + } + ] + }, { "title": "Constraint", "column": 3, From 4335933bee5f95480b6b96d39136bea5c13f2199 Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Tue, 24 Sep 2024 16:38:03 +0530 Subject: [PATCH 07/23] Update csv stream bbe --- .../csv_streams_to_record_array.bal | 6 +++--- .../csv_streams_to_record_array.md | 2 +- examples/index.json | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.bal b/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.bal index 325ce0177d..0bc02043de 100644 --- a/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.bal +++ b/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.bal @@ -21,16 +21,16 @@ public function main() returns io:Error? { ["Effective Java", "Joshua Bloch", "45.99", "2018-01-01"], ["Clean Code", "Robert C. Martin", "37.5", "2008-08-01"]]); - // Read the CSV file as a `byte stream`. + // Read the CSV file as a byte stream. stream csvStream = check io:fileReadBlocksAsStream(csvFilePath); - // Parse CSV `byte stream` to an array of records. + // Parse as CSV byte stream to an array of records. Book[]|csv:Error bookStream = csv:parseStream(csvStream); io:println(bookStream); stream csvStream2 = check io:fileReadBlocksAsStream(csvFilePath); - // Parse CSV `byte stream` to an array of records with data projection. + // Parse the CSV byte stream as an array of records with data projection. // Here only the fields specified in the target record type (`name` and `author` fields) // are included in the constructed value. BookMinimal[]|csv:Error briefBookStream = csv:parseStream(csvStream2); diff --git a/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.md b/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.md index 2dd7b9fb6b..fdef23383f 100644 --- a/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.md +++ b/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.md @@ -1,6 +1,6 @@ # Parse CSV byte stream to Ballerina record array -The Ballerina `data.csv` library allows parsing CSV byte streams into arrays of Ballerina records, making it easier to process CSV data directly from `streams`. This functionality supports data projection, enabling developers to extract and map only the required fields to the target `record` type. +The Ballerina `data.csv` library allows parsing the CSV byte streams as arrays of Ballerina records, making it easier to process CSV data directly from `streams`. This functionality supports data projection, enabling developers to extract and map only the required fields to the target `record` type. ::: code csv_streams_to_record_array.bal ::: diff --git a/examples/index.json b/examples/index.json index d9cbb62862..933af9db75 100644 --- a/examples/index.json +++ b/examples/index.json @@ -4706,7 +4706,7 @@ "category": "Common libraries", "samples": [ { - "name": "Parse CSV byte streams to array of records", + "name": "Parse CSV byte streams to records", "url": "csv-streams-to-record-array", "verifyBuild": true, "verifyOutput": true, From 34420a664df77590f8cf46b6b137d10ca9cdfa12 Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Tue, 24 Sep 2024 16:48:33 +0530 Subject: [PATCH 08/23] Update the csv to arrays bbe --- .../csv_string_to_anydata_array.bal | 9 +++++---- .../csv_string_to_anydata_array.md | 4 ++-- .../csv_string_to_anydata_array.metatags | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.bal b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.bal index ea86dce3c3..264d1c7d28 100644 --- a/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.bal +++ b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.bal @@ -6,16 +6,17 @@ public function main() { Effective Java,Joshua Bloch,45.99,2018-01-01 Clean Code,Robert C. Martin,37.50,2008-08-01`; - // Parse the CSV string to a array of string arrays. + // Parse the CSV string as an array of `string` arrays. string[][]|csv:Error bookArray = csv:parseString(csvString); io:println(bookArray); - // Parse the CSV string to a array of anydata arrays. + // Parse the CSV string to an array of `anydata` arrays. anydata[][]|csv:Error bookArray2 = csv:parseString(csvString); io:println(bookArray2); - // Parse the CSV string to a array of string arrays with data projection. - // Here only the first two headers are extracted to the resulting array. + // Parse the CSV string to an array of `string` arrays with data projection. + // Here only the first two headers are extracted to the resulting array, + // based on the array member size specified in the expected type. string[][2]|csv:Error bookArray3 = csv:parseString(csvString); io:println(bookArray3); } diff --git a/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.md b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.md index b2795de5e9..743d3767ed 100644 --- a/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.md +++ b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.md @@ -1,6 +1,6 @@ -# Convert CSV string to array of anydata arrays +# Parse CSV string to arrays -The Ballerina `data.csv` library offers a variety of functions for converting CSV strings to sub types of `anydata[][]`, enabling developers to effectively parse CSV string data to flexible data structures. Additionally, it provides the option to select specific fields for inclusion in the resulting array of anydata arrays, ensuring that developers can tailor the output to meet their application's needs +The Ballerina `data.csv` library offers a number of functions for parsing CSV strings to subtypes of `anydata[][]`, enabling developers to effectively parse CSV string data into flexible data structures. Additionally, it provides the option to select specific fields for inclusion in the resulting arrays of `anydata` arrays, allowing developers to extract only the required data. ::: code csv_string_to_anydata_array.bal ::: diff --git a/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.metatags b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.metatags index 82651de3b9..83022a8b7c 100644 --- a/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.metatags +++ b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.metatags @@ -1,2 +1,2 @@ -description: This BBE demonstrates conversion of CSV string to a Ballerina array of anydata arrays with and without data projection. +description: This BBE demonstrates the parsing of a CSV string into an array of anydata arrays, both with and without data projection. keywords: ballerina, ballerina by example, bbe, csv, csv string, anydata, anydata array, parseString, csv data module, data.csv, data projection From ff2edbab290badfe5293fb95d130ce3fd67e96a8 Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Tue, 24 Sep 2024 16:51:25 +0530 Subject: [PATCH 09/23] Update the csv to records array header --- .../csv-string-to-record-array/csv_string_to_record_array.md | 4 ++-- examples/index.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/csv-string-to-record-array/csv_string_to_record_array.md b/examples/csv-string-to-record-array/csv_string_to_record_array.md index d924d30b31..b97aaa5185 100644 --- a/examples/csv-string-to-record-array/csv_string_to_record_array.md +++ b/examples/csv-string-to-record-array/csv_string_to_record_array.md @@ -1,6 +1,6 @@ -# Convert CSV string to array of records +# Parse CSV string to array of records -The ballerina `data.csv` library offers a range of functions for converting CSV strings to array of records, enabling developers to parse CSV string data to structured records while allowing for the selection of specific fields within the expected record array. +The ballerina `data.csv` library offers a range of functions for parsing CSV strings to array of records, enabling developers to parse CSV string data to structured records while allowing for the selection of specific fields within the expected record array. ::: code csv_string_to_record_array.bal ::: diff --git a/examples/index.json b/examples/index.json index e4e0535dbe..44daa9dc09 100644 --- a/examples/index.json +++ b/examples/index.json @@ -4706,7 +4706,7 @@ "category": "Common libraries", "samples": [ { - "name": "Convert CSV string to Ballerina record array", + "name": "Convert CSV string to records", "url": "csv-string-to-record-array", "verifyBuild": true, "verifyOutput": true, From f792b1e144eb4e6438de23b7b6502f2dae7cc4a5 Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Tue, 24 Sep 2024 16:51:59 +0530 Subject: [PATCH 10/23] Update the index file of the csv to arrays bbe --- examples/index.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/index.json b/examples/index.json index 7334f700b1..682947aeff 100644 --- a/examples/index.json +++ b/examples/index.json @@ -4706,7 +4706,7 @@ "category": "Common libraries", "samples": [ { - "name": "Convert CSV string to array of anydata arrays", + "name": "Convert CSV string to arrays", "url": "csv-string-to-anydata-array", "verifyBuild": true, "verifyOutput": true, From 882f2869992d06480b65c49516753ec1d8c9e86c Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Wed, 25 Sep 2024 10:27:49 +0530 Subject: [PATCH 11/23] Add bbe for csv transform api --- examples/index.json | 14 +++++++ .../transform_csv_to_ballerina_types.bal | 39 +++++++++++++++++++ .../transform_csv_to_ballerina_types.md | 7 ++++ .../transform_csv_to_ballerina_types.metatags | 2 + .../transform_csv_to_ballerina_types.out | 3 ++ 5 files changed, 65 insertions(+) create mode 100644 examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.bal create mode 100644 examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.md create mode 100644 examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.metatags create mode 100644 examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.out diff --git a/examples/index.json b/examples/index.json index 6a45c8a9f1..032d99c168 100644 --- a/examples/index.json +++ b/examples/index.json @@ -4700,6 +4700,20 @@ } ] }, + { + "title": "CSV data", + "column": 3, + "category": "Common libraries", + "samples": [ + { + "name": "Transform CSV records to custom types", + "url": "transform-csv-to-ballerina-types", + "verifyBuild": true, + "verifyOutput": true, + "isLearnByExample": true + } + ] + }, { "title": "Constraint", "column": 3, diff --git a/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.bal b/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.bal new file mode 100644 index 0000000000..0f7d00b547 --- /dev/null +++ b/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.bal @@ -0,0 +1,39 @@ +import ballerina/data.csv; +import ballerina/io; + +// Represents the details of an employee. +type Employee record {| + int empId; + string empName; + string department; + float salary; +|}; + +// Represents the salary details of an employee. +type EmployeeSalary record {| + // This annotation is used to map the `empId` field in the source CSV record + // to the `id` field in the target record. + @csv:Name { + value: "empId" + } + int id; + float salary; +|}; + +public function main() returns error? { + Employee[] employees = [ + {empId: 1, empName: "John", department: "Engineering", salary: 1000.0}, + {empId: 2, empName: "Doe", department: "HR", salary: 2000.0}, + {empId: 3, empName: "Jane", department: "Finance", salary: 3000.0} + ]; + + // Transform the `employees` array into an array of `EmployeeSalary` records. + // Only the fields specified in the `EmployeeSalary` type (`id` and `salary`) + // are included in the resulting array. + EmployeeSalary[] employeeSalaries = check csv:transform(employees); + io:println(employeeSalaries); + + // Transform the `employees` array into an array of `anydata` arrays. + anydata[][] employeesArray = check csv:transform(employees); + io:println(employeesArray); +} diff --git a/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.md b/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.md new file mode 100644 index 0000000000..1d9d95a248 --- /dev/null +++ b/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.md @@ -0,0 +1,7 @@ +# Transform CSV records to custom types + +Ballerina allows to transform CSV records into different data structures such as custom `record` types or arrays. It allows for mapping CSV records to target types with selected fields, with or without reshaping the data. + +::: code transform_csv_to_ballerina_types.bal ::: + +::: out transform_csv_to_ballerina_types.out ::: diff --git a/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.metatags b/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.metatags new file mode 100644 index 0000000000..e720181067 --- /dev/null +++ b/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.metatags @@ -0,0 +1,2 @@ +description: This BBE demonstrates the transformation of CSV records into Ballerina arrays, both with and without data projection. +keywords: ballerina, ballerina by example, bbe, csv, csv records, record, record array, transform, csv data module, data.csv, data projection diff --git a/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.out b/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.out new file mode 100644 index 0000000000..357df0d1bc --- /dev/null +++ b/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.out @@ -0,0 +1,3 @@ +$ bal run transform_csv_to_ballerina_types.bal +[{"id":1,"salary":1000.0},{"id":2,"salary":2000.0},{"id":3,"salary":3000.0}] +[[1,"John","Engineering",1000.0],[2,"Doe","HR",2000.0],[3,"Jane","Finance",3000.0]] From b9f447e7fcf34f79f644f97654fa8db69aa56fe2 Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Wed, 25 Sep 2024 11:09:42 +0530 Subject: [PATCH 12/23] Add bbe for csv parse lists --- examples/index.json | 14 ++++++ .../parse_csv_lists_to_ballerina_types.bal | 44 +++++++++++++++++++ .../parse_csv_lists_to_ballerina_types.md | 7 +++ ...arse_csv_lists_to_ballerina_types.metatags | 2 + .../parse_csv_lists_to_ballerina_types.out | 4 ++ 5 files changed, 71 insertions(+) create mode 100644 examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.bal create mode 100644 examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.md create mode 100644 examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.metatags create mode 100644 examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.out diff --git a/examples/index.json b/examples/index.json index 6a45c8a9f1..a20709470e 100644 --- a/examples/index.json +++ b/examples/index.json @@ -4700,6 +4700,20 @@ } ] }, + { + "title": "CSV data", + "column": 3, + "category": "Common libraries", + "samples": [ + { + "name": "Parse CSV lists to custom types", + "url": "parse-csv-lists", + "verifyBuild": true, + "verifyOutput": true, + "isLearnByExample": true + } + ] + }, { "title": "Constraint", "column": 3, diff --git a/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.bal b/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.bal new file mode 100644 index 0000000000..02ee97b025 --- /dev/null +++ b/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.bal @@ -0,0 +1,44 @@ +import ballerina/data.csv; +import ballerina/io; + +// Represents the details of an employee. +type Employee record {| + int empId; + string empName; + string department; + float salary; +|}; + +// Represents the salary details of an employee. +type EmployeeSalary record {| + // This annotation is used to map the empId field in the source CSV record + // to the id field in the target record. + @csv:Name { + value: "empId" + } + int id; + float salary; +|}; + +public function main() returns error? { + // Sample CSV string array representing employee data. + string[][] csvList = [ + ["1", "John", "Engineering", "1000.0"], + ["2", "Doe", "HR", "2000.0"], + ["3", "Jane", "Finance", "3000.0"] + ]; + + // Parse the CSV list into an array of `Employee` records by specifying custom headers. + Employee[] employees = check csv:parseList(csvList, {customHeaders: ["empId", "empName", "department", "salary"]}); + io:println(employees); + + // Parse the CSV list into an array of `EmployeeSalary` records by specifying custom headers. + // Only the fields specified in the EmployeeSalary type (`id` and `salary`) + // are included in the resulting array. + EmployeeSalary[] employeeSalaries = check csv:parseList(csvList, {customHeaders: ["empId", "empName", "department", "salary"]}); + io:println(employeeSalaries); + + // Parse the CSV list into an array of `anydata` arrays. + anydata[][] employeesArray = check csv:parseList(csvList); + io:println(employeesArray); +} diff --git a/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.md b/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.md new file mode 100644 index 0000000000..9b94b48b31 --- /dev/null +++ b/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.md @@ -0,0 +1,7 @@ +# Parse CSV lists to custom types + +Ballerina enables parsing CSV lists into various data structures, including custom `record` types and arrays. It facilitates parsing CSV lists into record types with specified fields, allowing for both reshaping of the data and maintaining its original structure. + +::: code parse_csv_lists_to_ballerina_types.bal ::: + +::: out parse_csv_lists_to_ballerina_types.out ::: diff --git a/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.metatags b/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.metatags new file mode 100644 index 0000000000..63b95ea1a5 --- /dev/null +++ b/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.metatags @@ -0,0 +1,2 @@ +description: This BBE demonstrates the parsing of CSV lists into Ballerina records and arrays, both with and without data projection. +keywords: ballerina, ballerina by example, bbe, csv, csv lists, record, record array, parseList, csv data module, data.csv, data projection diff --git a/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.out b/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.out new file mode 100644 index 0000000000..8c86d532f5 --- /dev/null +++ b/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.out @@ -0,0 +1,4 @@ +$ bal run parse_csv_lists_to_ballerina_types.bal +[{"empId":1,"empName":"John","department":"Engineering","salary":1000.0},{"empId":2,"empName":"Doe","department":"HR","salary":2000.0},{"empId":3,"empName":"Jane","department":"Finance","salary":3000.0}] +[{"id":1,"salary":1000.0},{"id":2,"salary":2000.0},{"id":3,"salary":3000.0}] +[[1,"John","Engineering",1000.0],[2,"Doe","HR",2000.0],[3,"Jane","Finance",3000.0]] From c917dab5b80bebab8b0d0aec5abf48f63959d07f Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Wed, 25 Sep 2024 16:42:00 +0530 Subject: [PATCH 13/23] Add bbe for csv with custom configs --- .../csv_user_configurations.bal | 49 +++++++++++++++++++ .../csv_user_configurations.md | 7 +++ .../csv_user_configurations.metatags | 2 + .../csv_user_configurations.out | 3 ++ examples/index.json | 14 ++++++ 5 files changed, 75 insertions(+) create mode 100644 examples/csv-user-configurations/csv_user_configurations.bal create mode 100644 examples/csv-user-configurations/csv_user_configurations.md create mode 100644 examples/csv-user-configurations/csv_user_configurations.metatags create mode 100644 examples/csv-user-configurations/csv_user_configurations.out diff --git a/examples/csv-user-configurations/csv_user_configurations.bal b/examples/csv-user-configurations/csv_user_configurations.bal new file mode 100644 index 0000000000..25c710f861 --- /dev/null +++ b/examples/csv-user-configurations/csv_user_configurations.bal @@ -0,0 +1,49 @@ +import ballerina/data.csv; +import ballerina/io; + +// Defines the structure of an `Employee` record. +type Employee record {| + int id; + string name; + string department; +|}; + +// Defines the structure of an `EmployeeDepartment` record. +type EmployeeDepartment record {| + int id; + string department; +|}; + +public function main() returns error? { + // This CSV string contains `|` separated values and a comment. + // The `"` character in `O\"Connor` value is escaped using the `\` character. + string csvString = string `id|name|department # This is a comment + 1|O\"Connor|Engineering + 2|Doe|HR + 3|Jane|Finance + 4|John|Engineering`; + + // Parse the CSV string into an array of `Employee` records with specified options. + // The `|` character is used as the delimiter for separating fields. + // The `\` character is used to escape characters (e.g., quotes) within the data. + // The `#` character indicates the start of a comment, lines starting with `#` will be ignored. + // The second and fourth data rows will be skipped during parsing. + Employee[] employees = check csv:parseString(csvString, { + delimiter: "|", + escapeChar: "\\", + comment: "#", + skipLines: [2, 4] + }); + io:println(employees); + + Employee[] employeeRecords = [ + {id: 1, name: "John", department: "Engineering"}, + {id: 2, name: "Doe", department: "HR"}, + {id: 3, name: "Jane", department: "Finance"}, + {id: 4, name: "John", department: "Engineering"} + ]; + + // Transform the employee records into `EmployeeDepartment` records, skipping the second and fourth lines. + EmployeeDepartment[] employeeDepartments = check csv:transform(employeeRecords, {skipLines: [2, 4]}); + io:println(employeeDepartments); +} diff --git a/examples/csv-user-configurations/csv_user_configurations.md b/examples/csv-user-configurations/csv_user_configurations.md new file mode 100644 index 0000000000..1c8b47634f --- /dev/null +++ b/examples/csv-user-configurations/csv_user_configurations.md @@ -0,0 +1,7 @@ +# Handle CSV with custom configurations + +Ballerina provides flexible CSV data handling through custom configurations, enabling developers to define delimiters, escape characters, and skip specific lines. This approach allows efficient parsing and transformation of CSV files, streamlining the integration process. + +::: code csv_user_configurations.bal ::: + +::: out csv_user_configurations.out ::: diff --git a/examples/csv-user-configurations/csv_user_configurations.metatags b/examples/csv-user-configurations/csv_user_configurations.metatags new file mode 100644 index 0000000000..2c2282e197 --- /dev/null +++ b/examples/csv-user-configurations/csv_user_configurations.metatags @@ -0,0 +1,2 @@ +description: This BBE demonstrates how to process CSV data using custom configurations. +keywords: ballerina, ballerina by example, bbe, csv, csv records, record, record array, transform, parseString, csv data module, data.csv, data projection diff --git a/examples/csv-user-configurations/csv_user_configurations.out b/examples/csv-user-configurations/csv_user_configurations.out new file mode 100644 index 0000000000..2c625d5585 --- /dev/null +++ b/examples/csv-user-configurations/csv_user_configurations.out @@ -0,0 +1,3 @@ +$ bal run csv_user_configurations.bal +[{"id":1,"name":"O"Connor","department":"Engineering"},{"id":3,"name":"Jane","department":"Finance"}] +[{"id":1,"department":"Engineering"},{"id":3,"department":"Finance"}] diff --git a/examples/index.json b/examples/index.json index 15332e6527..c00bf486f6 100644 --- a/examples/index.json +++ b/examples/index.json @@ -4721,6 +4721,20 @@ } ] }, + { + "title": "CSV data", + "column": 3, + "category": "Common libraries", + "samples": [ + { + "name": "Handle CSV with custom configurations", + "url": "csv-user-configurations", + "verifyBuild": true, + "verifyOutput": true, + "isLearnByExample": true + } + ] + }, { "title": "Constraint", "column": 3, From 3e16a7d78daac86ff30cc25131a338e871ba3713 Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Thu, 26 Sep 2024 17:11:53 +0530 Subject: [PATCH 14/23] Update csv transform bbe --- examples/index.json | 2 +- .../transform_csv_records_to_custom_types.bal} | 6 +++--- .../transform_csv_records_to_custom_types.md | 7 +++++++ .../transform_csv_records_to_custom_types.metatags} | 0 .../transform_csv_records_to_custom_types.out} | 2 +- .../transform_csv_to_ballerina_types.md | 7 ------- 6 files changed, 12 insertions(+), 12 deletions(-) rename examples/{transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.bal => transform-csv-records-to-custom-types/transform_csv_records_to_custom_types.bal} (92%) create mode 100644 examples/transform-csv-records-to-custom-types/transform_csv_records_to_custom_types.md rename examples/{transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.metatags => transform-csv-records-to-custom-types/transform_csv_records_to_custom_types.metatags} (100%) rename examples/{transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.out => transform-csv-records-to-custom-types/transform_csv_records_to_custom_types.out} (75%) delete mode 100644 examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.md diff --git a/examples/index.json b/examples/index.json index 032d99c168..ac465aff0e 100644 --- a/examples/index.json +++ b/examples/index.json @@ -4707,7 +4707,7 @@ "samples": [ { "name": "Transform CSV records to custom types", - "url": "transform-csv-to-ballerina-types", + "url": "transform-csv-records-to-custom-types", "verifyBuild": true, "verifyOutput": true, "isLearnByExample": true diff --git a/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.bal b/examples/transform-csv-records-to-custom-types/transform_csv_records_to_custom_types.bal similarity index 92% rename from examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.bal rename to examples/transform-csv-records-to-custom-types/transform_csv_records_to_custom_types.bal index 0f7d00b547..19e2afe079 100644 --- a/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.bal +++ b/examples/transform-csv-records-to-custom-types/transform_csv_records_to_custom_types.bal @@ -6,7 +6,7 @@ type Employee record {| int empId; string empName; string department; - float salary; + decimal salary; |}; // Represents the salary details of an employee. @@ -17,7 +17,7 @@ type EmployeeSalary record {| value: "empId" } int id; - float salary; + decimal salary; |}; public function main() returns error? { @@ -29,7 +29,7 @@ public function main() returns error? { // Transform the `employees` array into an array of `EmployeeSalary` records. // Only the fields specified in the `EmployeeSalary` type (`id` and `salary`) - // are included in the resulting array. + // are included in the values in the resulting array. EmployeeSalary[] employeeSalaries = check csv:transform(employees); io:println(employeeSalaries); diff --git a/examples/transform-csv-records-to-custom-types/transform_csv_records_to_custom_types.md b/examples/transform-csv-records-to-custom-types/transform_csv_records_to_custom_types.md new file mode 100644 index 0000000000..a9ab297f30 --- /dev/null +++ b/examples/transform-csv-records-to-custom-types/transform_csv_records_to_custom_types.md @@ -0,0 +1,7 @@ +# Transform CSV records to custom types + +Ballerina supports transforming CSV records into various data structures, such as custom records or arrays. It allows users to map CSV records to target types by specifying only the required fields. + +::: code transform_csv_records_to_custom_types.bal ::: + +::: out transform_csv_records_to_custom_types.out ::: diff --git a/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.metatags b/examples/transform-csv-records-to-custom-types/transform_csv_records_to_custom_types.metatags similarity index 100% rename from examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.metatags rename to examples/transform-csv-records-to-custom-types/transform_csv_records_to_custom_types.metatags diff --git a/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.out b/examples/transform-csv-records-to-custom-types/transform_csv_records_to_custom_types.out similarity index 75% rename from examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.out rename to examples/transform-csv-records-to-custom-types/transform_csv_records_to_custom_types.out index 357df0d1bc..4dfa9d0d49 100644 --- a/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.out +++ b/examples/transform-csv-records-to-custom-types/transform_csv_records_to_custom_types.out @@ -1,3 +1,3 @@ -$ bal run transform_csv_to_ballerina_types.bal +$ bal run transform_csv_records_to_custom_types.bal [{"id":1,"salary":1000.0},{"id":2,"salary":2000.0},{"id":3,"salary":3000.0}] [[1,"John","Engineering",1000.0],[2,"Doe","HR",2000.0],[3,"Jane","Finance",3000.0]] diff --git a/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.md b/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.md deleted file mode 100644 index 1d9d95a248..0000000000 --- a/examples/transform-csv-to-ballerina-types/transform_csv_to_ballerina_types.md +++ /dev/null @@ -1,7 +0,0 @@ -# Transform CSV records to custom types - -Ballerina allows to transform CSV records into different data structures such as custom `record` types or arrays. It allows for mapping CSV records to target types with selected fields, with or without reshaping the data. - -::: code transform_csv_to_ballerina_types.bal ::: - -::: out transform_csv_to_ballerina_types.out ::: From f7f3c7c3af1cc49c3c071f6d2660bea811bbed21 Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Fri, 27 Sep 2024 09:21:42 +0530 Subject: [PATCH 15/23] Update metatags with csv to array --- .../csv_string_to_anydata_array.metatags | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.metatags b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.metatags index 83022a8b7c..5d1ccb7785 100644 --- a/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.metatags +++ b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.metatags @@ -1,2 +1,2 @@ description: This BBE demonstrates the parsing of a CSV string into an array of anydata arrays, both with and without data projection. -keywords: ballerina, ballerina by example, bbe, csv, csv string, anydata, anydata array, parseString, csv data module, data.csv, data projection +keywords: ballerina, ballerina by example, bbe, csv, csv to array, csv string, anydata, anydata array, parseString, csv data module, data.csv, data projection From dd59155743e2f0937db012fb644f5483a28b8e8d Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Fri, 27 Sep 2024 09:36:29 +0530 Subject: [PATCH 16/23] Update csv streams bbe --- .../csv_streams_to_record_array.bal | 14 +++++++------- .../csv_streams_to_record_array.metatags | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.bal b/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.bal index 0bc02043de..9d45daabec 100644 --- a/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.bal +++ b/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.bal @@ -15,24 +15,24 @@ type BookMinimal record {| const csvFilePath = "./files/csv_file.csv"; -public function main() returns io:Error? { +public function main() returns error? { // Write a CSV file. check io:fileWriteCsv(csvFilePath, [["name", "author", "price", "publishedDate"], ["Effective Java", "Joshua Bloch", "45.99", "2018-01-01"], ["Clean Code", "Robert C. Martin", "37.5", "2008-08-01"]]); // Read the CSV file as a byte stream. - stream csvStream = check io:fileReadBlocksAsStream(csvFilePath); + stream bookStream = check io:fileReadBlocksAsStream(csvFilePath); // Parse as CSV byte stream to an array of records. - Book[]|csv:Error bookStream = csv:parseStream(csvStream); - io:println(bookStream); + Book[] bookArray = check csv:parseStream(bookStream); + io:println(bookArray); - stream csvStream2 = check io:fileReadBlocksAsStream(csvFilePath); + bookStream = check io:fileReadBlocksAsStream(csvFilePath); // Parse the CSV byte stream as an array of records with data projection. // Here only the fields specified in the target record type (`name` and `author` fields) // are included in the constructed value. - BookMinimal[]|csv:Error briefBookStream = csv:parseStream(csvStream2); - io:println(briefBookStream); + BookMinimal[] briefBookArray = check csv:parseStream(bookStream); + io:println(briefBookArray); } diff --git a/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.metatags b/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.metatags index 1ff76aa976..88daca8c8e 100644 --- a/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.metatags +++ b/examples/csv-streams-to-record-arrays/csv_streams_to_record_array.metatags @@ -1,2 +1,2 @@ description: This BBE demonstrates the parsing of CSV byte streams into Ballerina record arrays, both with and without data projection. -keywords: ballerina, ballerina by example, bbe, csv, csv byte streams, csv streams, record, record array, parseStream, csv data module, data.csv, data projection +keywords: ballerina, ballerina by example, bbe, csv, csv byte streams, csv streams, record, record array, parseStream, csv data module, data.csv, data projection, csv to stream From 999e036575fd155154ffbf3376beca1ae189f386 Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Fri, 27 Sep 2024 09:43:43 +0530 Subject: [PATCH 17/23] Update csv to arrays bbe --- .../csv_string_to_anydata_array.bal | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.bal b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.bal index 264d1c7d28..0dbb629a71 100644 --- a/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.bal +++ b/examples/csv-string-to-anydata-array/csv_string_to_anydata_array.bal @@ -1,22 +1,22 @@ import ballerina/data.csv; import ballerina/io; -public function main() { +public function main() returns error? { string csvString = string `name,author,price,publishedDate Effective Java,Joshua Bloch,45.99,2018-01-01 Clean Code,Robert C. Martin,37.50,2008-08-01`; // Parse the CSV string as an array of `string` arrays. - string[][]|csv:Error bookArray = csv:parseString(csvString); + string[][] bookArray = check csv:parseString(csvString); io:println(bookArray); // Parse the CSV string to an array of `anydata` arrays. - anydata[][]|csv:Error bookArray2 = csv:parseString(csvString); + anydata[][] bookArray2 = check csv:parseString(csvString); io:println(bookArray2); // Parse the CSV string to an array of `string` arrays with data projection. // Here only the first two headers are extracted to the resulting array, // based on the array member size specified in the expected type. - string[][2]|csv:Error bookArray3 = csv:parseString(csvString); + string[][2] bookArray3 = check csv:parseString(csvString); io:println(bookArray3); } From 17d96ce2322d8a347cba3b4dc050c91bc80f35b8 Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Fri, 27 Sep 2024 09:47:52 +0530 Subject: [PATCH 18/23] Add csv string to records bbe --- examples/index.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/index.json b/examples/index.json index f64e044de2..1fcc8082ca 100644 --- a/examples/index.json +++ b/examples/index.json @@ -4738,6 +4738,13 @@ "column": 3, "category": "Common libraries", "samples": [ + { + "name": "Convert CSV string to records", + "url": "csv-string-to-record-array", + "verifyBuild": true, + "verifyOutput": true, + "isLearnByExample": true + }, { "name": "Parse CSV byte streams to records", "url": "csv-streams-to-record-array", From ab756e8ec837ab7f04e27090334557140dd5dc2b Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Fri, 27 Sep 2024 09:50:34 +0530 Subject: [PATCH 19/23] Add csv to record array tag for csv bbes --- .../parse-csv-lists/parse_csv_lists_to_ballerina_types.metatags | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.metatags b/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.metatags index 63b95ea1a5..41bb1663d9 100644 --- a/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.metatags +++ b/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.metatags @@ -1,2 +1,2 @@ description: This BBE demonstrates the parsing of CSV lists into Ballerina records and arrays, both with and without data projection. -keywords: ballerina, ballerina by example, bbe, csv, csv lists, record, record array, parseList, csv data module, data.csv, data projection +keywords: ballerina, ballerina by example, bbe, csv, csv lists, record, record array, parseList, csv data module, data.csv, data projection, csv to record array From 144fb28525e97d1f4c89fda07c6aaadcfde39697 Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Fri, 27 Sep 2024 10:00:29 +0530 Subject: [PATCH 20/23] update csv config bbe --- .../csv_user_configurations.bal | 21 +++++++++++++------ .../csv_user_configurations.metatags | 2 +- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/examples/csv-user-configurations/csv_user_configurations.bal b/examples/csv-user-configurations/csv_user_configurations.bal index 25c710f861..43a6cf8a2e 100644 --- a/examples/csv-user-configurations/csv_user_configurations.bal +++ b/examples/csv-user-configurations/csv_user_configurations.bal @@ -22,18 +22,21 @@ public function main() returns error? { 2|Doe|HR 3|Jane|Finance 4|John|Engineering`; - - // Parse the CSV string into an array of `Employee` records with specified options. + + // Defines the options for parsing the CSV string. // The `|` character is used as the delimiter for separating fields. // The `\` character is used to escape characters (e.g., quotes) within the data. // The `#` character indicates the start of a comment, lines starting with `#` will be ignored. // The second and fourth data rows will be skipped during parsing. - Employee[] employees = check csv:parseString(csvString, { + csv:ParseOptions parseOptions = { delimiter: "|", escapeChar: "\\", comment: "#", skipLines: [2, 4] - }); + }; + + // Parse the CSV string into an array of `Employee` records with specified options. + Employee[] employees = check csv:parseString(csvString, parseOptions); io:println(employees); Employee[] employeeRecords = [ @@ -43,7 +46,13 @@ public function main() returns error? { {id: 4, name: "John", department: "Engineering"} ]; - // Transform the employee records into `EmployeeDepartment` records, skipping the second and fourth lines. - EmployeeDepartment[] employeeDepartments = check csv:transform(employeeRecords, {skipLines: [2, 4]}); + // Defines the options for transforming the `Employee` records. + // The second and fourth data rows will be skipped during transformation. + csv:TransformOptions transformOptions = { + skipLines: [2, 4] + }; + + // Transform the `employee` records into `EmployeeDepartment` records with specified options. + EmployeeDepartment[] employeeDepartments = check csv:transform(employeeRecords, transformOptions); io:println(employeeDepartments); } diff --git a/examples/csv-user-configurations/csv_user_configurations.metatags b/examples/csv-user-configurations/csv_user_configurations.metatags index 2c2282e197..8952d4f702 100644 --- a/examples/csv-user-configurations/csv_user_configurations.metatags +++ b/examples/csv-user-configurations/csv_user_configurations.metatags @@ -1,2 +1,2 @@ description: This BBE demonstrates how to process CSV data using custom configurations. -keywords: ballerina, ballerina by example, bbe, csv, csv records, record, record array, transform, parseString, csv data module, data.csv, data projection +keywords: ballerina, ballerina by example, bbe, csv, csv records, record, record array, transform, parseString, csv data module, data.csv, data projection, csv configs, csv configurations From 2a879f755ace3c2111e8323d96c58e1d551adfe5 Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Wed, 2 Oct 2024 14:40:33 +0530 Subject: [PATCH 21/23] rename the parse csv list ballerina by example --- ...e_csv_lists_to_ballerina_types.bal => parse_csv_lists.bal} | 0 ...rse_csv_lists_to_ballerina_types.md => parse_csv_lists.md} | 4 ++-- ...s_to_ballerina_types.metatags => parse_csv_lists.metatags} | 0 ...e_csv_lists_to_ballerina_types.out => parse_csv_lists.out} | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename examples/parse-csv-lists/{parse_csv_lists_to_ballerina_types.bal => parse_csv_lists.bal} (100%) rename examples/parse-csv-lists/{parse_csv_lists_to_ballerina_types.md => parse_csv_lists.md} (74%) rename examples/parse-csv-lists/{parse_csv_lists_to_ballerina_types.metatags => parse_csv_lists.metatags} (100%) rename examples/parse-csv-lists/{parse_csv_lists_to_ballerina_types.out => parse_csv_lists.out} (88%) diff --git a/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.bal b/examples/parse-csv-lists/parse_csv_lists.bal similarity index 100% rename from examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.bal rename to examples/parse-csv-lists/parse_csv_lists.bal diff --git a/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.md b/examples/parse-csv-lists/parse_csv_lists.md similarity index 74% rename from examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.md rename to examples/parse-csv-lists/parse_csv_lists.md index 9b94b48b31..fce07a794d 100644 --- a/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.md +++ b/examples/parse-csv-lists/parse_csv_lists.md @@ -2,6 +2,6 @@ Ballerina enables parsing CSV lists into various data structures, including custom `record` types and arrays. It facilitates parsing CSV lists into record types with specified fields, allowing for both reshaping of the data and maintaining its original structure. -::: code parse_csv_lists_to_ballerina_types.bal ::: +::: code parse_csv_lists.bal ::: -::: out parse_csv_lists_to_ballerina_types.out ::: +::: out parse_csv_lists.out ::: diff --git a/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.metatags b/examples/parse-csv-lists/parse_csv_lists.metatags similarity index 100% rename from examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.metatags rename to examples/parse-csv-lists/parse_csv_lists.metatags diff --git a/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.out b/examples/parse-csv-lists/parse_csv_lists.out similarity index 88% rename from examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.out rename to examples/parse-csv-lists/parse_csv_lists.out index 8c86d532f5..89ff880d1c 100644 --- a/examples/parse-csv-lists/parse_csv_lists_to_ballerina_types.out +++ b/examples/parse-csv-lists/parse_csv_lists.out @@ -1,4 +1,4 @@ -$ bal run parse_csv_lists_to_ballerina_types.bal +$ bal run parse_csv_lists.bal [{"empId":1,"empName":"John","department":"Engineering","salary":1000.0},{"empId":2,"empName":"Doe","department":"HR","salary":2000.0},{"empId":3,"empName":"Jane","department":"Finance","salary":3000.0}] [{"id":1,"salary":1000.0},{"id":2,"salary":2000.0},{"id":3,"salary":3000.0}] [[1,"John","Engineering",1000.0],[2,"Doe","HR",2000.0],[3,"Jane","Finance",3000.0]] From 0bfb74cdc9764c16b35ea09ddf8e56e900ad6089 Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Wed, 2 Oct 2024 15:19:19 +0530 Subject: [PATCH 22/23] Update checkstyles in csv ballerina by examples --- examples/parse-csv-lists/parse_csv_lists.bal | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/parse-csv-lists/parse_csv_lists.bal b/examples/parse-csv-lists/parse_csv_lists.bal index 02ee97b025..6ce89440f9 100644 --- a/examples/parse-csv-lists/parse_csv_lists.bal +++ b/examples/parse-csv-lists/parse_csv_lists.bal @@ -29,13 +29,17 @@ public function main() returns error? { ]; // Parse the CSV list into an array of `Employee` records by specifying custom headers. - Employee[] employees = check csv:parseList(csvList, {customHeaders: ["empId", "empName", "department", "salary"]}); + Employee[] employees = check csv:parseList(csvList, { + customHeaders: ["empId", "empName", "department", "salary"] + }); io:println(employees); // Parse the CSV list into an array of `EmployeeSalary` records by specifying custom headers. // Only the fields specified in the EmployeeSalary type (`id` and `salary`) // are included in the resulting array. - EmployeeSalary[] employeeSalaries = check csv:parseList(csvList, {customHeaders: ["empId", "empName", "department", "salary"]}); + EmployeeSalary[] employeeSalaries = check csv:parseList(csvList, { + customHeaders: ["empId", "empName", "department", "salary"] + }); io:println(employeeSalaries); // Parse the CSV list into an array of `anydata` arrays. From 1fd12e9e6e93baff01b9fdf1671f94931b0eb8b6 Mon Sep 17 00:00:00 2001 From: Sasindu Alahakoon Date: Wed, 2 Oct 2024 16:43:52 +0530 Subject: [PATCH 23/23] Disable verify output for csv bbes --- examples/index.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/index.json b/examples/index.json index b8ca758373..ffe520fab0 100644 --- a/examples/index.json +++ b/examples/index.json @@ -4730,42 +4730,42 @@ "name": "Convert CSV string to records", "url": "csv-string-to-record-array", "verifyBuild": true, - "verifyOutput": true, + "verifyOutput": false, "isLearnByExample": true }, { "name": "Convert CSV string to arrays", "url": "csv-string-to-anydata-array", "verifyBuild": true, - "verifyOutput": true, + "verifyOutput": false, "isLearnByExample": true }, { "name": "Parse CSV byte streams to records", "url": "csv-streams-to-record-array", "verifyBuild": true, - "verifyOutput": true, + "verifyOutput": false, "isLearnByExample": true }, { "name": "Parse CSV lists to custom types", "url": "parse-csv-lists", "verifyBuild": true, - "verifyOutput": true, + "verifyOutput": false, "isLearnByExample": true }, { "name": "Transform CSV records to custom types", "url": "transform-csv-records-to-custom-types", "verifyBuild": true, - "verifyOutput": true, + "verifyOutput": false, "isLearnByExample": true }, { "name": "Handle CSV with custom configurations", "url": "csv-user-configurations", "verifyBuild": true, - "verifyOutput": true, + "verifyOutput": false, "isLearnByExample": true } ]