diff --git a/.github/workflows/build-with-bal-test-graalvm.yml b/.github/workflows/build-with-bal-test-graalvm.yml index 291054d..77a583e 100644 --- a/.github/workflows/build-with-bal-test-graalvm.yml +++ b/.github/workflows/build-with-bal-test-graalvm.yml @@ -35,3 +35,5 @@ jobs: lang_tag: ${{ inputs.lang_tag }} lang_version: ${{ inputs.lang_version }} native_image_options: '-J-Xmx7G ${{ inputs.native_image_options }}' + # TODO : Enable after fixing this issue : https://github.com/ballerina-platform/ballerina-lang/issues/38882 + additional_windows_build_flags: '-x test' diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index 9454d4e..0edaae9 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -12,5 +12,5 @@ jobs: uses: ballerina-platform/ballerina-library/.github/workflows/release-package-template.yml@main secrets: inherit with: - package-name: data.jsondata + package-name: data.csv package-org: ballerina diff --git a/.github/workflows/stale_check.yml b/.github/workflows/stale_check.yml new file mode 100644 index 0000000..8763360 --- /dev/null +++ b/.github/workflows/stale_check.yml @@ -0,0 +1,19 @@ +name: 'Close stale pull requests' + +on: + schedule: + - cron: '30 19 * * *' + workflow_dispatch: + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v3 + with: + stale-pr-message: 'This PR has been open for more than 15 days with no activity. This will be closed in 3 days unless the `stale` label is removed or commented.' + close-pr-message: 'Closed PR due to inactivity for more than 18 days.' + days-before-pr-stale: 15 + days-before-pr-close: 3 + days-before-issue-stale: -1 + days-before-issue-close: -1 diff --git a/.gitignore b/.gitignore index 1d2a7ee..38d72e0 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ *.zip *.tar.gz *.rar +*.deb # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.csv hs_err_pid* diff --git a/README.md b/README.md index ba9bc60..4236cc1 100644 --- a/README.md +++ b/README.md @@ -1,407 +1,160 @@ -# Ballerina CSV Data Library +# Ballerina CSV Data Library -The Ballerina CSV Data Library is a comprehensive toolkit designed to facilitate the handling and manipulation of CSV data within Ballerina applications. It streamlines the process of converting CSV data to native Ballerina data types, enabling developers to work with CSV content seamlessly and efficiently. +[![Build](https://github.com/ballerina-platform/module-ballerina-data.csv/actions/workflows/build-timestamped-master.yml/badge.svg)](https://github.com/ballerina-platform/module-ballerina-data.csv/actions/workflows/build-timestamped-master.yml) +[![codecov](https://codecov.io/gh/ballerina-platform/module-ballerina-data.csv/branch/main/graph/badge.svg)](https://codecov.io/gh/ballerina-platform/module-ballerina-data.csv) +[![Trivy](https://github.com/ballerina-platform/module-ballerina-data.csv/actions/workflows/trivy-scan.yml/badge.svg)](https://github.com/ballerina-platform/module-ballerina-data.csv/actions/workflows/trivy-scan.yml) +[![GraalVM Check](https://github.com/ballerina-platform/module-ballerina-data.csv/actions/workflows/build-with-bal-test-graalvm.yml/badge.svg)](https://github.com/ballerina-platform/module-ballerina-data.csv/actions/workflows/build-with-bal-test-graalvm.yml) +[![GitHub Last Commit](https://img.shields.io/github/last-commit/ballerina-platform/module-ballerina-data.csv.svg)](https://github.com/ballerina-platform/module-ballerina-data.csv/commits/master) +[![Github issues](https://img.shields.io/github/issues/ballerina-platform/ballerina-standard-library/module/data.csv.svg?label=Open%20Issues)](https://github.com/ballerina-platform/ballerina-standard-library/labels/module%2Fdata.csv) -This library is the refined successor of the `ballerina/csvdata` module, incorporating enhanced functionalities and improved performance. +The Ballerina CSV Data Library is a comprehensive toolkit designed to facilitate the handling and manipulation of CSV data within Ballerina applications. It streamlines the process of converting CSV data to native Ballerina data types, enabling developers to work with CSV content seamlessly and efficiently. ## Features -- **Versatile CSV Data Input**: Accept CSV data as a csv, a string, byte array, or a stream and convert it into a Record value. -- **CSV to Record Value Conversion**: Transform CSV data into Ballerina records with ease in compliance with OpenAPI 3 standards. -- **Projection Support**: Perform selective conversion of CSV data subsets into Record values through projection. +- **Versatile CSV Data Input**: Accept CSV data as a string, byte array, or a stream and convert it into a subtype of ballerina records or lists. +- **CSV to anydata Value transformation**: Transform CSV data into expected type which is subtype of ballerina record arrays or anydata arrays. +- **Projection Support**: Perform selective conversion of CSV data subsets into ballerina record array or anydata array values through projection. ## Usage -### Converting an CSV value to a Record value +### Converting CSV string to a record array -To convert an CSV value to a Record value, you can utilize the `fromCsvWithType` function provided by the library. The example below showcases the transformation of an CSV value into a Record value. +To convert a CSV string into a record array value, you can use the `parseString` function from the library. The following example demonstrates how to transform a CSV document into an array of records. ```ballerina import ballerina/data.csv; import ballerina/io; -public function main() returns error? { - csv data = csv ` - 0 - string - string - `; +type Book record { + string name; + string author; + int year; +}; - Book book = check csvdata:fromCsvWithType(data); - io:println(book); +public function main() returns error? { + string csvString = string `name,author,year + Clean Code,Robert C. Martin,2008 + The Pragmatic Programmer,Andrew Hunt and David Thomas,1999`; + + Book[] books = check csv:parseString(csvString); + foreach var book in books { + io:println(book); + } } - -type Book record {| - int id; - string title; - string author; -|}; ``` -### Converting an external CSV document to a Record value +### Converting external CSV document to a record value -For transforming CSV content from an external source into a Record value, the `fromCsvStringWithType` function can be used. This external source can be in the form of a string or a byte array/byte stream that houses the CSV data. This is commonly extracted from files or network sockets. The example below demonstrates the conversion of an CSV value from an external source into a Record value. +For transforming CSV content from an external source into a record value, the `parseString`, `parseBytes` and `parseStream` functions can be used. This external source can be in the form of a string or a byte array/byte block stream that houses the CSV data. This is commonly extracted from files or network sockets. The example below demonstrates the conversion of an CSV value from an external source into a record value. ```ballerina import ballerina/data.csv; import ballerina/io; +type Book record { + string name; + string author; + int year; +}; + public function main() returns error? { + // Read the CSV content as a string string csvContent = check io:fileReadString("path/to/file.csv"); - Book book = check csvdata:fromCsvStringWithType(csvContent); + Book[] book = check csv:parseString(csvContent); io:println(book); -} - -type Book record {| - int id; - string title; - string author; -|}; -``` - -Make sure to handle possible errors that may arise during the file reading or CSV to record conversion process. The `check` keyword is utilized to handle these errors, but more sophisticated error handling can be implemented as per your requirements. -## CSV to Record Canonical Representation - -The translation of CSV to a Record representation is a fundamental feature of the library. It facilitates a structured and type-safe approach to handling CSV data within Ballerina applications. - -Take for instance the following CSV snippet: - -```csv - - 0 - string - string - -``` - -CSV data is inherently hierarchical, forming a tree structure. In the given example, the root element is `book`, which encompasses three child elements: `id`, `title`, and `author`. The `id` element harbors a numeric value `0`, whereas both the `title` and `author` elements contain string values. - -A straightforward record representation of the above CSV data is: - -```ballerina -type Book record {| - int id; - string title; - string author; -|}; + // Read the CSV content as a stream + stream csvStream = check io:fileReadBlocksAsStream("path/to/file.csv"); + Book[] book2 = check csv:parseStream(csvStream); + io:println(book2); +} ``` -In this representation, the CSV data is efficiently translated into a record value. The `book` element is mapped to a record of type `Book`, and the child elements `id`, `title`, and `author` are converted into record fields of types `int` and `string` correspondingly. - -This record type definition can be further refined through annotations. Moreover, utilizing open and closed records grants control over the translation process, which is elaborated in subsequent sections. - -### CSV Element Names - -The name of the CSV element serves as the name of the record field, altered to fit a valid Ballerina identifier. Notably, the record field name corresponds to the local name of the CSV element, with any namespace prefixes being disregarded. +Make sure to handle possible errors that may arise during the file reading or CSV to record/array conversion process. The `check` keyword is utilized to handle these errors, but more sophisticated error handling can be implemented as per your requirements. -Consider the CSV snippet: +## CSV to record array/anydata array of array representation -```csv - - 0 - string - string - -``` - -The canonical representation of the above CSV as a Ballerina record is: - -```ballerina -type Book record {| - int id; - string 'title\-name'; - string 'author\-name'; -|}; -``` - -Observe how the CSV element names `title-name` and `author-name` are represented using delimited identifiers in Ballerina; the `-` characters in the CSV element names are escaped using the `\` character. - -Moreover, the `@Name` annotation can be utilized to explicitly specify the name of the record field, providing control over the translation process: +The CSV Object can be represented as a value of type `record/map array` or `string array of array` in Ballerina, which facilitates a structured and type-safe approach to handling CSV data. +The conversion of CSV data to subtype of `record array` or `anydata array of array` representation is a fundamental feature of the library. ```ballerina import ballerina/data.csv; +import ballerina/io; -type Book record {| - int id; - @csvdata:Name { value: "title-name" } - string title; - @csvdata:Name { value: "author-name" } - string author; -|}; -``` - -### CSV Attributes - -Similarly to CSV elements, CSV attributes are also represented into record fields within the corresponding parent Record type. The name of the CSV attribute is converted into the name of the record field, ensuring it is a valid Ballerina identifier. It is crucial to emphasize that the record field name aligns with the local name of the CSV attribute, and any namespace prefixes are ignored. - -Consider the following CSV snippet: - -```csv - - 0 - string - string - -``` - -The canonical representation of the above CSV as a Ballerina record is: - -```ballerina -type Book record {| - string lang; - decimal price; - int id; - string title; - string author; -|}; -``` - -Additionally the `@Attribute` annotation can be utilized to explicitly specify the name of the record field, providing control over the translation process. - -### Child Elements - -Child elements are mapped to record fields, with the type reflecting that of the corresponding child element. - -Examine the CSV snippet below: - -```csv - - 0 - string - - string - string - - -``` - -The canonical representation of the above CSV as a Ballerina record is: - -```ballerina -type Book record {| - int id; - string title; - Author author; -|}; - -type Author record {| +type Book record { string name; - string country; -|}; -``` - -In this transformation, child elements, like the `author` element containing its own sub-elements, are converted into nested records. This maintains the hierarchical structure of the CSV data within the Ballerina type system, enabling intuitive and type-safe data manipulation. - -Alternatively, inline type definitions offer a compact method for representing child elements as records within their parent record. This approach is particularly beneficial when the child record does not require reuse elsewhere and is unique to its parent record. - -Consider the subsequent Ballerina record definition, which employs inline type definition for the `author` field: - -```ballerina -type Book record {| - int id; - string title; - record {| - string name; - string country; - |} author; -|}; -``` - -### CSV Text Content - -The transformation of CSV text content into record fields typically involves types like `string`, `boolean`, `int`, `float`, or `decimal`, depending on the textual content. For numeric values where type information is not explicitly defined, the default conversion type is `decimal`. Conversely, for non-numeric content, the default type is `string`. - -Consider the CSV snippet below: + int year; +}; -```csv - - 0 - string - string - true - 10.5 - -``` - -The translation into a Ballerina record would be as follows: - -```ballerina -type Book record {| - int id; - string title; - string author; - boolean available; - decimal price; -|}; -``` - -In scenarios where the parent CSV element of text content also includes attributes, the CSV text content can be represented by a `string` type field named `#content` within a record type, with the attributes being mapped to their respective fields. - -For instance, examine this CSV: - -```csv - - 0 - string - 10.5 - -``` - -The canonical translation of CSV to a Ballerina record is as such: - -```ballerina -type Book record {| - int id; - Title title; - decimal price; -|}; - -type Title record {| - string \#content; - string lang; -|}; -``` - -Modifications to the default behavior for converting numerical values can be achieved by providing `Options` mappings to the respective functions. This enables developers to choose specific data types and exert finer control over the conversion process. - -### CSV Namespaces - -CSV namespaces are accommodated by the library, supporting the translation of CSV data that contains namespace prefixes. However, the presence of CSV namespaces is not mandatory, and the library is capable of processing CSV data without namespaces. Should namespaces be present, they will be utilized to resolve the names of CSV elements and attributes. - -It's important to note that, unlike in the `csvdata` module, the namespace prefixes do not reflect in the record field names, as the record field names align with the local names of the CSV elements. - -Examine the CSV snippet below with default namespaces: - -```csv - - 0 - string - string - -``` - -The translation into a Ballerina record would be: +public function main() returns error? { + string[][] bookArray = [["Clean Code","2008"],["Clean Architecture","2017"]]; + Book[] bookRecords = [{name: "Clean Code", year: 2008}, {name: "Clean Architecture", year: 2017}]; -```ballerina -type Book record {| - int id; - string title; - string author; -|}; -``` + // Parse and output a record array from a source of string array of arrays. + Book[] books = check csv:parseList(bookArray, {customHeaders: ["name", "year"]}); + io:println(books); -Incorporating namespace validation yields: + // Parse and output a tuple array from a source of string array of arrays. + [string, int][] books2 = check csv:parseList(bookArray, {customHeaders: ["name", "year"]}); + io:println(books2); -```ballerina -import ballerina/data.csv; - -@csvdata:Namespace { - uri: "http://example.com/book" + // Transform CSV records to a string array of arrays. + [string, int][] books3 = check csv:transform(bookRecords); + io:println(books3); } -type Book record {| - int id; - string title; - string author; -|}; ``` -Here is the same CSV snippet with a namespace prefix: +### Controlling the CSV value to record array conversion -```csv - - 0 - string - string - -``` - -The translation into a Ballerina record would be: +The library allows for selective conversion of CSV into closed record arrays. This is beneficial when the CSV data contains headers that are not necessary to be transformed into record fields. ```ballerina import ballerina/data.csv; +import ballerina/io; -@csvdata:Namespace { - uri: "http://example.com/book", - prefix: "bk" -} type Book record {| - int id; - string title; + string name; string author; |}; -``` - -In these examples, the CSV namespaces are appropriately acknowledged, ensuring the integrity of the CSV structure within the Ballerina records. - -### Working with Arrays - -The library is equipped to handle the transformation of CSV data containing arrays into Ballerina records. - -Take the following CSV snippet as an example: - -```csv - - 0 - string - string - string - string - -``` - -The canonical representation of this CSV as a Ballerina record is: - -```ballerina -type Book record {| - int id; - string title; - string[] author; -|}; -``` - -### Controlling Which Elements to Convert - -The library allows for selective conversion of CSV elements into records through the use of rest fields. This is beneficial when the CSV data contains elements that are not necessary to be transformed into record fields. -Take this CSV snippet as an example: - -```csv - - 0 - string - string - 10.5 - -``` - -Suppose that only the book `id`, and `title` elements are needed for conversion into record fields. This can be achieved by defining only the required fields in the record type and omitting the rest field: - -```ballerina -type Book record {| - int id; - string title; -|}; +public function main() returns error? { + record {}[] csvContent = [{ + "name": "Clean Code", + "author": "Robert C. Martin", + "year": 2008, + "publisher": "Prentice Hall" + }, { + "name": "The Pragmatic Programmer", + "author": "Andrew Hunt and David Thomas", + "year": 1999, + "publisher": "Addison-Wesley" + }]; + + // The CSV data above contains publisher and year fields which are not + // required to be converted into a record field. + Book[] book = check csv:transform(csvContent); + io:println(book); +} ``` -However, if the rest field is utilized (or if the record type is defined as an open record), all elements in the CSV data will be transformed into record fields: +However, if the rest field is utilized (or if the record type is defined as an open record), all members in the CSV data will be transformed into record fields: ```ballerina -type Book record {| - int id; - string title; -|}; +type Book record { + string name; + string author; +} ``` -In this instance, all other elements in the CSV data, such as `author` and `price` along with their attributes, will be transformed into `string` type fields with the corresponding element name as the key. +In this instance, all other CSV header values, such as `year` and `publisher` will be transformed into `anydata-typed` fields with the corresponding CSV header as the key-value pair. -This behavior extends to arrays as well. +This behavior extends to arrays as well. -The process of projecting CSV data into a record supports various use cases, including the filtering out of unnecessary elements. This functionality is anticipated to be enhanced in the future to accommodate more complex scenarios, such as filtering values based on regular expressions, among others. +The process of projecting CSV data into a record supports various use cases, including the filtering out of unnecessary members. This functionality is anticipated to be enhanced in the future to accommodate more complex scenarios, such as filtering values based on regular expressions, among others. ## Issues and projects -Issues and Projects tabs are disabled for this repository as this is part of the Ballerina standard library. To report bugs, request new features, start new discussions, view project boards, etc. please visit Ballerina standard library [parent repository](https://github.com/ballerina-platform/ballerina-standard-library). +Issues and Projects tabs are disabled for this repository as this is part of the Ballerina library. To report bugs, request new features, start new discussions, view project boards, etc. please visit Ballerina library [parent repository](https://github.com/ballerina-platform/ballerina-library). This repository only contains the source code for the package. @@ -410,8 +163,8 @@ This repository only contains the source code for the package. ### Set up the prerequisites 1. Download and install Java SE Development Kit (JDK) version 17 (from one of the following locations). - * [Oracle](https://www.oracle.com/java/technologies/downloads/) - * [OpenJDK](https://adoptium.net/) + * [Oracle](https://www.oracle.com/java/technologies/downloads/) + * [OpenJDK](https://adoptium.net/) 2. Export your GitHub personal access token with the read package permissions as follows. @@ -450,6 +203,5 @@ All contributors are encouraged to read the [Ballerina code of conduct](https:// ## Useful links -[//]: # (* For more information go to the [`csvdata` library](https://lib.ballerina.io/ballerina/data.csv/latest).) * Chat live with us via our [Discord server](https://discord.gg/ballerinalang). -* Post all technical questions on Stack Overflow with the [#ballerina](https://stackoverflow.com/questions/tagged/ballerina) tag. \ No newline at end of file +* Post all technical questions on Stack Overflow with the [#ballerina](https://stackoverflow.com/questions/tagged/ballerina) tag. diff --git a/ballerina-tests/build.gradle b/ballerina-tests/build.gradle new file mode 100644 index 0000000..c83dd9d --- /dev/null +++ b/ballerina-tests/build.gradle @@ -0,0 +1,242 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +plugins { + id 'jacoco' + id 'groovy' +} + +import org.apache.tools.ant.taskdefs.condition.Os + +description = 'Ballerina - CSV data module Ballerina Tests' + +def packageName = "data.csv" +def packageOrg = "ballerina" +def tomlVersion = stripBallerinaExtensionVersion("${project.version}") +def ballerinaTomlFilePlaceHolder = new File("${project.rootDir}/build-config/resources/BallerinaTest.toml") +def testCommonTomlFilePlaceHolder = new File("${project.rootDir}/build-config/resources/CsvTestCommon.toml") +def ballerinaDist = "${project.rootDir}/target/ballerina-runtime" +def distributionBinPath = "${ballerinaDist}/bin" +def testCoverageParam = "--code-coverage --coverage-format=xml --includes=io.ballerina.lib.data.csvdata.*:ballerina.*" +def testPackages = ["union-type-tests", "user-config-tests", "parse-string-record-types-tests", + "parse-string-array-types-tests", "parse-list-types-tests", "parse-record-types-tests", + "type-compatible-tests", "unicode-tests", "constraint-validation-tests"] +def testCommonPackage = "csv-commons" + +def stripBallerinaExtensionVersion(String extVersion) { + if (extVersion.matches(project.ext.timestampedVersionRegex)) { + def splitVersion = extVersion.split('-'); + if (splitVersion.length > 3) { + def strippedValues = splitVersion[0..-4] + return strippedValues.join('-') + } else { + return extVersion + } + } else { + return extVersion.replace("${project.ext.snapshotVersion}", "") + } +} + +clean { + delete "${project.projectDir}/${testCommonPackage}/target" + + testPackages.each { testPackage -> + delete "${project.projectDir}/${testPackage}/target" + } +} + +task updateTomlVersions { + doLast { + testPackages.each { testPackage -> + def ballerinaTomlFile = new File("${project.projectDir}/${testPackage}/Ballerina.toml") + def newBallerinaToml = ballerinaTomlFilePlaceHolder.text.replace("@project.version@", project.version) + newBallerinaToml = newBallerinaToml.replace("@toml.version@", tomlVersion) + newBallerinaToml = newBallerinaToml.replace("@test.common@", testCommonPackage.replaceAll("-", "_")) + newBallerinaToml = newBallerinaToml.replace("@package.name@", testPackage.replaceAll("-", "_")) + ballerinaTomlFile.text = newBallerinaToml + } + + def ballerinaTomlFile = new File("${project.projectDir}/${testCommonPackage}/Ballerina.toml") + def newBallerinaToml = testCommonTomlFilePlaceHolder.text.replace("@project.version@", project.version) + newBallerinaToml = newBallerinaToml.replace("@toml.version@", tomlVersion) + newBallerinaToml = newBallerinaToml.replace("@package.name@", testCommonPackage.replaceAll("-", "_")) + ballerinaTomlFile.text = newBallerinaToml + } +} + +def groupParams = "" +def disableGroups = "" +def windowsDisableGroups = "--disable-groups disabledOnWindows" +def debugParams = "" +def balJavaDebugParam = "" +def testParams = "" +def graalvmFlag = "" +def parallelTestFlag = "" +def skipTests = false + +task deleteDependencyTomlFile { + if (project.hasProperty("deleteDependencies")) { + delete "${project.projectDir}/${testCommonPackage}/Dependencies.toml" + + testPackages.each { testPackage -> + delete "${project.projectDir}/${testPackage}/Dependencies.toml" + } + } +} + +task initializeVariables { + if (project.hasProperty("groups")) { + groupParams = "--groups ${project.findProperty("groups")}" + } + if (project.hasProperty("disable")) { + disableGroups = "--disable-groups ${project.findProperty("disable")}" + } + if (project.hasProperty("debug")) { + debugParams = "--debug ${project.findProperty("debug")}" + } + if (project.hasProperty("balJavaDebug")) { + balJavaDebugParam = "BAL_JAVA_DEBUG=${project.findProperty("balJavaDebug")}" + } + if (project.hasProperty('balGraalVMTest')) { + graalvmFlag = '--graalvm' + } + if (project.hasProperty('balParallelTest')) { + parallelTestFlag = '--parallel' + } + if (project.hasProperty('balTests')) { + testPackages = project.findProperty('balTests').toString().split(",") + } + if (project.hasProperty('skipBalTests')) { + project.findProperty('skipBalTests').toString().split(",").each {testPackage -> + testPackages.remove(testPackage) + } + } + + + gradle.taskGraph.whenReady { graph -> + if (graph.hasTask(":${packageName}-ballerina-tests:test")) { + if (!project.hasProperty('balGraalVMTest')) { + testParams = "${testCoverageParam}" + } + } else { + skipTests = true + } + } +} + +task publishTestCommonPackageToLocal { + dependsOn(":${packageName}-${packageOrg}:build") + dependsOn(updateTomlVersions) + doLast { + if (!skipTests) { + exec { + workingDir "${project.projectDir}/${testCommonPackage}" + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + commandLine 'cmd', '/c', "${distributionBinPath}/bal.bat pack && exit %%ERRORLEVEL%%" + } else { + commandLine 'sh', '-c', "${distributionBinPath}/bal pack" + } + } + exec { + workingDir "${project.projectDir}/${testCommonPackage}" + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + commandLine 'cmd', '/c', "${distributionBinPath}/bal.bat push --repository=local" + + " && exit %%ERRORLEVEL%%" + } else { + commandLine 'sh', '-c', "${distributionBinPath}/bal push --repository=local" + } + } + } + } +} + +task commitTomlFiles { + doLast { + project.exec { + ignoreExitValue true + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + commandLine 'cmd', '/c', "git commit -m \"[Automated] Update the native jar versions\" Ballerina.toml Dependencies.toml" + } else { + commandLine 'sh', '-c', "git commit -m '[Automated] Update the native jar versions' Ballerina.toml Dependencies.toml" + } + } + } +} + +task ballerinaTest { + dependsOn(":${packageName}-${packageOrg}:build") + dependsOn(updateTomlVersions) + dependsOn(initializeVariables) + dependsOn(publishTestCommonPackageToLocal) + finalizedBy(commitTomlFiles) + + doLast { + testPackages.each { testPackage -> + if (!skipTests) { + exec { + workingDir "${project.projectDir}/${testPackage}" + environment "JAVA_OPTS", "-DBALLERINA_DEV_COMPILE_BALLERINA_ORG=true" + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + commandLine 'cmd', '/c', "${balJavaDebugParam} ${distributionBinPath}/bal.bat test ${graalvmFlag}" + + " ${parallelTestFlag} ${testParams} ${groupParams} ${disableGroups} ${windowsDisableGroups}" + + " ${debugParams} && exit %%ERRORLEVEL%%" + } else { + commandLine 'sh', '-c', "${distributionBinPath}/bal test ${graalvmFlag} ${parallelTestFlag} ${testParams}" + + " ${groupParams} ${disableGroups} ${debugParams}" + } + } + if (project.hasProperty('balGraalVMTest')) { + exec { + workingDir "${project.projectDir}/${testPackage}" + environment "JAVA_OPTS", "-DBALLERINA_DEV_COMPILE_BALLERINA_ORG=true" + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + commandLine 'cmd', '/c', "${distributionBinPath}/bal.bat clean" + } else { + commandLine 'sh', '-c', "${distributionBinPath}/bal clean" + } + } + } + } + } + } +} + +publishing { + repositories { + maven { + name = "GitHubPackages" + url = uri("https://maven.pkg.github.com/ballerina-platform/module-${packageOrg}-${packageName}") + credentials { + username = System.getenv("packageUser") + password = System.getenv("packagePAT") + } + } + } +} + +test { + dependsOn(ballerinaTest) +} + +build.dependsOn ":${packageName}-ballerina:build" +build.dependsOn ":${packageName}-compiler-plugin:build" + +test.dependsOn ":${packageName}-ballerina:build" +test.dependsOn ":${packageName}-compiler-plugin:build" +test.dependsOn ":${packageName}-compiler-plugin-tests:test" + +publishToMavenLocal.dependsOn build +publish.dependsOn build diff --git a/ballerina-tests/constraint-validation-tests/Ballerina.toml b/ballerina-tests/constraint-validation-tests/Ballerina.toml new file mode 100644 index 0000000..84093d9 --- /dev/null +++ b/ballerina-tests/constraint-validation-tests/Ballerina.toml @@ -0,0 +1,16 @@ +[package] +org = "ballerina" +name = "constraint_validation_tests" +version = "0.1.0" + +[[dependency]] +org = "ballerina" +name = "csv_commons" +repository = "local" +version = "0.1.0" + +[platform.java17] +graalvmCompatible = true + +[build-options] +graalvmBuildOptions = "-H:+IncludeAllLocales" diff --git a/ballerina-tests/constraint-validation-tests/Dependencies.toml b/ballerina-tests/constraint-validation-tests/Dependencies.toml new file mode 100644 index 0000000..7f54d3a --- /dev/null +++ b/ballerina-tests/constraint-validation-tests/Dependencies.toml @@ -0,0 +1,101 @@ +# AUTO-GENERATED FILE. DO NOT MODIFY. + +# This file is auto-generated by Ballerina for managing dependency versions. +# It should not be modified by hand. + +[ballerina] +dependencies-toml-version = "2" +distribution-version = "2201.9.0" + +[[package]] +org = "ballerina" +name = "constraint" +version = "1.5.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] +modules = [ + {org = "ballerina", packageName = "constraint", moduleName = "constraint"} +] + +[[package]] +org = "ballerina" +name = "constraint_validation_tests" +version = "0.1.0" +dependencies = [ + {org = "ballerina", name = "constraint"}, + {org = "ballerina", name = "data.csv"}, + {org = "ballerina", name = "test"} +] +modules = [ + {org = "ballerina", packageName = "constraint_validation_tests", moduleName = "constraint_validation_tests"} +] + +[[package]] +org = "ballerina" +name = "data.csv" +version = "0.1.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] +modules = [ + {org = "ballerina", packageName = "data.csv", moduleName = "data.csv"} +] + +[[package]] +org = "ballerina" +name = "jballerina.java" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "lang.__internal" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.object"} +] + +[[package]] +org = "ballerina" +name = "lang.array" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.__internal"} +] + +[[package]] +org = "ballerina" +name = "lang.error" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "lang.object" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "test" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.array"}, + {org = "ballerina", name = "lang.error"} +] +modules = [ + {org = "ballerina", packageName = "test", moduleName = "test"} +] + diff --git a/ballerina-tests/constraint-validation-tests/tests/constraint_validation_test.bal b/ballerina-tests/constraint-validation-tests/tests/constraint_validation_test.bal new file mode 100644 index 0000000..577f783 --- /dev/null +++ b/ballerina-tests/constraint-validation-tests/tests/constraint_validation_test.bal @@ -0,0 +1,130 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/constraint; +import ballerina/data.csv; +import ballerina/test; + +type ConstrainedRec record { + @constraint:Int { + minValue: 3, + maxValue: 10 + } + int a?; + @constraint:String { + minLength: 2 + } + string b; +}; + +@constraint:Array {length: 4} +type ConstrainedList int[][]; + +@test:Config +function testConstraintWithLists() returns error? { + ConstrainedList|csv:Error cList1 = csv:parseString(string `1 + 2 + 3 + 4`, {header: (), customHeadersIfHeadersAbsent: ["a", "b", "c", "d"]}); + test:assertEquals(cList1, [[1], [2], [3], [4]]); + + cList1 = csv:parseString(string `1 + 2 + 3 + 4 + 5 + 6`, {header: null, customHeadersIfHeadersAbsent: ["a", "b", "c", "d", "e", "f"]}); + test:assertTrue(cList1 is csv:Error); + test:assertTrue((cList1).message().startsWith("Validation failed") + && (cList1).message().includes("length")); + + cList1 = csv:parseString(string `1 + 2 + 3 + 4 + 5 + 6`, {header: (), customHeadersIfHeadersAbsent: ["a", "b", "c", "d", "e", "f"], enableConstraintValidation: false}); + test:assertEquals(cList1, [[1], [2], [3], [4], [5], [6]]); + + cList1 = csv:transform([{"a": 1}, {"a": 1}, {"a": 1}, {"a": 1}], {}); + test:assertEquals(cList1, [[1], [1], [1], [1]]); + + cList1 = csv:transform([{"a": 1}, {"a": 1}, {"a": 1}, {"a": 1}, {"a": 1}, {"a": 1}], {}); + test:assertTrue(cList1 is csv:Error); + test:assertTrue((cList1).message().startsWith("Validation failed") + && (cList1).message().includes("length")); + + cList1 = csv:parseList([["1"], ["2"], ["3"], ["4"]], {customHeaders: ["a", "b", "c", "d"]}); + test:assertEquals(cList1, [[1], [2], [3], [4]]); + + cList1 = csv:parseList([["1"], ["2"], ["3"], ["4"], ["5"], ["6"]], {customHeaders: ["a", "b", "c", "d"]}); + test:assertTrue(cList1 is csv:Error); + test:assertTrue((cList1).message().startsWith("Validation failed") + && (cList1).message().includes("length")); +} + +@test:Config +function testConstraintWithRecords() returns error? { + ConstrainedRec[]|csv:Error cRec1 = csv:parseString(string `a,b + 4,abc + 3, cde`); + test:assertEquals(cRec1, [{a: 4, b: "abc"}, {a: 3, b: "cde"}]); + + ConstrainedRec[]|csv:Error cRec1_2 = csv:parseString(string `a,b + 4,abc + 3, cde`, {enableConstraintValidation: false}); + test:assertEquals(cRec1_2, [{a: 4, b: "abc"}, {a: 3, b: "cde"}]); + + cRec1 = csv:parseString(string `a,b + 4,abc + 11, cde`); + test:assertTrue(cRec1 is csv:Error); + test:assertTrue((cRec1).message().startsWith("Validation failed") + && (cRec1).message().includes("maxValue")); + + cRec1 = csv:parseString(string `a,b + 4,abc + 5, "b"`, {}); + test:assertTrue(cRec1 is csv:Error); + test:assertTrue((cRec1).message().startsWith("Validation failed") + && (cRec1).message().includes("minLength")); + + cRec1 = csv:transform([{"a": 4, "b": "abc"}, {"a": 3, "b": "cde"}], {}); + test:assertEquals(cRec1, [{a: 4, b: "abc"}, {a: 3, b: "cde"}]); + + cRec1 = csv:transform([{"a": 4, "b": "abc"}, {"a": 11, "b": "cde"}], {}); + test:assertTrue(cRec1 is csv:Error); + test:assertTrue((cRec1).message().startsWith("Validation failed") + && (cRec1).message().includes("maxValue")); + + cRec1 = csv:transform([{"a": 4, "b": "abc"}, {"a": 5, "b": "b"}], {}); + test:assertTrue(cRec1 is csv:Error); + test:assertTrue((cRec1).message().startsWith("Validation failed") + && (cRec1).message().includes("minLength")); + + cRec1 = csv:parseList([["4", "abc"], ["3", "cde"]], {customHeaders: ["a", "b"]}); + test:assertEquals(cRec1, [{a: 4, b: "abc"}, {a: 3, b: "cde"}]); + + cRec1 = csv:parseList([["4", "abc"], ["11", "cde"]], {customHeaders: ["a", "b"]}); + test:assertTrue(cRec1 is csv:Error); + test:assertTrue((cRec1).message().startsWith("Validation failed") + && (cRec1).message().includes("maxValue")); + + cRec1 = csv:parseList([["4", "abc"], ["5", "b"]], {customHeaders: ["a", "b"]}); + test:assertTrue(cRec1 is csv:Error); + test:assertTrue((cRec1).message().startsWith("Validation failed") + && (cRec1).message().includes("minLength")); +} diff --git a/ballerina-tests/csv-commons/Ballerina.toml b/ballerina-tests/csv-commons/Ballerina.toml new file mode 100644 index 0000000..13c0bc8 --- /dev/null +++ b/ballerina-tests/csv-commons/Ballerina.toml @@ -0,0 +1,7 @@ +[package] +org = "ballerina" +name = "csv_commons" +version = "0.1.0" + +[platform.java17] +graalvmCompatible = true diff --git a/ballerina-tests/csv-commons/Dependencies.toml b/ballerina-tests/csv-commons/Dependencies.toml new file mode 100644 index 0000000..cf3bbba --- /dev/null +++ b/ballerina-tests/csv-commons/Dependencies.toml @@ -0,0 +1,17 @@ +# AUTO-GENERATED FILE. DO NOT MODIFY. + +# This file is auto-generated by Ballerina for managing dependency versions. +# It should not be modified by hand. + +[ballerina] +dependencies-toml-version = "2" +distribution-version = "2201.10.0-20240801-104200-87df251c" + +[[package]] +org = "ballerina" +name = "csv_commons" +version = "0.1.0" +modules = [ + {org = "ballerina", packageName = "csv_commons", moduleName = "csv_commons"} +] + diff --git a/ballerina-tests/csv-commons/Package.md b/ballerina-tests/csv-commons/Package.md new file mode 100644 index 0000000..8a7ecb6 --- /dev/null +++ b/ballerina-tests/csv-commons/Package.md @@ -0,0 +1,3 @@ +## Package overview + +This package provides APIs for testing ballerina CSV data module. diff --git a/ballerina-tests/csv-commons/test_utils.bal b/ballerina-tests/csv-commons/test_utils.bal new file mode 100644 index 0000000..2609cd9 --- /dev/null +++ b/ballerina-tests/csv-commons/test_utils.bal @@ -0,0 +1,39 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +public function generateErrorMessageForMissingRequiredField(string 'field) returns string { + return string `no matching header value is found for the required field '${'field}'`; +} + +public function generateErrorMessageForInvalidCast(string value, string 'type) returns string { + return string `value '${value}' cannot be cast into '${'type}'`; +} + +public function generateErrorMessageForInvalidFieldType(string value, string 'key) returns string { + return string `no mapping type found for value '${value}' in key '${'key}'`; +} + +public function generateErrorMessageForInvalidValueForArrayType(string value, string index, string arrayType) returns string { + return string `value '${value}' in index '${index}' is not compatible with array type '${arrayType}'`; +} + +public function generateErrorMessageForInvalidHeaders(string value, string 'type) returns string{ + return string `value '${value}' cannot be cast into '${'type}', because fields in '${'type}' or the provided expected headers are not matching with the '${value}'`; +} + +public function generateErrorMessageForInvalidCustomHeader(string header) returns string { + return string `Header '${header}' cannot be found in data rows`; +} diff --git a/ballerina-tests/parse-list-types-tests/Ballerina.toml b/ballerina-tests/parse-list-types-tests/Ballerina.toml new file mode 100644 index 0000000..258e9ff --- /dev/null +++ b/ballerina-tests/parse-list-types-tests/Ballerina.toml @@ -0,0 +1,16 @@ +[package] +org = "ballerina" +name = "parse_list_types_tests" +version = "0.1.0" + +[[dependency]] +org = "ballerina" +name = "csv_commons" +repository = "local" +version = "0.1.0" + +[platform.java17] +graalvmCompatible = true + +[build-options] +graalvmBuildOptions = "-H:+IncludeAllLocales" diff --git a/ballerina-tests/parse-list-types-tests/Dependencies.toml b/ballerina-tests/parse-list-types-tests/Dependencies.toml new file mode 100644 index 0000000..3e4b472 --- /dev/null +++ b/ballerina-tests/parse-list-types-tests/Dependencies.toml @@ -0,0 +1,98 @@ +# AUTO-GENERATED FILE. DO NOT MODIFY. + +# This file is auto-generated by Ballerina for managing dependency versions. +# It should not be modified by hand. + +[ballerina] +dependencies-toml-version = "2" +distribution-version = "2201.10.0-20240801-104200-87df251c" + +[[package]] +org = "ballerina" +name = "csv_commons" +version = "0.1.0" +scope = "testOnly" +modules = [ + {org = "ballerina", packageName = "csv_commons", moduleName = "csv_commons"} +] + +[[package]] +org = "ballerina" +name = "data.csv" +version = "0.1.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] +modules = [ + {org = "ballerina", packageName = "data.csv", moduleName = "data.csv"} +] + +[[package]] +org = "ballerina" +name = "jballerina.java" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "lang.__internal" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.object"} +] + +[[package]] +org = "ballerina" +name = "lang.array" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.__internal"} +] + +[[package]] +org = "ballerina" +name = "lang.error" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "lang.object" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "parse_list_types_tests" +version = "0.1.0" +dependencies = [ + {org = "ballerina", name = "csv_commons"}, + {org = "ballerina", name = "data.csv"}, + {org = "ballerina", name = "test"} +] +modules = [ + {org = "ballerina", packageName = "parse_list_types_tests", moduleName = "parse_list_types_tests"} +] + +[[package]] +org = "ballerina" +name = "test" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.array"}, + {org = "ballerina", name = "lang.error"} +] +modules = [ + {org = "ballerina", packageName = "test", moduleName = "test"} +] + diff --git a/ballerina-tests/parse-list-types-tests/tests/parse_list_type_as_list_test.bal b/ballerina-tests/parse-list-types-tests/tests/parse_list_type_as_list_test.bal new file mode 100644 index 0000000..a094cdf --- /dev/null +++ b/ballerina-tests/parse-list-types-tests/tests/parse_list_type_as_list_test.bal @@ -0,0 +1,598 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/csv_commons as common; +import ballerina/data.csv; +import ballerina/test; + +@test:Config +function testFromCsvWithTypeForTupleAndTupleAsExpectedType() { + StringTuple1Array|csv:Error st1st1 = csv:parseList([st1, st1], {}, StringTuple1Array); + test:assertEquals(st1st1, [ + [s1, s2, "", ""], + [s1, s2, "", ""] + ]); + + StringTuple1Array|csv:Error st2st1 = csv:parseList([st2, st2], {}, StringTuple1Array); + test:assertEquals(st2st1, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + StringTuple1Array|csv:Error st3st1 = csv:parseList([st3, st3], {}, StringTuple1Array); + test:assertEquals(st3st1, [ + [s1, s2, "", ""], + [s1, s2, "", ""] + ]); + + StringTuple1Array|csv:Error st4st1 = csv:parseList([st4, st4], {}, StringTuple1Array); + test:assertEquals(st4st1, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + StringTuple2Array|csv:Error st1st2 = csv:parseList([st1, st1], {}, StringTuple2Array); + test:assertEquals(st1st2, [ + [s1, s2], + [s1, s2] + ]); + + StringTuple2Array|csv:Error st2st2 = csv:parseList([st2, st2], {}, StringTuple2Array); + test:assertEquals(st2st2, [ + [s1, s2], + [s1, s2] + ]); + + StringTuple2Array|csv:Error st3st2 = csv:parseList([st3, st3], {}, StringTuple2Array); + test:assertEquals(st3st2, [ + [s1, s2], + [s1, s2] + ]); + + StringTuple2Array|csv:Error st4st2 = csv:parseList([st4, st4], {}, StringTuple2Array); + test:assertEquals(st4st2, [ + [s1, s2], + [s1, s2] + ]); + + StringTuple3Array|csv:Error st1st3 = csv:parseList([st1, st1], {}, StringTuple3Array); + test:assertEquals(st1st3, [ + [s1, s2], + [s1, s2] + ]); + + StringTuple3Array|csv:Error st2st3 = csv:parseList([st2, st2], {}, StringTuple3Array); + test:assertEquals(st2st3, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + StringTuple3Array|csv:Error st3st3 = csv:parseList([st3, st3], {}, StringTuple3Array); + test:assertEquals(st3st3, [ + [s1, s2], + [s1, s2] + ]); + + StringTuple3Array|csv:Error st4st3 = csv:parseList([st4, st4], {}, StringTuple3Array); + test:assertEquals(st4st3, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); +} + +@test:Config +function testFromCsvWithTypeForTupleAndTupleAsExpectedType2() { + StringTuple4Array|csv:Error st1st4 = csv:parseList([st1, st1], {}, StringTuple4Array); + test:assertEquals(st1st4, [ + [s1, s2], + [s1, s2] + ]); + + StringTuple4Array|csv:Error st2st4 = csv:parseList([st2, st2], {}, StringTuple4Array); + test:assertEquals(st2st4, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + StringTuple4Array|csv:Error st3st4 = csv:parseList([st3, st3], {}, StringTuple4Array); + test:assertEquals(st3st4, [ + [s1, s2], + [s1, s2] + ]); + + StringTuple4Array|csv:Error st4st4 = csv:parseList([st4, st4], {}, StringTuple4Array); + test:assertEquals(st4st4, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + AnydataTuple3Array|csv:Error st1anydt3 = csv:parseList([st1, st1], {}, AnydataTuple3Array); + test:assertEquals(st1anydt3, [ + [s1, s2], + [s1, s2] + ]); + + AnydataTuple3Array|csv:Error st2anydt3 = csv:parseList([st2, st2], {}, AnydataTuple3Array); + test:assertEquals(st2anydt3, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + AnydataTuple3Array|csv:Error st3anydt3 = csv:parseList([st3, st3], {}, AnydataTuple3Array); + test:assertEquals(st3anydt3, [ + [s1, s2], + [s1, s2] + ]); + + AnydataTuple3Array|csv:Error st4anydt3 = csv:parseList([st4, st4], {}, AnydataTuple3Array); + test:assertEquals(st4anydt3, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + JsonTuple3Array|csv:Error st1jt3 = csv:parseList([st1, st1], {}, JsonTuple3Array); + test:assertEquals(st1jt3, [ + [s1, s2], + [s1, s2] + ]); + + JsonTuple3Array|csv:Error st2jt3 = csv:parseList([st2, st2], {}, JsonTuple3Array); + test:assertEquals(st2jt3, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + JsonTuple3Array|csv:Error st3jt3 = csv:parseList([st3, st3], {}, JsonTuple3Array); + test:assertEquals(st3jt3, [ + [s1, s2], + [s1, s2] + ]); + + JsonTuple3Array|csv:Error st4jt3 = csv:parseList([st4, st4], {}, JsonTuple3Array); + test:assertEquals(st4jt3, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + DecimalTuple4Array|csv:Error st1dta = csv:parseList([st1, st1], {}, DecimalTuple4Array); + test:assertTrue(st1dta is csv:Error); + test:assertEquals((st1dta).message(), common:generateErrorMessageForInvalidValueForArrayType("string", "0", "decimal")); + + IntegerTuple3Array|csv:Error st2bta = csv:parseList([st2, st2], {}, IntegerTuple3Array); + test:assertTrue(st2bta is csv:Error); + test:assertEquals((st2bta).message(), common:generateErrorMessageForInvalidValueForArrayType("string", "0", "int")); + + NilTuple3Array|csv:Error st3nta = csv:parseList([st3, st3], {}, NilTuple3Array); + test:assertTrue(st3nta is csv:Error); + test:assertEquals((st3nta).message(), common:generateErrorMessageForInvalidValueForArrayType("string", "0", "()")); + + BooleanTuple4Array|csv:Error st4bta = csv:parseList([st4, st4], {}, BooleanTuple4Array); + test:assertTrue(st4bta is csv:Error); + test:assertEquals((st4bta).message(), common:generateErrorMessageForInvalidValueForArrayType("string", "0", "boolean")); +} + +@test:Config +function testFromCsvWithTypeForTupleAndTupleAsExpectedType3() { + [string, boolean, int][]|csv:Error ct1bt4 = csv:parseList([["a", "true", "1"], ["a", "true", "1"]], {}); + test:assertEquals(ct1bt4, [ + ["a", true, 1], + ["a", true, 1] + ]); + + [(), float, decimal, boolean, int, string][]|csv:Error ct1bt6 = csv:parseList( + [["null", "2.23", "0", "true", "1", "a"], ["null", "0", "2.23", "true", "1", "a"]]); + test:assertEquals(ct1bt6, [ + [(), 2.23, 0, true, 1, "a"], + [(), 0, 2.23, true, 1, "a"] + ]); + + [decimal, boolean, int, string][]|csv:Error ct1bt7 = csv:parseList( + [["0", "true", "1", "a"], ["2.23", "true", "1", "a"]]); + test:assertEquals(ct1bt7, [ + [0, true, 1, "a"], + [2.23, true, 1, "a"] + ]); + + [decimal, boolean, int, string, anydata...][]|csv:Error ct1bt8 = csv:parseList( + [["0", "true", "1", "a", "null", "2.23"], ["2.23", "true", "1", "a", "null", "0"]]); + test:assertEquals(ct1bt8, [ + [0, true, 1, "a", (), 2.23], + [2.23, true, 1, "a", (), 0] + ]); + + [(), float, decimal, boolean, int, string, string...][]|csv:Error ct1bt9 = csv:parseList( + [["null", "2.23", "0", "true", "1", "a"], ["null", "0", "2.23", "true", "1", "a"]]); + test:assertEquals(ct1bt9, [ + [(), 2.23, 0, true, 1, "a"], + [(), 0, 2.23, true, 1, "a"] + ]); + + [decimal, boolean, int, string, string...][]|csv:Error ct1bt10 = csv:parseList( + [["0", "true", "1", "a", "null", "2.23"], ["2.23", "true", "1", "a", "null", "0"]]); + test:assertEquals(ct1bt10, [ + [0, true, 1, "a", "null", "2.23"], + [2.23, true, 1, "a", "null", "0"] + ]); + + [decimal, boolean, int, string, ()...][]|csv:Error ct1bt11 = csv:parseList( + [["null", "2.23", "0", "true", "1", "a"], ["null", "0", "2.23", "true", "1", "a"]]); + test:assertTrue(ct1bt11 is csv:Error); + //TODO: Fix the message + test:assertEquals((ct1bt11).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "0", "decimal")); + + [(), decimal, float, boolean, ()...][]|csv:Error ct1bt11_2 = csv:parseList( + [["null", "2.23", "0", "true", "1", "a"], ["null", "0", "2.23", "true", "1", "a"]]); + test:assertTrue(ct1bt11_2 is csv:Error); + test:assertEquals((ct1bt11_2).message(), common:generateErrorMessageForInvalidValueForArrayType("1", "4", "()")); + + [()...][]|csv:Error ct1bt12 = csv:parseList( + [["null", "2.23", "0", "true", "1", "a"], ["null", "0", "2.23", "true", "1", "a"]]); + test:assertTrue(ct1bt12 is csv:Error); + test:assertEquals((ct1bt12).message(), common:generateErrorMessageForInvalidValueForArrayType("2.23", "1", "()")); + + [string...][]|csv:Error ct1bt13 = csv:parseList( + [["1", "a"], ["1", "a"]]); + test:assertEquals(ct1bt13, [ + ["1", "a"], + ["1", "a"] + ]); + + [boolean...][]|csv:Error ct1bt14 = csv:parseList( + [["2.23", "null"], ["7", "()"]]); + test:assertTrue(ct1bt14 is csv:Error); + test:assertEquals((ct1bt14).message(), common:generateErrorMessageForInvalidValueForArrayType("2.23", "0", "boolean")); + + int?[][]|csv:Error ct1bt15 = csv:parseList( + [["1", "()"], ["1", "2"]]); + test:assertEquals(ct1bt15, [ + [1, ()], + [1, 2] + ]); + + int[][]|csv:Error ct1bt16 = csv:parseList( + [["1", "2"], ["1", "()"]]); + test:assertTrue(ct1bt16 is csv:Error); + test:assertEquals((ct1bt16).message(), common:generateErrorMessageForInvalidValueForArrayType("()", "1", "int")); + + int[][]|csv:Error ct1bt17 = csv:parseList( + [["a", "b"], ["a", "b"]]); + test:assertTrue(ct1bt17 is csv:Error); + test:assertEquals((ct1bt17).message(), common:generateErrorMessageForInvalidValueForArrayType("a", "0", "int")); +} + +@test:Config +function testFromCsvWithTypeForTupleAndArrayAsExpectedType() { + StringArrayArray|csv:Error st1saa = csv:parseList([st1, st1], {}, StringArrayArray); + test:assertEquals(st1saa, [ + [s1, s2], + [s1, s2] + ]); + + StringArrayArray|csv:Error st2saa = csv:parseList([st2, st2], {}, StringArrayArray); + test:assertEquals(st2saa, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + StringArrayArray|csv:Error st3saa = csv:parseList([st3, st3], {}, StringArrayArray); + test:assertEquals(st3saa, [ + [s1, s2], + [s1, s2] + ]); + + StringArrayArray|csv:Error st4saa = csv:parseList([st4, st4], {}, StringArrayArray); + test:assertEquals(st4saa, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + NillableStringArrayArray|csv:Error st1nsaa = csv:parseList([st1, st1], {}, NillableStringArrayArray); + test:assertEquals(st1nsaa, [ + [s1, s2], + [s1, s2] + ]); + + NillableStringArrayArray|csv:Error st2nsaa = csv:parseList([st2, st2], {}, NillableStringArrayArray); + test:assertEquals(st2nsaa, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + NillableStringArrayArray|csv:Error st3nsaa = csv:parseList([st3, st3], {}, NillableStringArrayArray); + test:assertEquals(st3nsaa, [ + [s1, s2], + [s1, s2] + ]); + + NillableStringArrayArray|csv:Error st4nsaa = csv:parseList([st4, st4], {}, NillableStringArrayArray); + test:assertEquals(st4nsaa, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + NillableIntOrUnionStringArrayArray|csv:Error st1nsuiaa = csv:parseList([st1, st1], {}, NillableIntOrUnionStringArrayArray); + test:assertEquals(st1nsuiaa, [ + [s1, s2], + [s1, s2] + ]); +} + +@test:Config +function testFromCsvWithTypeForTupleAndTupleAsExpectedType4() { + NillableIntOrUnionStringArrayArray|csv:Error st2nsuiaa = csv:parseList([st2, st2], {}, NillableIntOrUnionStringArrayArray); + test:assertEquals(st2nsuiaa, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + NillableIntOrUnionStringArrayArray|csv:Error st3nsuiaa = csv:parseList([st3, st3], {}, NillableIntOrUnionStringArrayArray); + test:assertEquals(st3nsuiaa, [ + [s1, s2], + [s1, s2] + ]); + + NillableIntOrUnionStringArrayArray|csv:Error st4nsuiaa = csv:parseList([st4, st4], {}, NillableIntOrUnionStringArrayArray); + test:assertEquals(st4nsuiaa, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + StringArray2Array|csv:Error st1saa2 = csv:parseList([st1, st1], {}, StringArray2Array); + test:assertEquals(st1saa2, [ + [s1, s2], + [s1, s2] + ]); + + StringArray2Array|csv:Error st2saa2 = csv:parseList([st2, st2], {}, StringArray2Array); + test:assertEquals(st2saa2, [ + [s1, s2], + [s1, s2] + ]); + + StringArray2Array|csv:Error st3saa2 = csv:parseList([st3, st3], {}, StringArray2Array); + test:assertEquals(st3saa2, [ + [s1, s2], + [s1, s2] + ]); + + StringArray2Array|csv:Error st4saa2 = csv:parseList([st4, st4], {}, StringArray2Array); + test:assertEquals(st4saa2, [ + [s1, s2], + [s1, s2] + ]); + + JsonArray1Array|csv:Error st1jaa = csv:parseList([st1, st1], {}, JsonArray1Array); + test:assertEquals(st1jaa, [ + [s1, s2], + [s1, s2] + ]); + + JsonArray1Array|csv:Error st2jaa = csv:parseList([st2, st2], {}, JsonArray1Array); + test:assertEquals(st2jaa, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + JsonArray1Array|csv:Error st3jaa = csv:parseList([st3, st3], {}, JsonArray1Array); + test:assertEquals(st3jaa, [ + [s1, s2], + [s1, s2] + ]); + + JsonArray1Array|csv:Error st4jaa = csv:parseList([st4, st4], {}, JsonArray1Array); + test:assertEquals(st4jaa, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + AnydataArray1Array|csv:Error st1anyda = csv:parseList([st1, st1], {}, AnydataArray1Array); + test:assertEquals(st1anyda, [ + [s1, s2], + [s1, s2] + ]); + + AnydataArray1Array|csv:Error st2anyda = csv:parseList([st2, st2], {}, AnydataArray1Array); + test:assertEquals(st2anyda, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + AnydataArray1Array|csv:Error st3anyda = csv:parseList([st3, st3], {}, AnydataArray1Array); + test:assertEquals(st3anyda, [ + [s1, s2], + [s1, s2] + ]); + + AnydataArray1Array|csv:Error st4anyda = csv:parseList([st4, st4], {}, AnydataArray1Array); + test:assertEquals(st4anyda, [ + [s1, s2, s3, s2], + [s1, s2, s3, s2] + ]); + + DecimalArray1Array|csv:Error st1dda = csv:parseList([st1, st1], {}, DecimalArray1Array); + test:assertTrue(st1dda is csv:Error); + test:assertEquals((st1dda).message(), common:generateErrorMessageForInvalidValueForArrayType("string", "0", "decimal")); + + DecimalArray1Array|csv:Error st3dda = csv:parseList([st3, st3], {}, DecimalArray1Array); + test:assertTrue(st3dda is csv:Error); + test:assertEquals((st3dda).message(), common:generateErrorMessageForInvalidValueForArrayType("string", "0", "decimal")); +} + +@test:Config +function testArrayIndexes() { + string[][] csv = [["1", "2", "3"], ["3", "4", "5"], ["5", "6", "7"], ["7", "8", "9"]]; + + [int, int][2]|csv:Error rec3_2 = csv:parseList(csv); + test:assertEquals(rec3_2, [ + [1, 2], + [3, 4] + ]); + + int[2][]|csv:Error rec5 = csv:parseList(csv); + test:assertEquals(rec5, [ + [1, 2, 3], + [3, 4, 5] + ]); +} + +@test:Config +function testParseListsWithOutputHeaders() { + [string, boolean, int][]|csv:Error ct1bt1 = csv:parseList([["a", "b", "c"], ["a", "true", "1"]], {headerRows: 1}); + test:assertEquals(ct1bt1, [ + ["a", true, 1] + ]); + + [string, boolean, int][]|[anydata...][]|csv:Error ct1bt1_union = csv:parseList([["a", "b", "c"], ["a", "b", "c"]], {headerRows: 2}); + test:assertTrue(ct1bt1_union is csv:Error); + test:assertEquals((ct1bt1_union).message(), "custom headers should be provided"); + + ct1bt1 = csv:parseList([["a", "b", "c"], ["a", "true", "1"]], {headerRows: 1, customHeaders: ["h1", "h2", "h3"]}); + test:assertEquals(ct1bt1, [ + ["a", true, 1] + ]); + + ct1bt1 = csv:parseList([["a", "true", "1"], ["a", "true", "1"]], {headerRows: 0, customHeaders: ["h1", "h2", "h3"]}); + test:assertEquals(ct1bt1, [ + ["a", true, 1], + ["a", true, 1] + ]); + + ct1bt1 = csv:parseList([["a", "true", "1"]], {headerRows: 1}); + test:assertEquals(ct1bt1, []); + + (boolean|int|string)[][]|csv:Error ct1bt1_2 = csv:parseList([["a", "b", "c"], ["a", "true", "1"]], {headerRows: 1}); + test:assertEquals(ct1bt1_2, [ + ["a", true, 1] + ]); + + ct1bt1_2 = csv:parseList([["a", "b", "c"], ["a", "true", "1"]], {headerRows: 1, customHeaders: ["h1", "h2", "h3"]}); + test:assertEquals(ct1bt1_2, [ + ["a", true, 1] + ]); + + ct1bt1_2 = csv:parseList([["a", "true", "1"], ["a", "true", "1"]], {headerRows: 0, customHeaders: ["h1", "h2", "h3"]}); + test:assertEquals(ct1bt1_2, [ + ["a", true, 1], + ["a", true, 1] + ]); + + [string, boolean, int][]|csv:Error ct1bt1_3 = csv:parseList([["a", "true", "1"], ["a", "true", "1"]], {headerRows: 1, outputWithHeaders: true}); + test:assertEquals(ct1bt1_3, [ + ["a", true, 1], + ["a", true, 1] + ]); + + ct1bt1_3 = csv:parseList([["a", "true", "1"], ["a", "true", "1"]], {headerRows: 1, customHeaders: ["h1", "h2", "h3"], outputWithHeaders: true}); + test:assertTrue(ct1bt1_3 is csv:Error); + test:assertEquals((ct1bt1_3).message(), common:generateErrorMessageForInvalidValueForArrayType("h2", "1", "boolean")); + + ct1bt1_3 = csv:parseList([["a", "true", "1"], ["a", "true", "1"]], {headerRows: 0, customHeaders: ["h1", "h2", "h3"], outputWithHeaders: true}); + test:assertTrue(ct1bt1_3 is csv:Error); + test:assertEquals((ct1bt1_3).message(), common:generateErrorMessageForInvalidValueForArrayType("h2", "1", "boolean")); + + [string, boolean, int|string][]|[anydata...][]|csv:Error ct1bt1_3_with_union = csv:parseList([["a", "true", "1"], ["a", "true", "1"]], {headerRows: 21, outputWithHeaders: true}); + test:assertTrue(ct1bt1_3_with_union is csv:Error); + test:assertEquals((ct1bt1_3_with_union).message(), "custom headers should be provided"); + + string[][]|csv:Error ct1bt1_4 = csv:parseList([["a", "b", "c"], ["a", "true", "1"]], {headerRows: 1, outputWithHeaders: true}); + test:assertEquals(ct1bt1_4, [ + ["a", "b", "c"], + ["a", "true", "1"] + ]); + + ct1bt1_4 = csv:parseList([["a", "true", "1"], ["a", "true", "1"]], {headerRows: 0, customHeaders: ["h1", "h2", "h3"], outputWithHeaders: true}); + test:assertEquals(ct1bt1_4, [ + ["h1", "h2", "h3"], + ["a", "true", "1"], + ["a", "true", "1"] + ]); + + (int|boolean|string)[][]|csv:Error ct1bt1_4_2 = csv:parseList([["a", "b", "c"], ["a", "true", "1"]], {headerRows: 1, outputWithHeaders: true}); + test:assertEquals(ct1bt1_4_2, [ + ["a", "b", "c"], + ["a", true, 1] + ]); + + string[][]|csv:Error ct1bt1_4_3 = csv:parseList([["a", "b", "c"], ["a", "true", "1"], ["a", "true", "1"], ["a", "true", "1"]], {headerRows: 2, customHeaders: ["h1", "h2", "h3"], outputWithHeaders: true}); + test:assertEquals(ct1bt1_4_3, [ + ["h1", "h2", "h3"], + ["a", "true", "1"], + ["a", "true", "1"] + ]); + + (int|boolean|string)[][]|csv:Error ct1bt1_4_4 = csv:parseList([["a", "b", "c"], ["a", "true", "1"]], {headerRows: 2, customHeaders: ["h1", "h2", "h3"], outputWithHeaders: true}); + test:assertEquals(ct1bt1_4_4, [ + ["h1", "h2", "h3"] + ]); + + [string, boolean|string, int|string][]|csv:Error ct1bt1_5 = csv:parseList([["a", "true", "1"], ["a", "true", "1"]], {headerRows: 1, customHeaders: ["h1", "h2", "h3"], outputWithHeaders: true}); + test:assertEquals(ct1bt1_5, [ + ["h1", "h2", "h3"], + ["a", true, 1] + ]); + + ct1bt1_5 = csv:parseList([["a", "true", "2"], ["a", "true", "1"]], {headerRows: 1, customHeaders: ["h1", "h2", "h3"], outputWithHeaders: true}); + test:assertEquals(ct1bt1_5, [ + ["h1", "h2", "h3"], + ["a", true, 1] + ]); + + ct1bt1_5 = csv:parseList([["a", "true", "1"], ["a", "true", "1"]], {headerRows: 0, customHeaders: ["h1", "h2", "h3"], outputWithHeaders: true}); + test:assertEquals(ct1bt1_5, [ + ["h1", "h2", "h3"], + ["a", true, 1], + ["a", true, 1] + ]); + + [string...][]|csv:Error ct1bt2 = csv:parseList([["1", "a"], ["1", "a"]]); + test:assertEquals(ct1bt2, [ + ["1", "a"], + ["1", "a"] + ]); + + [string, boolean|string, int|string...][]|csv:Error ct1bt2_2 = csv:parseList([["a", "true", "1"], ["a", "true", "1"]], {headerRows: 1, customHeaders: ["h1", "h2", "h3"], outputWithHeaders: true}); + test:assertEquals(ct1bt2_2, [ + ["h1", "h2", "h3"], + ["a", true, 1] + ]); + + [string...][]|csv:Error ct1bt2_3 = csv:parseList([["a", "true", "2"], ["a", "true", "1"]], {headerRows: 1, customHeaders: ["h1", "h2", "h3"], outputWithHeaders: true}); + test:assertEquals(ct1bt2_3, [ + ["h1", "h2", "h3"], + ["a", "true", "1"] + ]); + + ct1bt2_2 = csv:parseList([["a", "true", "1"], ["a", "true", "1"]], {headerRows: 0, customHeaders: ["h1", "h2", "h3"], outputWithHeaders: true}); + test:assertEquals(ct1bt2_2, [ + ["h1", "h2", "h3"], + ["a", true, 1], + ["a", true, 1] + ]); + + string[2][1]|csv:Error ct1bt6 = csv:parseList([["a", "b", "c"], ["a", "true", "1"], ["a", "true", "1"], ["a", "true", "1"]], {headerRows: 2, customHeaders: ["h1", "h2", "h3"], outputWithHeaders: true}); + test:assertEquals(ct1bt6, [ + ["h1"], + ["a"] + ]); + [string, boolean|string, int|string...][1]|csv:Error ct1bt6_2 = csv:parseList([["a", "true", "1"], ["a", "true", "1"]], {headerRows: 1, customHeaders: ["h1", "h2", "h3"], outputWithHeaders: true}); + test:assertEquals(ct1bt6_2, [ + ["h1", "h2", "h3"] + ]); + [string...][1]|csv:Error ct1bt6_3 = csv:parseList([["a", "true", "2"], ["a", "true", "1"]], {headerRows: 1, customHeaders: ["h1", "h2", "h3"], outputWithHeaders: true}); + test:assertEquals(ct1bt6_3, [ + ["h1", "h2", "h3"] + ]); +} diff --git a/ballerina-tests/parse-list-types-tests/tests/parse_list_type_as_record_test.bal b/ballerina-tests/parse-list-types-tests/tests/parse_list_type_as_record_test.bal new file mode 100644 index 0000000..79162d7 --- /dev/null +++ b/ballerina-tests/parse-list-types-tests/tests/parse_list_type_as_record_test.bal @@ -0,0 +1,931 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/csv_commons as common; +import ballerina/data.csv; +import ballerina/test; + +@test:Config +function testFromCsvWithTypeForTupleAndRecordAsExpectedType() { + StringRecord1Array|csv:Error st1sr1 = csv:parseList([st1, st1], {}); + test:assertTrue(st1sr1 is csv:Error); + test:assertEquals((st1sr1).message(), common:generateErrorMessageForMissingRequiredField("s3")); + + StringRecord1Array|csv:Error st2sr1 = csv:parseList([st2, st2], {}); + test:assertTrue(st2sr1 is csv:Error); + test:assertEquals((st2sr1).message(), common:generateErrorMessageForMissingRequiredField("s3")); + + StringRecord2Array|csv:Error st1sr2 = csv:parseList([st1, st1], {}); + test:assertTrue(st1sr2 is csv:Error); + test:assertEquals((st1sr2).message(), common:generateErrorMessageForInvalidHeaders("[\"string\",\"\"]", "parse_list_types_tests:StringRecord2")); + + StringRecord2Array|csv:Error st2sr2 = csv:parseList([st2, st2], {}); + test:assertTrue(st2sr2 is csv:Error); + test:assertEquals((st2sr2).message(), common:generateErrorMessageForInvalidHeaders("[\"string\",\"\",\"a\",\"\"]","parse_list_types_tests:StringRecord2")); + + StringRecord9Array|csv:Error st1sr9 = csv:parseList([st1, st1], {}); + test:assertTrue(st1sr9 is csv:Error); + test:assertEquals((st1sr9).message(), common:generateErrorMessageForInvalidHeaders("[\"string\",\"\"]", "parse_list_types_tests:StringRecord9")); + + StringRecord9Array|csv:Error st2sr9 = csv:parseList([st2, st2], {}); + test:assertTrue(st2sr9 is csv:Error); + test:assertEquals((st2sr9).message(), common:generateErrorMessageForInvalidHeaders("[\"string\",\"\",\"a\",\"\"]","parse_list_types_tests:StringRecord9")); + + StringRecord10Array|csv:Error st1sr10 = csv:parseList([st1, st1], {}); + test:assertEquals(st1sr10, [ + {'1: "string", '2: ""}, + {'1: "string", '2: ""} + ]); + + StringRecord10Array|csv:Error st2sr10 = csv:parseList([st2, st2], {}); + test:assertEquals(st2sr10, [ + {'1: "string", '2: "", '3: "a", '4: ""}, + {'1: "string", '2: "", '3: "a", '4: ""} + ]); + + StringRecord19Array|csv:Error st1sr19 = csv:parseList([st1, st1], {}); + test:assertEquals(st1sr19, [ + {s1: "", s2: "", "1": s1, "2": s2}, + {s1: "", s2: "", "1": s1, "2": s2} + ]); + + StringRecord19Array|csv:Error st2sr19 = csv:parseList([st2, st2], {}); + test:assertEquals(st2sr19, [ + {s1: "", s2: "", '1: s1, '2: s2, '3: s3, '4: s2}, + {s1: "", s2: "", '1: s1, '2: s2, '3: s3, '4: s2} + ]); + + StringRecord20Array|csv:Error st1sr20 = csv:parseList([st1, st1], {}); + test:assertEquals(st1sr20, [ + {s1: "", s2: ""}, + {s1: "", s2: ""} + ]); +} + +@test:Config +function testFromCsvWithTypeForTupleAndRecordAsExpectedType2() { + StringRecord20Array|csv:Error st2sr20 = csv:parseList([st2, st2], {}); + test:assertEquals(st2sr20, [ + {s1: "", s2: ""}, + {s1: "", s2: ""} + ]); + + StringRecord21Array|csv:Error st1sr21 = csv:parseList([st1, st1], {}); + test:assertEquals(st1sr21, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); + + StringRecord21Array|csv:Error st2sr21 = csv:parseList([st2, st2], {}); + test:assertEquals(st2sr21, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + StringRecord22Array|csv:Error st1sr22 = csv:parseList([st1, st1], {}); + test:assertEquals(st1sr22, [ + {s1: "", s2: "", "1": s1, "2": s2}, + {s1: "", s2: "", "1": s1, "2": s2} + ]); + + StringRecord22Array|csv:Error st2sr22 = csv:parseList([st2, st2], {}); + test:assertEquals(st2sr22, [ + {s1: "", s2: "", '1: s1, '2: s2, '3: s3, '4: s2}, + {s1: "", s2: "", '1: s1, '2: s2, '3: s3, '4: s2} + ]); + + StringRecord23Array|csv:Error st1sr23 = csv:parseList([st1, st1], {}); + test:assertEquals(st1sr23, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); + + StringRecord23Array|csv:Error st2sr23 = csv:parseList([st2, st2], {}); + test:assertEquals(st2sr23, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomRecord15Array|csv:Error st1cr15 = csv:parseList([st1, st1], {}); + test:assertEquals(st1cr15, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); + + CustomRecord15Array|csv:Error st2cr15 = csv:parseList([st2, st2], {}); + test:assertEquals(st2cr15, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); + + CustomRecord16Array|csv:Error st1cr16 = csv:parseList([st1, st1], {}); + test:assertTrue(st1cr16 is csv:Error); + test:assertEquals((st1cr16).message(), common:generateErrorMessageForMissingRequiredField("3")); + + CustomRecord16Array|csv:Error st2cr16 = csv:parseList([st2, st2], {}); + test:assertEquals(st2cr16, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomRecord17Array|csv:Error st1cr17 = csv:parseList([st1, st1], {}); + test:assertEquals(st1cr17, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); +} + +@test:Config +function testFromCsvWithTypeForTupleAndRecordAsExpectedType3() { + CustomRecord17Array|csv:Error st2cr17 = csv:parseList([st2, st2], {}); + test:assertEquals(st2cr17, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomRecord18Array|csv:Error st1cr18 = csv:parseList([st1, st1], {}); + test:assertTrue(st1cr18 is csv:Error); + test:assertEquals((st1cr18).message(), common:generateErrorMessageForMissingRequiredField("3")); + + CustomRecord18Array|csv:Error st2cr18 = csv:parseList([st2, st2], {}); + test:assertEquals(st2cr18, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomRecord19Array|csv:Error st1cr19 = csv:parseList([st1, st1], {}); + test:assertEquals(st1cr19, [ + {'1: s1, '2: s2, '3: "", '4: ""}, + {'1: s1, '2: s2, '3: "", '4: ""} + ]); + + CustomRecord19Array|csv:Error st2cr19 = csv:parseList([st2, st2], {}); + test:assertEquals(st2cr19, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomRecord20Array|csv:Error st1cr20 = csv:parseList([st1, st1], {}); + test:assertEquals(st1cr20, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); + + CustomRecord20Array|csv:Error st2cr20 = csv:parseList([st2, st2], {}); + test:assertEquals(st2cr20, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomRecord21Array|csv:Error st1cr21 = csv:parseList([st1, st1], {}); + test:assertEquals(st1cr21, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); + + CustomRecord21Array|csv:Error st2cr21 = csv:parseList([st2, st2], {}); + test:assertEquals(st2cr21, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomRecord22Array|csv:Error st1cr22 = csv:parseList([st1, st1], {}); + test:assertEquals(st1cr22, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); +} + +@test:Config +function testFromCsvWithTypeForTupleAndRecordAsExpectedType4() { + CustomRecord22Array|csv:Error st2cr22 = csv:parseList([st2, st2], {}); + test:assertEquals(st2cr22, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomRecord23Array|csv:Error st1cr23 = csv:parseList([st1, st1], {}); + test:assertEquals(st1cr23, [ + {"1": s1, "2": s2, a: ""}, + {"1": s1, "2": s2, a: ""} + ]); + + CustomRecord23Array|csv:Error st2cr23 = csv:parseList([st2, st2], {}); + test:assertEquals(st2cr23, [ + {'1: s1, '2: s2, '3: s3, '4: s2, a: ""}, + {'1: s1, '2: s2, '3: s3, '4: s2, a: ""} + ]); + + CustomRecord24Array|csv:Error st1cr24 = csv:parseList([st1, st1], {}); + test:assertEquals(st1cr24, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); + + CustomRecord24Array|csv:Error st2cr24 = csv:parseList([st2, st2], {}); + test:assertEquals(st2cr24, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomRecord25Array|csv:Error st1cr25 = csv:parseList([st1, st1], {}); + test:assertTrue(st1cr25 is csv:Error); + test:assertEquals((st1cr25).message(), common:generateErrorMessageForInvalidFieldType("string", "1")); + + CustomRecord25Array|csv:Error st2cr25 = csv:parseList([st2, st2], {}); + test:assertTrue(st2cr25 is csv:Error); + test:assertEquals((st2cr25).message(), common:generateErrorMessageForInvalidFieldType("string", "1")); + + CustomRecord25Array|csv:Error st3cr25 = csv:parseList([st3, st3], {}); + test:assertTrue(st3cr25 is csv:Error); + test:assertEquals((st3cr25).message(), common:generateErrorMessageForInvalidFieldType("string", "1")); + + CustomRecord25Array|csv:Error st4cr25 = csv:parseList([st4, st4], {}); + test:assertTrue(st4cr25 is csv:Error); + test:assertEquals((st4cr25).message(), common:generateErrorMessageForInvalidFieldType("string", "1")); + + CustomRecord26Array|csv:Error st1cr26 = csv:parseList([st1, st1], {}); + test:assertEquals(st1cr26 , [ + {'1: s1}, + {'1: s1} + ]); + + CustomRecord26Array|csv:Error st2cr26 = csv:parseList([st2, st2], {}); + test:assertEquals(st2cr26 , [ + {'1: s1}, + {'1: s1} + ]); + + CustomRecord26Array|csv:Error st3cr26 = csv:parseList([st3, st3], {}); + test:assertEquals(st3cr26 , [ + {'1: s1}, + {'1: s1} + ]); + + CustomRecord26Array|csv:Error st4cr26 = csv:parseList([st4, st4], {}); + test:assertEquals(st4cr26 , [ + {'1: s1}, + {'1: s1} + ]); +} + +@test:Config +function testFromCsvWithTypeForTupleAndRecordAsExpectedType5() { + StringRecord1Array|csv:Error st3sr1 = csv:parseList([st3, st3], {}); + test:assertTrue(st3sr1 is csv:Error); + test:assertEquals((st3sr1).message(), common:generateErrorMessageForMissingRequiredField("s3")); + + StringRecord1Array|csv:Error st4sr1 = csv:parseList([st4, st4], {}); + test:assertTrue(st4sr1 is csv:Error); + test:assertEquals((st4sr1).message(), common:generateErrorMessageForMissingRequiredField("s3")); + + StringRecord2Array|csv:Error st3sr2 = csv:parseList([st3, st3], {}); + test:assertTrue(st3sr2 is csv:Error); + test:assertEquals((st3sr2).message(), common:generateErrorMessageForInvalidHeaders("[\"string\",\"\"]", "parse_list_types_tests:StringRecord2")); + + StringRecord2Array|csv:Error st4sr2 = csv:parseList([st4, st4], {}); + test:assertTrue(st4sr2 is csv:Error); + test:assertEquals((st4sr2).message(), common:generateErrorMessageForInvalidHeaders("[\"string\",\"\",\"a\",\"\"]","parse_list_types_tests:StringRecord2")); + + StringRecord9Array|csv:Error st3sr9 = csv:parseList([st3, st3], {}); + test:assertTrue(st3sr9 is csv:Error); + test:assertEquals((st3sr9).message(), common:generateErrorMessageForMissingRequiredField("s1")); + + StringRecord9Array|csv:Error st4sr9 = csv:parseList([st4, st4], {}); + test:assertTrue(st4sr9 is csv:Error); + test:assertEquals((st4sr9).message(), common:generateErrorMessageForMissingRequiredField("s1")); + + StringRecord10Array|csv:Error st3sr10 = csv:parseList([st3, st3], {}); + test:assertEquals(st3sr10, [ + {'1: "string", '2: ""}, + {'1: "string", '2: ""} + ]); + + StringRecord10Array|csv:Error st4sr10 = csv:parseList([st4, st4], {}); + test:assertEquals(st4sr10, [ + {'1: "string", '2: "", '3: "a", '4: ""}, + {'1: "string", '2: "", '3: "a", '4: ""} + ]); + + StringRecord19Array|csv:Error st3sr19 = csv:parseList([st3, st3], {}); + test:assertEquals(st3sr19, [ + {s1: "", s2: "", "1": s1, "2": s2}, + {s1: "", s2: "", "1": s1, "2": s2} + ]); + + StringRecord19Array|csv:Error st4sr19 = csv:parseList([st4, st4], {}); + test:assertEquals(st4sr19, [ + {s1: "", s2: "", '1: s1, '2: s2, '3: s3, '4: s2}, + {s1: "", s2: "", '1: s1, '2: s2, '3: s3, '4: s2} + ]); + + StringRecord20Array|csv:Error st3sr20 = csv:parseList([st3, st3], {}); + test:assertEquals(st3sr20, [ + {s1: "", s2: ""}, + {s1: "", s2: ""} + ]); + + StringRecord20Array|csv:Error st4sr20 = csv:parseList([st4, st4], {}); + test:assertEquals(st4sr20, [ + {s1: "", s2: ""}, + {s1: "", s2: ""} + ]); + + StringRecord21Array|csv:Error st3sr21 = csv:parseList([st3, st3], {}); + test:assertEquals(st3sr21, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); + + StringRecord21Array|csv:Error st4sr21 = csv:parseList([st4, st4], {}); + test:assertEquals(st4sr21, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + StringRecord22Array|csv:Error st3sr22 = csv:parseList([st3, st3], {}); + test:assertEquals(st3sr22, [ + {s1: "", s2: "", "1": s1, "2": s2}, + {s1: "", s2: "", "1": s1, "2": s2} + ]); + + StringRecord22Array|csv:Error st4sr22 = csv:parseList([st4, st4], {}); + test:assertEquals(st4sr22, [ + {s1: "", s2: "", '1: s1, '2: s2, '3: s3, '4: s2}, + {s1: "", s2: "", '1: s1, '2: s2, '3: s3, '4: s2} + ]); + + StringRecord23Array|csv:Error st3sr23 = csv:parseList([st3, st3], {}); + test:assertEquals(st3sr23, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); + + StringRecord23Array|csv:Error st4sr23 = csv:parseList([st4, st4], {}); + test:assertEquals(st4sr23, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomRecord15Array|csv:Error st3cr15 = csv:parseList([st3, st3], {}); + test:assertEquals(st3cr15, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); +} + +@test:Config +function testFromCsvWithTypeForTupleAndRecordAsExpectedType6() { + CustomRecord15Array|csv:Error st4cr15 = csv:parseList([st4, st4], {}); + test:assertEquals(st4cr15, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); + + CustomRecord16Array|csv:Error st3cr16 = csv:parseList([st3, st3], {}); + test:assertTrue(st3cr16 is csv:Error); + test:assertEquals((st3cr16).message(), common:generateErrorMessageForMissingRequiredField("3")); + + CustomRecord16Array|csv:Error st4cr16 = csv:parseList([st4, st4], {}); + test:assertEquals(st4cr16, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomRecord17Array|csv:Error st3cr17 = csv:parseList([st3, st3], {}); + test:assertEquals(st3cr17, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); + + CustomRecord17Array|csv:Error st4cr17 = csv:parseList([st4, st4], {}); + test:assertEquals(st4cr17, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomRecord18Array|csv:Error st3cr18 = csv:parseList([st3, st3], {}); + test:assertTrue(st3cr18 is csv:Error); + test:assertEquals((st3cr18).message(), common:generateErrorMessageForMissingRequiredField("3")); + + CustomRecord18Array|csv:Error st4cr18 = csv:parseList([st4, st4], {}); + test:assertEquals(st4cr18, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomRecord19Array|csv:Error st3cr19 = csv:parseList([st3, st3], {}); + test:assertEquals(st3cr19, [ + {'1: s1, '2: s2, '3: "", '4: ""}, + {'1: s1, '2: s2, '3: "", '4: ""} + ]); + + CustomRecord19Array|csv:Error st4cr19 = csv:parseList([st4, st4], {}); + test:assertEquals(st4cr19, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomRecord20Array|csv:Error st3cr20 = csv:parseList([st3, st3], {}); + test:assertEquals(st3cr20, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); + + CustomRecord20Array|csv:Error st4cr20 = csv:parseList([st4, st4], {}); + test:assertEquals(st4cr20, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomRecord21Array|csv:Error st3cr21 = csv:parseList([st3, st3], {}); + test:assertEquals(st3cr21, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); + + CustomRecord21Array|csv:Error st4cr21 = csv:parseList([st4, st4], {}); + test:assertEquals(st4cr21, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomRecord22Array|csv:Error st3cr22 = csv:parseList([st3, st3], {}); + test:assertEquals(st3cr22, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); + + CustomRecord22Array|csv:Error st4cr22 = csv:parseList([st4, st4], {}); + test:assertEquals(st4cr22, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomRecord23Array|csv:Error st3cr23 = csv:parseList([st3, st3], {}); + test:assertEquals(st3cr23, [ + {"1": s1, "2": s2, a: ""}, + {"1": s1, "2": s2, a: ""} + ]); + + CustomRecord23Array|csv:Error st4cr23 = csv:parseList([st4, st4], {}); + test:assertEquals(st4cr23, [ + {'1: s1, '2: s2, '3: s3, '4: s2, a: ""}, + {'1: s1, '2: s2, '3: s3, '4: s2, a: ""} + ]); + + CustomRecord24Array|csv:Error st3cr24 = csv:parseList([st3, st3], {}); + test:assertEquals(st3cr24, [ + {"1": s1, "2": s2}, + {"1": s1, "2": s2} + ]); + + CustomRecord24Array|csv:Error st4cr24 = csv:parseList([st4, st4], {}); + test:assertEquals(st4cr24, [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); +} + +@test:Config +function testFromCsvWithTypeForTupleAndRecordAsExpectedType7() { + record {string a; boolean b; int c;}[]|csv:Error ct1br4 = csv:parseList([["a", "1", "true"], ["a", "1", "true"]], {customHeaders: ["a", "c", "b"]}); + test:assertEquals(ct1br4, [ + {a: "a", b: true, c: 1}, + {a: "a", b: true, c: 1} + ]); + + record {() a; float b; decimal c; boolean d; int e; string f;}[]|csv:Error ct1br6 = csv:parseList( + [["a", "1", "true", "0", "2.23", "null"], ["a", "1", "true", "2.23", "0", "()"]], + {customHeaders: ["f", "e", "d", "c", "b", "a"]}); + test:assertEquals(ct1br6, [ + {a: (), b: 2.23, c: 0, d: true, e: 1, f: "a"}, + {a: (), b: 0, c: 2.23, d: true, e: 1, f: "a"} + ]); + + record {|decimal c; boolean d; int e; string f;|}[]|csv:Error ct1br7 = csv:parseList( + [["a", "1", "true", "0", "2.23", "null"], ["a", "1", "true", "2.23", "0", "()"]], + {customHeaders: ["f", "e", "d", "c", "b", "a"]}); + test:assertEquals(ct1br7, [ + {c: 0, d: true, e: 1, f: "a"}, + {c: 2.23, d: true, e: 1, f: "a"} + ]); + + record {decimal c; boolean d; int e; string f;}[]|csv:Error ct1br8 = csv:parseList( + [["a", "1", "true", "0", "2.23", "null"], ["a", "1", "true", "2.23", "0", "()"]], + {customHeaders: ["f", "e", "d", "c", "b", "a"]}); + test:assertEquals(ct1br8, [ + {a: (), b: 2.23, c: 0, d: true, e: 1, f: "a"}, + {a: (), b: 0, c: 2.23, d: true, e: 1, f: "a"} + ]); + + record {|int|() a; float b; decimal? c; boolean d; int e; string f; string...;|}[]|csv:Error ct1br9 = csv:parseList( + [["a", "1", "true", "0", "2.23", "null"], ["a", "1", "true", "2.23", "0", "()"]], + {customHeaders: ["f", "e", "d", "c", "b", "a"]}); + test:assertEquals(ct1br9, [ + {a: (), b: 2.23, c: 0, d: true, e: 1, f: "a"}, + {a: (), b: 0, c: 2.23, d: true, e: 1, f: "a"} + ]); + + record {|int|() a; float b; decimal? c; string|boolean d; int|string e; string f; string...;|}[]|csv:Error ct1br9_2 = csv:parseList( + [["a", "1", "true", "0", "2.23", "null"], ["a", "1", "true", "2.23", "0", "()"]], + {customHeaders: ["f", "e", "d", "c", "b", "a"]}); + test:assertEquals(ct1br9_2, [ + {a: (), b: 2.23, c: 0, d: "true", e: 1, f: "a"}, + {a: (), b: 0, c: 2.23, d: "true", e: 1, f: "a"} + ]); + + record {|int|() a; float b; decimal? c; boolean|string d; int|string e; string f; string...;|}[]|csv:Error ct1br9_3 = csv:parseList( + [["a", "1", "true", "0", "2.23", "null"], ["a", "1", "true", "2.23", "0", "()"]], + {customHeaders: ["f", "e", "d", "c", "b", "a"]}); + test:assertEquals(ct1br9_3, [ + {a: (), b: 2.23, c: 0, d: true, e: 1, f: "a"}, + {a: (), b: 0, c: 2.23, d: true, e: 1, f: "a"} + ]); +} + +@test:Config +function testFromCsvWithTypeForTupleAndRecordAsExpectedTypeWithHeaders() { + record {string a; boolean b; int c;}[]|csv:Error ct1br4 = csv:parseList([["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"]], {headerRows: 0, customHeaders: ["a", "c", "b"]}); + test:assertEquals(ct1br4, [ + {a: "a", b: true, c: 1}, + {a: "a", b: true, c: 1}, + {a: "a", b: true, c: 1} + ]); + + record {string a; boolean b; int c;}[]|csv:Error ct1br4_2 = csv:parseList([["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"]], {headerRows: 1, customHeaders: ["a", "c", "b"]}); + test:assertEquals(ct1br4_2, [ + {a: "a", b: true, c: 1}, + {a: "a", b: true, c: 1} + ]); + + record {string a; boolean b; int c;}[]|csv:Error ct1br4_3 = csv:parseList([["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"]], {headerRows: 1, customHeaders: ["a", "c", "b"]}); + test:assertEquals(ct1br4_3, [ + {a: "a", b: true, c: 1}, + {a: "a", b: true, c: 1} + ]); + + + record {string a; boolean b; int c;}[]|csv:Error ct1br4_4 = csv:parseList( + [["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"]], {headerRows: 2, customHeaders: ["a", "c", "b"]}); + test:assertEquals(ct1br4_4, [ + {a: "a", b: true, c: 1} + ]); + + record {string a; boolean b; int c;}[]|record {}[]|csv:Error ct1br4_5 = csv:parseList( + [["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"]], {headerRows: 2}); + test:assertTrue(ct1br4_5 is csv:Error); + test:assertEquals((ct1br4_5).message(), "custom headers should be provided"); + + record {|string a; boolean b; int...;|}[]|csv:Error ct1br4_4_2 = csv:parseList( + [["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"]], {headerRows: 2, customHeaders: ["a", "c", "b"]}); + test:assertEquals(ct1br4_4_2, [ + {a: "a", b: true, c: 1} + ]); + + record {|string a; boolean b; int...;|}[]|record {}[]|csv:Error ct1br4_5_2 = csv:parseList( + [["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"]], {headerRows: 2}); + test:assertTrue(ct1br4_5_2 is csv:Error); + test:assertEquals((ct1br4_5).message(), "custom headers should be provided"); + + map[]|csv:Error ct2br4_3 = csv:parseList([["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"]], {headerRows: 1, customHeaders: ["a", "c", "b"]}); + test:assertEquals(ct2br4_3, [ + {a: "a", b: true, c: 1}, + {a: "a", b: true, c: 1} + ]); + + map[]|csv:Error ct2br4_3_2 = csv:parseList([["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"]], {headerRows: 3, customHeaders: ["a", "c", "b"]}); + test:assertEquals(ct2br4_3_2, []); + + map[]|csv:Error ct2br4_4 = csv:parseList( + [["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"]], {headerRows: 2, customHeaders: ["a", "c", "b"]}); + test:assertEquals(ct2br4_4, [ + {a: "a", b: true, c: 1} + ]); + + map[]|csv:Error ct2br4_5 = csv:parseList( + [["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"]], {headerRows: 2, customHeaders: ["a", "c", "b"]}); + test:assertEquals(ct2br4_5, [ + {a: "a", b: "true", c: "1"}, + {a: "a", b: "true", c: "1"} + ]); + + map[]|record {}[]|csv:Error ct2br4_7 = csv:parseList( + [["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"]], {headerRows: 4, customHeaders: ["a", "c", "b"], outputWithHeaders: true}); + test:assertEquals(ct2br4_7, []); + + map[]|record {}[]|csv:Error ct2br4_6 = csv:parseList( + [["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"]], {headerRows: 2, outputWithHeaders: true}); + test:assertTrue(ct2br4_6 is csv:Error); + test:assertEquals((ct1br4_5).message(), "custom headers should be provided"); + + map[]|csv:Error ct2br4_8 = csv:parseList( + [["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"]], {headerRows: 1}); + test:assertEquals(ct2br4_8, [ + {"a": "a", "1": "1", "true": "true"}, + {"a": "a", "1": "1", "true": "true"}, + {"a": "a", "1": "1", "true": "true"} + ]); + + map[]|csv:Error ct2br4_8_2 = csv:parseList( + [["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"], ["a", "1", "true"]], {headerRows: 1}); + test:assertEquals(ct2br4_8_2, [ + {"a": "a", "1": "1", "true": "true"}, + {"a": "a", "1": "1", "true": "true"}, + {"a": "a", "1": "1", "true": "true"} + ]); + + map[]|csv:Error ct2br4_9 = csv:parseList( + [["a", "c", "b"], ["a", "2", "true"], ["a", "3", "true"], ["a", "4", "true"], ["a", "5", "true"]], {headerRows: 1}); + test:assertEquals(ct2br4_9, [ + {a: "a", b: "true", c: "2"}, + {a: "a", b: "true", c: "3"}, + {a: "a", b: "true", c: "4"}, + {a: "a", b: "true", c: "5"} + ]); + + map[]|csv:Error ct2br4_10 = csv:parseList( + [["a", "b", "c"], ["a", "2", "true"], ["a", "3", "true"], ["a", "4", "true"], ["a", "5", "true"]], {headerRows: 1}); + test:assertEquals(ct2br4_10, [ + {a: "a", c: true, b: 2}, + {a: "a", c: true, b: 3}, + {a: "a", c: true, b: 4}, + {a: "a", c: true, b: 5} + ]); + + map[]|csv:Error ct2br4_10_2 = csv:parseList( + [["a", "b", "c"], ["a", "2", "true"], ["a", "3", "true"], ["a", "4", "true"], ["a", "5", "true"]], {headerRows: 1}); + test:assertEquals(ct2br4_10_2, [ + {a: "a", c: true, b: 2}, + {a: "a", c: true, b: 3}, + {a: "a", c: true, b: 4}, + {a: "a", c: true, b: 5} + ]); + + map[]|record {}[]|csv:Error ct2br4_10_3 = csv:parseList( + [["a", "b", "c"], ["a", "2", "true"], ["a", "3", "true"], ["a", "4", "true"], ["a", "5", "true"]], {headerRows: 1, customHeaders: ["c", "d", "e"], outputWithHeaders: false}); + test:assertEquals(ct2br4_10_3, [ + {c: "a", e: true, d: 2}, + {c: "a", e: true, d: 3}, + {c: "a", e: true, d: 4}, + {c: "a", e: true, d: 5} + ]); +} + +@test:Config +function testFromCsvWithTypeForTupleAndRecordAsExpectedType8() { + record {|decimal c; boolean|string d; int e; string f; string...;|}[]|csv:Error ct1br10 = csv:parseList( + [["a", "1", "true", "0", "2.23", "null"], ["a", "1", "true", "2.23", "0", "()"]], + {customHeaders: ["f", "e", "d", "c", "b", "a"]}); + test:assertEquals(ct1br10, [ + {a: "null", b: "2.23", c: 0, d: true, e: 1, f: "a"}, + {a: "()", b: "0", c: 2.23, d: true, e: 1, f: "a"} + ]); + + record {|decimal? c; boolean d; int? e; string f; ()...;|}[]|csv:Error ct1br11 = csv:parseList( + [["a", "1", "true", "0", "2.23", "null"], ["a", "1", "true", "2.23", "0", "()"]], + {customHeaders: ["f", "e", "d", "c", "b", "a"]}); + test:assertEquals(ct1br11, [ + {a: (), c: 0, d: true, e: 1, f: "a"}, + {a: (), c: 2.23, d: true, e: 1, f: "a"} + ]); + + record {|()...;|}[]|csv:Error ct1br12 = csv:parseList( + [["a", "1", "true", "0", "2.23", "null"], ["a", "1", "true", "2.23", "0", "()"]], + {customHeaders: ["f", "e", "d", "c", "b", "a"]}); + test:assertEquals(ct1br12, [ + {a: ()}, + {a: ()} + ]); + + record {|string?...;|}[]|csv:Error ct1br13 = csv:parseList( + [["a", "1"], ["a", "1"]], + {customHeaders: ["f", "e"]}); + test:assertEquals(ct1br13, [ + {e: "1", f: "a"}, + {e: "1", f: "a"} + ]); + + record {|boolean...;|}[]|csv:Error ct1br14 = csv:parseList( + [["2.23", "null"], ["7", "()"]], + {customHeaders: ["b", "a"]}); + test:assertEquals(ct1br14, [ + {}, + {} + ]); + + map[]|csv:Error ct1br15 = csv:parseList( + [["2", "()"], ["2", "1"], ["()", "2"]], + {customHeaders: ["f", "e"]}); + test:assertEquals(ct1br15, [ + {e: (), f: 2}, + {e: 1, f: 2}, + {e: 2, f: ()} + ]); + + record {|boolean...;|}[]|csv:Error ct1br16 = csv:parseList( + [["2.23", "null"], ["7", "()"]], + {customHeaders: ["b", "a"]}); + test:assertEquals(ct1br16, [ + {}, + {} + ]); +} + +@test:Config +function testArrayIndexesWithRecordAsExpectedType() { + string[][] csv = [["1", "2", "3"], ["3", "4", "5"], ["5", "6", "7"], ["7", "8", "9"]]; + + map[2]|csv:Error rec_2 = csv:parseList(csv, {customHeaders: ["a", "b", "c"]}); + test:assertEquals(rec_2, [ + {a: 1, b: 2, c: 3}, + {a: 3, b: 4, c: 5} + ]); +} + +@test:Config +function testFromCsvWithTypeForTupleAndMapAsExpectedType() { + StringMapArray|csv:Error st1sma = csv:parseList([st1, st1], {}); + test:assertEquals(st1sma , [ + {'1: s1, '2: s2}, + {'1: s1, '2: s2} + ]); + + StringMapArray|csv:Error st2sma = csv:parseList([st2, st2], {}); + test:assertEquals(st2sma , [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + StringMapArray|csv:Error st3sma = csv:parseList([st3, st3], {}); + test:assertEquals(st3sma , [ + {'1: s1, '2: s2}, + {'1: s1, '2: s2} + ]); + + StringMapArray|csv:Error st4sma = csv:parseList([st4, st4], {}); + test:assertEquals(st4sma , [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + NillableIntUnionStringMapArray|csv:Error st1niusma = csv:parseList([st1, st1], {}); + test:assertEquals(st1niusma , [ + {'1: s1, '2: s2}, + {'1: s1, '2: s2} + ]); + + NillableIntUnionStringMapArray|csv:Error st2niusma = csv:parseList([st2, st2], {}); + test:assertEquals(st2niusma , [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + NillableIntUnionStringMapArray|csv:Error st3niusma = csv:parseList([st3, st3], {}); + test:assertEquals(st3niusma , [ + {'1: s1, '2: s2}, + {'1: s1, '2: s2} + ]); + + NillableIntUnionStringMapArray|csv:Error st4niusma = csv:parseList([st4, st4], {}); + test:assertEquals(st4niusma , [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + IntUnionStringMapArray|csv:Error st1iusma = csv:parseList([st1, st1], {}); + test:assertEquals(st1iusma , [ + {'1: s1, '2: s2}, + {'1: s1, '2: s2} + ]); + + IntUnionStringMapArray|csv:Error st2iusma = csv:parseList([st2, st2], {}); + test:assertEquals(st2iusma , [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + IntUnionStringMapArray|csv:Error st3iusma = csv:parseList([st3, st3], {}); + test:assertEquals(st3iusma , [ + {'1: s1, '2: s2}, + {'1: s1, '2: s2} + ]); + + IntUnionStringMapArray|csv:Error st4iusma = csv:parseList([st4, st4], {}); + test:assertEquals(st4iusma , [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + JsonMapArray|csv:Error st1jma = csv:parseList([st1, st1], {}); + test:assertEquals(st1jma , [ + {'1: s1, '2: s2}, + {'1: s1, '2: s2} + ]); +} + +@test:Config +function testFromCsvWithTypeForTupleAndMapAsExpectedType2() { + JsonMapArray|csv:Error st2jma = csv:parseList([st2, st2], {}); + test:assertEquals(st2jma , [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + JsonMapArray|csv:Error st3jma = csv:parseList([st3, st3], {}); + test:assertEquals(st3jma , [ + {'1: s1, '2: s2}, + {'1: s1, '2: s2} + ]); + + JsonMapArray|csv:Error st4jma = csv:parseList([st4, st4], {}); + test:assertEquals(st4jma , [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + AnydataMapArray|csv:Error st1anydma = csv:parseList([st1, st1], {}); + test:assertEquals(st1anydma , [ + {'1: s1, '2: s2}, + {'1: s1, '2: s2} + ]); + + AnydataMapArray|csv:Error st2anydma = csv:parseList([st2, st2], {}); + test:assertEquals(st2anydma , [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + AnydataMapArray|csv:Error st3anydma = csv:parseList([st3, st3], {}); + test:assertEquals(st3anydma , [ + {'1: s1, '2: s2}, + {'1: s1, '2: s2} + ]); + + AnydataMapArray|csv:Error st4anydma = csv:parseList([st4, st4], {}); + test:assertEquals(st4anydma , [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomMapArray|csv:Error st1cma = csv:parseList([st1, st1], {}); + test:assertEquals(st1cma , [ + {'1: s1, '2: s2}, + {'1: s1, '2: s2} + ]); + + CustomMapArray|csv:Error st2cma = csv:parseList([st2, st2], {}); + test:assertEquals(st2cma , [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + CustomMapArray|csv:Error st3cma = csv:parseList([st3, st3], {}); + test:assertEquals(st3cma , [ + {'1: s1, '2: s2}, + {'1: s1, '2: s2} + ]); + + CustomMapArray|csv:Error st4cma = csv:parseList([st4, st4], {}); + test:assertEquals(st4cma , [ + {'1: s1, '2: s2, '3: s3, '4: s2}, + {'1: s1, '2: s2, '3: s3, '4: s2} + ]); + + NilMapArray|csv:Error st1nma = csv:parseList([st1, st1], {}); + test:assertEquals(st1nma, ([ + {}, + {} + ])); + + IntegerMapArray|csv:Error st2ima = csv:parseList([st2, st2], {}); + test:assertEquals(st2ima, ([ + {}, + {} + ])); + + DecimalMapArray|csv:Error st3dma = csv:parseList([st3, st3], {}); + test:assertEquals(st3dma, ([ + {}, + {} + ])); + + BooleanMapArray|csv:Error st4bma = csv:parseList([st4, st4], {}); + test:assertEquals(st4bma, ([ + {}, + {} + ])); +} diff --git a/ballerina-tests/parse-list-types-tests/tests/test_data_values.bal b/ballerina-tests/parse-list-types-tests/tests/test_data_values.bal new file mode 100644 index 0000000..b2bc041 --- /dev/null +++ b/ballerina-tests/parse-list-types-tests/tests/test_data_values.bal @@ -0,0 +1,24 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +final string s1 = "string"; +final string s2 = ""; +final string:Char s3 = "a"; + +final [string, string] st1 = [s1, s2]; +final [string, string, string, string] st2 = [s1, s2, s3, s2]; +final [string...] st3 = [s1, s2]; +final [string, string, string...] st4 = [s1, s2, s3, s2]; diff --git a/ballerina-tests/parse-list-types-tests/tests/types.bal b/ballerina-tests/parse-list-types-tests/tests/types.bal new file mode 100644 index 0000000..930205f --- /dev/null +++ b/ballerina-tests/parse-list-types-tests/tests/types.bal @@ -0,0 +1,307 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +type StringRecord1 record { + string s1; + string s2; + string s3; +}; + +type StringRecord2 record {| + string s1; + string s2; + string s3; +|}; + +type StringRecord9 record {| + string s1; + string s2; + string...; +|}; + +type StringRecord10 record {| + string...; +|}; + +type StringRecord19 record { + string s1 = ""; + string s2 = ""; +}; + +type StringRecord20 record {| + string s1 = ""; + string s2 = ""; +|}; + +type StringRecord21 record { +}; + +type StringRecord22 record {| + string s1 = ""; + string s2 = ""; + json...; +|}; + +type StringRecord23 record {| + json...; +|}; + +type CustomRecord15 record {| + string '1; + string '2; +|}; + +type CustomRecord16 record {| + string '1; + string '2; + string '3; + string '4; +|}; + +type CustomRecord17 record { + string '1; + string '2; +}; + +type CustomRecord18 record { + string '1; + string '2; + string '3; + string '4; +}; + +type CustomRecord19 record { + string '1; + string '2; + string '3 = ""; + string '4 = ""; +}; + +type CustomRecord20 record { + string '1; +}; + +type CustomRecord21 record {| + string '1; + json...; +|}; + +type CustomRecord22 record {| + string '1; + string...; +|}; + +type CustomRecord23 record {| + string '1; + string a = ""; + string...; +|}; + +type CustomRecord24 record {| + string '1; + string '2 = ""; + string...; +|}; + +type CustomRecord25 record {| + int '1; + string...; +|}; + +type CustomRecord26 record {| + string '1; + int...; +|}; + +type BooleanTuple4 [boolean...]; + +type NilTuple3 [(), ()...]; + +type IntegerTuple3 [int, int...]; + +type IntegerTuple4 [int...]; + +type DecimalTuple4 [decimal...]; + +type StringTuple1 [string, string, string, string]; + +type StringTuple2 [string, string]; + +type StringTuple3 [string, string...]; + +type StringTuple4 [string...]; + +type AnydataTuple3 [anydata, anydata...]; + +type JsonTuple3 [json, json...]; + +type IntegerArray1 int[]; + +type IntegerArray2 int[2]; + +type IntegerArray3 int[]; + +type IntegerArray4 int[2]; + +type IntegerArray5 int[3]; + +type IntegerArray6 int[4]; + +type StringArray string[]; + +type NillableStringArray string?[]; + +type NillableIntOrUnionStringArray (int|string?)[]; + +type StringArray1 string[]; + +type StringArray2 string[2]; + +type DecimalArray1 decimal[]; + +type JsonArray1 json[]; + +type AnydataArray1 anydata[]; + +type IntegerMap map; + +type StringMap map; + +type NillableIntUnionStringMap map; + +type IntUnionStringMap map; + +type DecimalMap map; + +type BooleanMap map; + +type NilMap map<()>; + +type JsonMap map; + +type AnydataMap map; + +type CustomMap map; + +type StringRecord1Array StringRecord1[]; + +type StringRecord2Array StringRecord2[]; + +type StringRecord9Array StringRecord9[]; + +type StringRecord10Array StringRecord10[]; + +type StringRecord19Array StringRecord19[]; + +type StringRecord20Array StringRecord20[]; + +type StringRecord21Array StringRecord21[]; + +type StringRecord22Array StringRecord22[]; + +type StringRecord23Array StringRecord23[]; + +type CustomRecord15Array CustomRecord15[]; + +type CustomRecord16Array CustomRecord16[]; + +type CustomRecord17Array CustomRecord17[]; + +type CustomRecord18Array CustomRecord18[]; + +type CustomRecord19Array CustomRecord19[]; + +type CustomRecord20Array CustomRecord20[]; + +type CustomRecord21Array CustomRecord21[]; + +type CustomRecord22Array CustomRecord22[]; + +type CustomRecord23Array CustomRecord23[]; + +type CustomRecord24Array CustomRecord24[]; + +type CustomRecord25Array CustomRecord25[]; + +type CustomRecord26Array CustomRecord26[]; + +type BooleanTuple4Array BooleanTuple4[]; + +type NilTuple3Array NilTuple3[]; +type IntegerTuple3Array IntegerTuple3[]; + +type IntegerTuple4Array IntegerTuple4[]; + +type DecimalTuple4Array DecimalTuple4[]; + +type StringTuple1Array StringTuple1[]; + +type StringTuple2Array StringTuple2[]; + +type StringTuple3Array StringTuple3[]; + +type StringTuple4Array StringTuple4[]; + +type AnydataTuple3Array AnydataTuple3[]; + +type JsonTuple3Array JsonTuple3[]; + +type IntegerArray1Array IntegerArray1[]; + +type IntegerArray2Array IntegerArray2[]; + +type IntegerArray3Array IntegerArray3[]; + +type IntegerArray4Array IntegerArray4[]; + +type IntegerArray5Array IntegerArray5[]; + +type IntegerArray6Array IntegerArray5[]; + +type StringArray1Array StringArray1[]; + +type StringArrayArray StringArray[]; + +type NillableStringArrayArray NillableStringArray[]; + +type NillableIntOrUnionStringArrayArray NillableIntOrUnionStringArray[]; + +type StringArray2Array StringArray2[]; +type DecimalArray1Array DecimalArray1[]; + +type JsonArray1Array JsonArray1[]; + +type AnydataArray1Array AnydataArray1[]; + +type IntegerMapArray IntegerMap[]; + +type StringMapArray StringMap[]; + +type NillableIntUnionStringMapArray NillableIntUnionStringMap[]; + +type IntUnionStringMapArray IntUnionStringMap[]; + +type DecimalMapArray DecimalMap[]; + +type BooleanMapArray BooleanMap[]; + +type NilMapArray NilMap[]; + +type JsonMapArray JsonMap[]; + +type AnydataMapArray AnydataMap[]; + +type CustomMapArray CustomMap[]; diff --git a/ballerina-tests/parse-record-types-tests/Ballerina.toml b/ballerina-tests/parse-record-types-tests/Ballerina.toml new file mode 100644 index 0000000..7cdb2d4 --- /dev/null +++ b/ballerina-tests/parse-record-types-tests/Ballerina.toml @@ -0,0 +1,16 @@ +[package] +org = "ballerina" +name = "parse_record_types_tests" +version = "0.1.0" + +[[dependency]] +org = "ballerina" +name = "csv_commons" +repository = "local" +version = "0.1.0" + +[platform.java17] +graalvmCompatible = true + +[build-options] +graalvmBuildOptions = "-H:+IncludeAllLocales" diff --git a/ballerina-tests/parse-record-types-tests/Dependencies.toml b/ballerina-tests/parse-record-types-tests/Dependencies.toml new file mode 100644 index 0000000..54bfd11 --- /dev/null +++ b/ballerina-tests/parse-record-types-tests/Dependencies.toml @@ -0,0 +1,98 @@ +# AUTO-GENERATED FILE. DO NOT MODIFY. + +# This file is auto-generated by Ballerina for managing dependency versions. +# It should not be modified by hand. + +[ballerina] +dependencies-toml-version = "2" +distribution-version = "2201.9.0" + +[[package]] +org = "ballerina" +name = "csv_commons" +version = "0.1.0" +scope = "testOnly" +modules = [ + {org = "ballerina", packageName = "csv_commons", moduleName = "csv_commons"} +] + +[[package]] +org = "ballerina" +name = "data.csv" +version = "0.1.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] +modules = [ + {org = "ballerina", packageName = "data.csv", moduleName = "data.csv"} +] + +[[package]] +org = "ballerina" +name = "jballerina.java" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "lang.__internal" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.object"} +] + +[[package]] +org = "ballerina" +name = "lang.array" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.__internal"} +] + +[[package]] +org = "ballerina" +name = "lang.error" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "lang.object" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "parse_record_types_tests" +version = "0.1.0" +dependencies = [ + {org = "ballerina", name = "csv_commons"}, + {org = "ballerina", name = "data.csv"}, + {org = "ballerina", name = "test"} +] +modules = [ + {org = "ballerina", packageName = "parse_record_types_tests", moduleName = "parse_record_types_tests"} +] + +[[package]] +org = "ballerina" +name = "test" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.array"}, + {org = "ballerina", name = "lang.error"} +] +modules = [ + {org = "ballerina", packageName = "test", moduleName = "test"} +] + diff --git a/ballerina-tests/parse-record-types-tests/tests/parse_record_type_as_list_test.bal b/ballerina-tests/parse-record-types-tests/tests/parse_record_type_as_list_test.bal new file mode 100644 index 0000000..51a1840 --- /dev/null +++ b/ballerina-tests/parse-record-types-tests/tests/parse_record_type_as_list_test.bal @@ -0,0 +1,682 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/csv_commons as common; +import ballerina/data.csv; +import ballerina/test; + +@test:Config +function testFromCsvWithTypeForMapAndArrayAsExpectedType() { + BooleanArrayArray|csv:Error bm1ba = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, BooleanArrayArray); + test:assertEquals(bm1ba, [ + [true, false], + [true, false] + ]); + + bm1ba = csv:transform([bm1, bm1], {headerOrder: ["b2", "b1"]}, BooleanArrayArray); + test:assertEquals(bm1ba, [ + [false, true], + [false, true] + ]); + + BooleanArrayArray|csv:Error bm2ba = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, BooleanArrayArray); + test:assertTrue(bm2ba is csv:Error); + test:assertEquals((bm2ba).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "2", "boolean")); + + BooleanArrayArray|csv:Error bm3ba = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, BooleanArrayArray); + test:assertTrue(bm3ba is csv:Error); + test:assertEquals((bm3ba).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "2", "boolean")); + + BooleanArrayArray|csv:Error bm4ba = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, BooleanArrayArray); + test:assertTrue(bm4ba is csv:Error); + test:assertEquals((bm4ba).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "0", "boolean")); + + BooleanArrayArray|csv:Error bm5ba = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, BooleanArrayArray); + test:assertTrue(bm5ba is csv:Error); + test:assertEquals((bm5ba).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "2", "boolean")); + + NillableBooleanArrayArray|csv:Error bm1nba = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, NillableBooleanArrayArray); + test:assertEquals(bm1nba, [ + [true, false], + [true, false] + ]); + + NillableBooleanArrayArray|csv:Error bm2nba = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, NillableBooleanArrayArray); + test:assertEquals(bm2nba, [ + [true, false, null, null, null], + [true, false, null, null, null] + ]); + + NillableBooleanArrayArray|csv:Error bm3nba = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, NillableBooleanArrayArray); + test:assertTrue(bm3nba is csv:Error); + test:assertEquals((bm3nba).message(), common:generateErrorMessageForInvalidValueForArrayType("1", "4", "boolean?")); + + NillableBooleanArrayArray|csv:Error bm4nba = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, NillableBooleanArrayArray); + test:assertEquals(bm4nba, [ + [(), ()], + [(), ()] + ]); + + NillableBooleanArrayArray|csv:Error bm5nba = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, NillableBooleanArrayArray); + test:assertEquals(bm5nba, [ + [true, false, (), true], + [true, false, (), true] + ]); + + bm5nba = csv:transform([bm5, bm5], {headerOrder: ["b1", "b3", "b2", "b4"]}, NillableBooleanArrayArray); + test:assertEquals(bm5nba, [ + [true, (), false, true], + [true, (), false, true] + ]); + + bm5nba = csv:transform([bm5, bm5], {headerOrder: ["b4", "b3", "b2", "b1"]}, NillableBooleanArrayArray); + test:assertEquals(bm5nba, [ + [true, (), false, true], + [true, (), false, true] + ]); + + NillableIntOrUnionBooleanArrayArray|csv:Error bm1niouba = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, NillableIntOrUnionBooleanArrayArray); + test:assertEquals(bm1niouba, [ + [true, false], + [true, false] + ]); + + NillableIntOrUnionBooleanArrayArray|csv:Error bm2niouba = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, NillableIntOrUnionBooleanArrayArray); + test:assertEquals(bm2niouba, [ + [true, false, null, null, null], + [true, false, null, null, null] + ]); + + NillableIntOrUnionBooleanArrayArray|csv:Error bm3niouba = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, NillableIntOrUnionBooleanArrayArray); + test:assertEquals(bm3niouba, [ + [true, false, null, false, 1], + [true, false, null, false, 1] + ]); + + NillableIntOrUnionBooleanArrayArray|csv:Error bm4niouba = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, NillableIntOrUnionBooleanArrayArray); + test:assertEquals(bm4niouba, [ + [(), ()], + [(), ()] + ]); + + NillableIntOrUnionBooleanArrayArray|csv:Error bm5niouba = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, NillableIntOrUnionBooleanArrayArray); + test:assertEquals(bm5niouba, [ + [true, false, (), true], + [true, false, (), true] + ]); + + JsonArray1Array|csv:Error bm1ja = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, JsonArray1Array); + test:assertEquals(bm1ja, [ + [true, false], + [true, false] + ]); +} + +@test:Config +function testFromCsvWithTypeForMapAndArrayAsExpectedType2() { + JsonArray1Array|csv:Error bm2ja = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, JsonArray1Array); + test:assertEquals(bm2ja, [ + [true, false, null, null, null], + [true, false, null, null, null] + ]); + + JsonArray1Array|csv:Error bm3ja = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, JsonArray1Array); + test:assertEquals(bm3ja, [ + [true, false, null, false, 1], + [true, false, null, false, 1] + ]); + + JsonArray1Array|csv:Error bm4ja = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, JsonArray1Array); + test:assertEquals(bm4ja, [ + [(), ()], + [(), ()] + ]); + + JsonArray1Array|csv:Error bm5ja = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, JsonArray1Array); + test:assertEquals(bm5ja, [ + [true, false, (), true], + [true, false, (), true] + ]); + + AnydataArray1Array|csv:Error bm1anyda = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, AnydataArray1Array); + test:assertEquals(bm1anyda, [ + [true, false], + [true, false] + ]); + + AnydataArray1Array|csv:Error bm2anyda = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, AnydataArray1Array); + test:assertEquals(bm2anyda, [ + [true, false, null, null, null], + [true, false, null, null, null] + ]); + + AnydataArray1Array|csv:Error bm3anyda = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, AnydataArray1Array); + test:assertEquals(bm3anyda, [ + [true, false, null, false, 1], + [true, false, null, false, 1] + ]); + + AnydataArray1Array|csv:Error bm4anyda = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, AnydataArray1Array); + test:assertEquals(bm4anyda, [ + [(), ()], + [(), ()] + ]); + + AnydataArray1Array|csv:Error bm5anyda = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, AnydataArray1Array); + test:assertEquals(bm5anyda, [ + [true, false, (), true], + [true, false, (), true] + ]); + + StringArray1Array|csv:Error bm1sa = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, StringArray1Array); + test:assertTrue(bm1sa is csv:Error); + test:assertEquals((bm1sa).message(), common:generateErrorMessageForInvalidValueForArrayType("true", "0", "string")); + + StringArray1Array|csv:Error bm2sa = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, StringArray1Array); + test:assertTrue(bm2sa is csv:Error); + test:assertEquals((bm2sa).message(), common:generateErrorMessageForInvalidValueForArrayType("true", "0", "string")); + + StringArray1Array|csv:Error bm3sa = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, StringArray1Array); + test:assertTrue(bm3sa is csv:Error); + test:assertEquals((bm3sa).message(), common:generateErrorMessageForInvalidValueForArrayType("true", "0", "string")); + + StringArray1Array|csv:Error bm4sa = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, StringArray1Array); + test:assertTrue(bm4sa is csv:Error); + test:assertEquals((bm4sa).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "0", "string")); + + StringArray1Array|csv:Error bm5sa = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, StringArray1Array); + test:assertTrue(bm5sa is csv:Error); + test:assertEquals((bm5sa).message(), common:generateErrorMessageForInvalidValueForArrayType("true", "0", "string")); +} + +@test:Config +function testFromCsvWithTypeForMapAndTupleAsExpectedType() { + BooleanTuple1Array|csv:Error bm1bt = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, BooleanTuple1Array); + test:assertEquals(bm1bt, [ + [true, false, false, false], + [true, false, false, false] + ]); + + BooleanTuple1Array|csv:Error bm2bt = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, BooleanTuple1Array); + test:assertTrue(bm2bt is csv:Error); + test:assertEquals((bm2bt).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "2", "boolean")); + + BooleanTuple1Array|csv:Error bm3bt = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, BooleanTuple1Array); + test:assertTrue(bm3bt is csv:Error); + test:assertEquals((bm3bt).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "2", "boolean")); + + BooleanTuple1Array|csv:Error bm4bt = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, BooleanTuple1Array); + test:assertTrue(bm4bt is csv:Error); + test:assertEquals((bm4bt).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "0", "boolean")); + + BooleanTuple1Array|csv:Error bm5bt = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, BooleanTuple1Array); + test:assertTrue(bm5bt is csv:Error); + test:assertEquals((bm5bt).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "2", "boolean")); + + BooleanTuple2Array|csv:Error bm1b2t = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, BooleanTuple2Array); + test:assertEquals(bm1b2t, [ + [true, false], + [true, false] + ]); + + BooleanTuple2Array|csv:Error bm2b2t = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, BooleanTuple2Array); + test:assertEquals(bm2b2t, [ + [true, false], + [true, false] + ]); + + BooleanTuple2Array|csv:Error bm3b2t = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, BooleanTuple2Array); + test:assertEquals(bm3b2t, [ + [true, false], + [true, false] + ]); + + BooleanTuple2Array|csv:Error bm4b2t = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, BooleanTuple2Array); + test:assertTrue(bm4b2t is csv:Error); + test:assertEquals((bm4b2t).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "0", "boolean")); + + BooleanTuple2Array|csv:Error bm5b2t = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, BooleanTuple2Array); + test:assertEquals(bm5b2t, [ + [true, false], + [true, false] + ]); + + BooleanTuple3Array|csv:Error bm1b3t = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, BooleanTuple3Array); + test:assertEquals(bm1b3t, [ + [true, false], + [true, false] + ]); + + BooleanTuple3Array|csv:Error bm2b3t = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, BooleanTuple3Array); + test:assertTrue(bm2b3t is csv:Error); + test:assertEquals((bm2b3t).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "2", "boolean")); + + BooleanTuple3Array|csv:Error bm3b3t = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, BooleanTuple3Array); + test:assertTrue(bm3b3t is csv:Error); + test:assertEquals((bm3b3t).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "2", "boolean")); + + BooleanTuple3Array|csv:Error bm4b3t = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, BooleanTuple3Array); + test:assertTrue(bm4b3t is csv:Error); + test:assertEquals((bm4b3t).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "0", "boolean")); + + BooleanTuple3Array|csv:Error bm5b3t = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, BooleanTuple3Array); + test:assertTrue(bm5b3t is csv:Error); + test:assertEquals((bm5b3t).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "2", "boolean")); + + BooleanTuple4Array|csv:Error bm1b4t = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, BooleanTuple4Array); + test:assertEquals(bm1b4t, [ + [true, false], + [true, false] + ]); + + BooleanTuple4Array|csv:Error bm2b4t = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, BooleanTuple4Array); + test:assertTrue(bm2b4t is csv:Error); + test:assertEquals((bm2b4t).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "2", "boolean")); +} + +@test:Config +function testFromCsvWithTypeForMapAndTupleAsExpectedType2() { + BooleanTuple4Array|csv:Error bm3b4t = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, BooleanTuple4Array); + test:assertTrue(bm3b4t is csv:Error); + test:assertEquals((bm3b4t).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "2", "boolean")); + + BooleanTuple4Array|csv:Error bm4b4t = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, BooleanTuple4Array); + test:assertTrue(bm4b4t is csv:Error); + test:assertEquals((bm4b4t).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "0", "boolean")); + + BooleanTuple4Array|csv:Error bm5b4t = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, BooleanTuple4Array); + test:assertTrue(bm5b4t is csv:Error); + test:assertEquals((bm5b4t).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "2", "boolean")); + + NillableBooleanTuple5Array|csv:Error bm1nbt = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, NillableBooleanTuple5Array); + test:assertEquals(bm1nbt, [ + [true, false, (), (), ()], + [true, false, (), (), ()] + ]); + + NillableBooleanTuple5Array|csv:Error bm2nbt = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, NillableBooleanTuple5Array); + test:assertEquals(bm2nbt, [ + [true, false, (), (), ()], + [true, false, (), (), ()] + ]); + + NillableBooleanTuple5Array|csv:Error bm3nbt = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, NillableBooleanTuple5Array); + test:assertTrue(bm3nbt is csv:Error); + test:assertEquals((bm3nbt).message(), common:generateErrorMessageForInvalidValueForArrayType("1", "4", "boolean?")); + + NillableBooleanTuple5Array|csv:Error bm4nbt = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, NillableBooleanTuple5Array); + test:assertEquals(bm4nbt, [ + [(), (), (), (), ()], + [(), (), (), (), ()] + ]); + + NillableBooleanTuple5Array|csv:Error bm5nbt = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, NillableBooleanTuple5Array); + test:assertEquals(bm5nbt, [ + [true, false, (), true, ()], + [true, false, (), true, ()] + ]); + + NillableBooleanTuple6Array|csv:Error bm1nb6t = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, NillableBooleanTuple6Array); + test:assertEquals(bm1nb6t, [ + [true, false], + [true, false] + ]); + + NillableBooleanTuple6Array|csv:Error bm2nb6t = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, NillableBooleanTuple6Array); + test:assertEquals(bm2nb6t, [ + [true, false], + [true, false] + ]); + + NillableBooleanTuple6Array|csv:Error bm3nb6t = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, NillableBooleanTuple6Array); + test:assertEquals(bm3nb6t, [ + [true, false], + [true, false] + ]); + + NillableBooleanTuple6Array|csv:Error bm4nb6t = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, NillableBooleanTuple6Array); + test:assertEquals(bm4nb6t, [ + [(), ()], + [(), ()] + ]); + + NillableBooleanTuple6Array|csv:Error bm5nb6t = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, NillableBooleanTuple6Array); + test:assertEquals(bm5nb6t, [ + [true, false], + [true, false] + ]); + + NillableBooleanTuple7Array|csv:Error bm1nb7t = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, NillableBooleanTuple7Array); + test:assertEquals(bm1nb7t, [ + [true, false], + [true, false] + ]); + + NillableBooleanTuple7Array|csv:Error bm2nb7t = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, NillableBooleanTuple7Array); + test:assertEquals(bm2nb7t, [ + [true, false, (), (), ()], + [true, false, (), (), ()] + ]); + + NillableBooleanTuple7Array|csv:Error bm3nb7t = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, NillableBooleanTuple7Array); + test:assertTrue(bm3nb7t is csv:Error); + test:assertEquals((bm3nb7t).message(), common:generateErrorMessageForInvalidValueForArrayType("1", "4", "boolean?")); + + NillableBooleanTuple7Array|csv:Error bm4nb7t = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, NillableBooleanTuple7Array); + test:assertEquals(bm4nb7t, [ + [(), ()], + [(), ()] + ]); + + NillableBooleanTuple7Array|csv:Error bm5nb7t = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, NillableBooleanTuple7Array); + test:assertEquals(bm5nb7t, [ + [true, false, (), true], + [true, false, (), true] + ]); +} + +@test:Config +function testFromCsvWithTypeForMapAndTupleAsExpectedType3() { + NillableBooleanTuple8Array|csv:Error bm1nb8t = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, NillableBooleanTuple8Array); + test:assertEquals(bm1nb8t, [ + [true, false], + [true, false] + ]); + + NillableBooleanTuple8Array|csv:Error bm2nb8t = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, NillableBooleanTuple8Array); + test:assertEquals(bm2nb8t, [ + [true, false, (), (), ()], + [true, false, (), (), ()] + ]); + + NillableBooleanTuple8Array|csv:Error bm3nb8t = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, NillableBooleanTuple8Array); + test:assertTrue(bm3nb8t is csv:Error); + test:assertEquals((bm3nb8t).message(), common:generateErrorMessageForInvalidValueForArrayType("1", "4", "boolean?")); + + NillableBooleanTuple8Array|csv:Error bm4nb8t = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, NillableBooleanTuple8Array); + test:assertEquals(bm4nb8t, [ + [(), ()], + [(), ()] + ]); + + NillableBooleanTuple8Array|csv:Error bm5nb8t = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, NillableBooleanTuple8Array); + test:assertEquals(bm5nb8t, [ + [true, false, (), true], + [true, false, (), true] + ]); + + NillableIntBooleanTuple9Array|csv:Error bm1nb9t = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, NillableIntBooleanTuple9Array); + test:assertEquals(bm1nb9t, [ + [true, false], + [true, false] + ]); + + NillableIntBooleanTuple9Array|csv:Error bm2nb9t = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, NillableIntBooleanTuple9Array); + test:assertEquals(bm2nb9t, [ + [true, false, (), (), ()], + [true, false, (), (), ()] + ]); + + NillableIntBooleanTuple9Array|csv:Error bm3nb9t = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, NillableIntBooleanTuple9Array); + test:assertEquals(bm3nb9t, [ + [true, false, (), false, 1], + [true, false, (), false, 1] + ]); + + NillableIntBooleanTuple9Array|csv:Error bm4nb9t = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, NillableIntBooleanTuple9Array); + test:assertEquals(bm4nb9t, [ + [(), ()], + [(), ()] + ]); + + NillableIntBooleanTuple9Array|csv:Error bm5nb9t = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, NillableIntBooleanTuple9Array); + test:assertEquals(bm5nb9t, [ + [true, false, (), true], + [true, false, (), true] + ]); + + NilTuple3Array|csv:Error bm1n3t = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, NilTuple3Array); + test:assertTrue(bm1n3t is csv:Error); + test:assertEquals((bm1n3t).message(), common:generateErrorMessageForInvalidValueForArrayType("true", "0", "()")); + + NilTuple3Array|csv:Error bm2n3t = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, NilTuple3Array); + test:assertTrue(bm2n3t is csv:Error); + test:assertEquals((bm2n3t).message(), common:generateErrorMessageForInvalidValueForArrayType("true", "0", "()")); + + NilTuple3Array|csv:Error bm3n3t = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, NilTuple3Array); + test:assertTrue(bm3n3t is csv:Error); + test:assertEquals((bm3n3t).message(), common:generateErrorMessageForInvalidValueForArrayType("true", "0", "()")); + + NilTuple3Array|csv:Error bm4n3t = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, NilTuple3Array); + test:assertEquals(bm4n3t, [ + [(), ()], + [(), ()] + ]); + + NilTuple3Array|csv:Error bm5n3t = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, NilTuple3Array); + test:assertTrue(bm5n3t is csv:Error); + test:assertEquals((bm5n3t).message(), common:generateErrorMessageForInvalidValueForArrayType("true", "0", "()")); +} + +@test:Config +function testFromCsvWithTypeForMapAndArrayAsExpectedType3() { + + AnydataTuple3Array|csv:Error bm1anyd3t = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"], outputWithHeaders: false}, AnydataTuple3Array); + test:assertEquals(bm1anyd3t, [ + [true, false], + [true, false] + ]); + + AnydataTuple3Array|csv:Error bm2anyd3t = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, AnydataTuple3Array); + test:assertEquals(bm2anyd3t, [ + [true, false, (), (), ()], + [true, false, (), (), ()] + ]); + + AnydataTuple3Array|csv:Error bm3anyd3t = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, AnydataTuple3Array); + test:assertEquals(bm3anyd3t, [ + [true, false, (), false, 1], + [true, false, (), false, 1] + ]); + + AnydataTuple3Array|csv:Error bm4anyd3t = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"], outputWithHeaders: false}, AnydataTuple3Array); + test:assertEquals(bm4anyd3t, [ + [(), ()], + [(), ()] + ]); + + AnydataTuple3Array|csv:Error bm5anyd3t = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, AnydataTuple3Array); + test:assertEquals(bm5anyd3t, [ + [true, false, (), true], + [true, false, (), true] + ]); + + JsonTuple3Array|csv:Error bm1j3t = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, JsonTuple3Array); + test:assertEquals(bm1j3t, [ + [true, false], + [true, false] + ]); + + JsonTuple3Array|csv:Error bm2j3t = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, JsonTuple3Array); + test:assertEquals(bm2j3t, [ + [true, false, (), (), ()], + [true, false, (), (), ()] + ]); + + JsonTuple3Array|csv:Error bm3j3t = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, JsonTuple3Array); + test:assertEquals(bm3j3t, [ + [true, false, (), false, 1], + [true, false, (), false, 1] + ]); + + JsonTuple3Array|csv:Error bm4j3t = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, JsonTuple3Array); + test:assertEquals(bm4j3t, [ + [(), ()], + [(), ()] + ]); + + JsonTuple3Array|csv:Error bm5j3t = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, JsonTuple3Array); + test:assertEquals(bm5j3t, [ + [true, false, (), true], + [true, false, (), true] + ]); + + StringTuple3Array|csv:Error bm1s3t = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}, StringTuple3Array); + test:assertTrue(bm1s3t is csv:Error); + test:assertEquals((bm1s3t).message(), common:generateErrorMessageForInvalidValueForArrayType("true", "0", "string")); + + StringTuple3Array|csv:Error bm2s3t = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"]}, StringTuple3Array); + test:assertTrue(bm2s3t is csv:Error); + test:assertEquals((bm2s3t).message(), common:generateErrorMessageForInvalidValueForArrayType("true", "0", "string")); + + StringTuple3Array|csv:Error bm3s3t = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"]}, StringTuple3Array); + test:assertTrue(bm3s3t is csv:Error); + test:assertEquals((bm3s3t).message(), common:generateErrorMessageForInvalidValueForArrayType("true", "0", "string")); + + StringTuple3Array|csv:Error bm4s3t = csv:transform([bm4, bm4], {headerOrder: ["n1", "n3"]}, StringTuple3Array); + test:assertTrue(bm4s3t is csv:Error); + test:assertEquals((bm4s3t).message(), common:generateErrorMessageForInvalidValueForArrayType("null", "0", "string")); + + StringTuple3Array|csv:Error bm5s3t = csv:transform([bm5, bm5], {headerOrder: ["b1", "b2", "b3", "b4"]}, StringTuple3Array); + test:assertTrue(bm5s3t is csv:Error); + test:assertEquals((bm5s3t).message(), common:generateErrorMessageForInvalidValueForArrayType("true", "0", "string")); +} + +@test:Config +function testArrayIndexes() { + map[] csv = [{a: 1, b: 2}, {a: 3, b: 4}, {a: 5, b: 6}, {a: 7, b: 8}]; + + int[][2]|csv:Error rec3 = csv:transform(csv, {headerOrder: ["a", "b"]}); + test:assertEquals(rec3, [ + [1, 2], + [3, 4], + [5, 6], + [7, 8] + ]); + + [int...][2]|csv:Error rec3_3 = csv:transform(csv, {headerOrder: ["a", "b"], skipLines: [1]}); + test:assertEquals(rec3_3, [ + [3, 4], + [5, 6] + ]); + + int[1][2]|csv:Error rec4 = csv:transform(csv, {headerOrder: ["a", "b"], skipLines: [2]}); + test:assertEquals(rec4, [ + [1, 2] + ]); +} + +@test:Config +function testParseRecordsAsListsWithHeaderOutput() { + AnydataTuple3Array|csv:Error bm1anyd3t = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"], outputWithHeaders: true}); + test:assertEquals(bm1anyd3t, [ + ["b1", "b2"], + [true, false], + [true, false] + ]); + + AnydataTuple3Array|csv:Error bm2anyd3t = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"], outputWithHeaders: true}); + test:assertEquals(bm2anyd3t, [ + ["b1", "b2", "b3", "n1", "n3"], + [true, false, (), (), ()], + [true, false, (), (), ()] + ]); + + AnydataTuple3Array|csv:Error bm3anyd3t = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"], outputWithHeaders: true}); + test:assertEquals(bm3anyd3t, [ + ["b1", "b2", "b3", "b4", "i1"], + [true, false, (), false, 1], + [true, false, (), false, 1] + ]); + + anydata[][]|csv:Error bm1anyd3t_2 = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"], outputWithHeaders: true}); + test:assertEquals(bm1anyd3t_2, [ + ["b1", "b2"], + [true, false], + [true, false] + ]); + + anydata[][]|csv:Error bm2anyd3t_2 = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"], outputWithHeaders: true}); + test:assertEquals(bm2anyd3t_2, [ + ["b1", "b2", "b3", "n1", "n3"], + [true, false, (), (), ()], + [true, false, (), (), ()] + ]); + + anydata[2][3]|csv:Error bm3anyd3t_2 = csv:transform([bm3, bm3], {headerOrder: ["b1", "b2", "b3", "b4", "i1"], outputWithHeaders: true}); + test:assertEquals(bm3anyd3t_2, [ + ["b1", "b2", "b3"], + [true, false, ()] + ]); + + anydata[][]|csv:Error bm2anyd3t_3 = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n1", "n3"], outputWithHeaders: false}); + test:assertEquals(bm2anyd3t_3, [ + [true, false, (), (), ()], + [true, false, (), (), ()] + ]); + + anydata[][]|csv:Error bm2anyd3t_4 = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n2", "n3"], outputWithHeaders: false}); + test:assertTrue(bm2anyd3t_4 is csv:Error); + test:assertEquals(( bm2anyd3t_4).message(), "CSV data rows with varying headers are not yet supported"); + + bm2anyd3t_4 = csv:transform([bm2, bm2], {headerOrder: ["b1", "b2", "b3", "n2", "n3"], outputWithHeaders: true}); + test:assertTrue(bm2anyd3t_4 is csv:Error); + test:assertEquals(( bm2anyd3t_4).message(), "CSV data rows with varying headers are not yet supported"); +} + +@test:Config +function testsRecordsOutputWithHeadrs() { + anydata[2][3]|csv:Error bm3anyd3t_2 = csv:transform([bm3, bm3], {outputWithHeaders: true}); + test:assertEquals(bm3anyd3t_2, [ + ["b1", "b2", "b3"], + [true, false, ()] + ]); + + anydata[][]|csv:Error bm3anyd3t_3 = csv:transform([bm3, bm3], {outputWithHeaders: true}); + test:assertEquals(bm3anyd3t_3, [ + ["b1", "b2", "b3", "b4", "i1"], + [true, false, (), false, 1], + [true, false, (), false, 1] + ]); + + bm3anyd3t_3 = csv:transform([{"a": 1, "b": 2}, {"a": 3, "b": 4}], {outputWithHeaders: true}); + test:assertEquals(bm3anyd3t_3, [ + ["a", "b"], + [1, 2], + [3, 4] + ]); + + bm3anyd3t_3 = csv:transform([{"a": 1, "b": 2}, {"c": 3, "d": 4}], {outputWithHeaders: true}); + test:assertTrue(bm3anyd3t_3 is csv:Error); + test:assertEquals(( bm3anyd3t_3).message(), "CSV data rows with varying headers are not yet supported"); + + bm3anyd3t_3 = csv:transform([{"a": 1, "b": 2}, {"c": 3, "d": 4}], {outputWithHeaders: false}); + test:assertTrue(bm3anyd3t_3 is csv:Error); + test:assertEquals(( bm3anyd3t_3).message(), "CSV data rows with varying headers are not yet supported"); + + record {}[]|csv:Error bm3anyd3t_4 = csv:transform([{"a": 1, "b": 2}, {"a": 3, "b": 4}]); + test:assertEquals(bm3anyd3t_4, [{"a": 1, "b": 2}, {"a": 3, "b": 4}]); + + bm3anyd3t_4 = csv:transform([{"a": 1, "b": 2}, {"c": 3, "d": 4}]); + test:assertEquals(bm3anyd3t_4, [{"a": 1, "b": 2}, {"c": 3, "d": 4}]); + + bm3anyd3t_3 = csv:parseList([["a", "b"], ["c", "d", "e"]], {outputWithHeaders: true}); + test:assertEquals(bm3anyd3t_3, [["a", "b"], ["c", "d", "e"]]); + + bm3anyd3t_4 = csv:parseList([["a", "b"], ["c", "d", "e"]]); + test:assertTrue(bm3anyd3t_4 is csv:Error); + test:assertEquals(( bm3anyd3t_4).message(), "CSV data rows with varying headers are not yet supported"); +} diff --git a/ballerina-tests/parse-record-types-tests/tests/parse_record_type_as_record_test.bal b/ballerina-tests/parse-record-types-tests/tests/parse_record_type_as_record_test.bal new file mode 100644 index 0000000..172ec9d --- /dev/null +++ b/ballerina-tests/parse-record-types-tests/tests/parse_record_type_as_record_test.bal @@ -0,0 +1,769 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/csv_commons as common; +import ballerina/data.csv; +import ballerina/test; + +@test:Config +function testFromCsvWithTypeForMapAndRecordAsExpectedType() { + BooleanRecord1Array|csv:Error bm1br1 = csv:transform([bm1, bm1], {}, BooleanRecord1Array); + test:assertTrue(bm1br1 is csv:Error); + test:assertEquals((bm1br1).message(), common:generateErrorMessageForMissingRequiredField("b3")); + + BooleanRecord1Array|csv:Error bm2br1 = csv:transform([bm2, bm2], {}, BooleanRecord1Array); + test:assertTrue(bm2br1 is csv:Error); + test:assertEquals((bm2br1).message(), common:generateErrorMessageForMissingRequiredField("b4")); + + BooleanRecord1Array|csv:Error bm3br1 = csv:transform([bm3, bm3], {}, BooleanRecord1Array); + test:assertEquals(bm3br1, [ + {b1: true, b2: false, b3: (), b4: false, i1: 1}, + {b1: true, b2: false, b3: (), b4: false, i1: 1} + ]); + + BooleanRecord1Array|csv:Error bm4br1 = csv:transform([bm4, bm4], {}, BooleanRecord1Array); + test:assertTrue(bm4br1 is csv:Error); + test:assertEquals((bm4br1).message(), common:generateErrorMessageForMissingRequiredField("b2")); + + BooleanRecord1Array|csv:Error bm5br1 = csv:transform([bm5, bm5], {}, BooleanRecord1Array); + test:assertEquals(bm5br1, [ + {b1: true, b2: false, b3: (), b4: true}, + {b1: true, b2: false, b3: (), b4: true} + ]); + + BooleanRecord2Array|csv:Error bm1br2 = csv:transform([bm1, bm1], {}, BooleanRecord2Array); + test:assertTrue(bm1br2 is csv:Error); + test:assertEquals((bm1br2).message(), common:generateErrorMessageForMissingRequiredField("b3")); + + BooleanRecord2Array|csv:Error bm2br2 = csv:transform([bm2, bm2], {}, BooleanRecord2Array); + test:assertTrue(bm2br2 is csv:Error); + test:assertEquals((bm2br2).message(), common:generateErrorMessageForMissingRequiredField("b4")); + + BooleanRecord2Array|csv:Error bm3br2 = csv:transform([bm3, bm3], {}, BooleanRecord2Array); + test:assertEquals(bm3br2, [ + {b1: true, b2: false, b3: (), b4: false}, + {b1: true, b2: false, b3: (), b4: false} + ]); + + BooleanRecord2Array|csv:Error bm4br2 = csv:transform([bm4, bm4], {}, BooleanRecord2Array); + test:assertTrue(bm4br2 is csv:Error); + test:assertEquals((bm4br2).message(), common:generateErrorMessageForMissingRequiredField("b2")); + + BooleanRecord2Array|csv:Error bm5br2 = csv:transform([bm5, bm5], {}, BooleanRecord2Array); + test:assertEquals(bm5br2, [ + {b1: true, b2: false, b3: (), b4: true}, + {b1: true, b2: false, b3: (), b4: true} + ]); + + BooleanRecord3Array|csv:Error bm1br3 = csv:transform([bm1, bm1], {}, BooleanRecord3Array); + test:assertTrue(bm1br3 is csv:Error); + test:assertEquals((bm1br3).message(), common:generateErrorMessageForMissingRequiredField("b3")); + + BooleanRecord3Array|csv:Error bm2br3 = csv:transform([bm2, bm2], {}, BooleanRecord3Array); + test:assertEquals(bm2br3, [ + {b1: true, b3: ()}, + {b1: true, b3: ()} + ]); + + BooleanRecord3Array|csv:Error bm3br3 = csv:transform([bm3, bm3], {}, BooleanRecord3Array); + test:assertEquals(bm3br3, [ + {b1: true, b3: ()}, + {b1: true, b3: ()} + ]); + + BooleanRecord3Array|csv:Error bm4br3 = csv:transform([bm4, bm4], {}, BooleanRecord3Array); + test:assertTrue(bm4br3 is csv:Error); + test:assertEquals((bm4br3).message(), common:generateErrorMessageForMissingRequiredField("b3")); + + BooleanRecord3Array|csv:Error bm5br3 = csv:transform([bm5, bm5], {}, BooleanRecord3Array); + test:assertEquals(bm5br3, [{b1: true, b3: ()}, {b1: true, b3: ()}]); + + BooleanRecord4Array|csv:Error bm1br4 = csv:transform([bm1, bm1], {}, BooleanRecord4Array); + test:assertTrue(bm1br4 is csv:Error); + test:assertEquals((bm1br4).message(), common:generateErrorMessageForMissingRequiredField("b3")); + + BooleanRecord4Array|csv:Error bm2br4 = csv:transform([bm2, bm2], {}, BooleanRecord4Array); + test:assertEquals(bm2br4, [ + {b1: true, b2: false, b3: (), n1: (), n3: ()}, + {b1: true, b2: false, b3: (), n1: (), n3: ()} + ]); + + BooleanRecord4Array|csv:Error bm3br4 = csv:transform([bm3, bm3], {}, BooleanRecord4Array); + test:assertEquals(bm3br4, [ + {b1: true, b2: false, b3: (), b4: false, i1: 1}, + {b1: true, b2: false, b3: (), b4: false, i1: 1} + ]); + + BooleanRecord4Array|csv:Error bm4br4 = csv:transform([bm4, bm4], {}, BooleanRecord4Array); + test:assertTrue(bm4br4 is csv:Error); + test:assertEquals((bm4br4).message(), common:generateErrorMessageForMissingRequiredField("b3")); + + BooleanRecord4Array|csv:Error bm5br4 = csv:transform([bm5, bm5], {}, BooleanRecord4Array); + test:assertEquals(bm5br4, [ + {b1: true, b2: false, b3: (), b4: true}, + {b1: true, b2: false, b3: (), b4: true} + ]); + + BooleanRecord5Array|csv:Error bm1br5 = csv:transform([bm1, bm1], {}, BooleanRecord5Array); + test:assertTrue(bm1br5 is csv:Error); + test:assertEquals((bm1br5).message(), common:generateErrorMessageForMissingRequiredField("b3")); + + BooleanRecord5Array|csv:Error bm2br5 = csv:transform([bm2, bm2], {}, BooleanRecord5Array); + test:assertEquals(bm2br5, [ + {b1: true, b3: (), defaultableField: "", nillableField: (), b2: false, n1: (), n3: ()}, + {b1: true, b3: (), defaultableField: "", nillableField: (), b2: false, n1: (), n3: ()} + ]); +} + +@test:Config +function testFromCsvWithTypeForMapAndRecordAsExpectedType2() { + BooleanRecord5Array|csv:Error bm3br5 = csv:transform([bm3, bm3], {}, BooleanRecord5Array); + test:assertEquals(bm3br5, [ + {b1: true, b3: (), defaultableField: "", nillableField: (), b2: false, i1: 1, b4: false}, + {b1: true, b3: (), defaultableField: "", nillableField: (), b2: false, i1: 1, b4: false} + ]); + + BooleanRecord5Array|csv:Error bm4br5 = csv:transform([bm4, bm4], {}, BooleanRecord5Array); + test:assertTrue(bm4br5 is csv:Error); + test:assertEquals((bm4br5).message(), common:generateErrorMessageForMissingRequiredField("b3")); + + BooleanRecord5Array|csv:Error bm5br5 = csv:transform([bm5, bm5], {}, BooleanRecord5Array); + test:assertEquals(bm5br5, [ + {b1: true, b2: false, b3: (), b4: true, defaultableField: "", nillableField: ()}, + {b1: true, b2: false, b3: (), b4: true, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord6Array|csv:Error bm1br6 = csv:transform([bm1, bm1], {}, BooleanRecord6Array); + test:assertTrue(bm1br6 is csv:Error); + test:assertEquals((bm1br6).message(), common:generateErrorMessageForMissingRequiredField("b3")); + + BooleanRecord6Array|csv:Error bm2br6 = csv:transform([bm2, bm2], {}, BooleanRecord6Array); + test:assertEquals(bm2br6, [ + {b1: true, b3: (), defaultableField: "", nillableField: ()}, + {b1: true, b3: (), defaultableField: "", nillableField: ()} + ]); + + BooleanRecord6Array|csv:Error bm3br6 = csv:transform([bm3, bm3], {}, BooleanRecord6Array); + test:assertEquals(bm3br6, [ + {b1: true, b3: (), defaultableField: "", nillableField: ()}, + {b1: true, b3: (), defaultableField: "", nillableField: ()} + ]); + + BooleanRecord6Array|csv:Error bm4br6 = csv:transform([bm4, bm4], {}, BooleanRecord6Array); + test:assertTrue(bm4br6 is csv:Error); + test:assertEquals((bm4br6).message(), common:generateErrorMessageForMissingRequiredField("b3")); + + BooleanRecord6Array|csv:Error bm5br6 = csv:transform([bm5, bm5], {}, BooleanRecord6Array); + test:assertEquals(bm5br6, [ + {b1: true, b3: (), defaultableField: "", nillableField: ()}, + {b1: true, b3: (), defaultableField: "", nillableField: ()} + ]); + + BooleanRecord7Array|csv:Error bm1br7 = csv:transform([bm1, bm1], {}, BooleanRecord7Array); + test:assertTrue(bm1br7 is csv:Error); + test:assertEquals((bm1br7).message(), common:generateErrorMessageForMissingRequiredField("b3")); + + BooleanRecord7Array|csv:Error bm2br7 = csv:transform([bm2, bm2], {}, BooleanRecord7Array); + test:assertTrue(bm2br7 is csv:Error); + test:assertEquals((bm2br7).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord7Array|csv:Error bm3br7 = csv:transform([bm3, bm3], {}, BooleanRecord7Array); + test:assertTrue(bm3br7 is csv:Error); + test:assertEquals((bm3br7).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord7Array|csv:Error bm4br7 = csv:transform([bm4, bm4], {}, BooleanRecord7Array); + test:assertTrue(bm4br7 is csv:Error); + test:assertEquals((bm4br7).message(), common:generateErrorMessageForMissingRequiredField("b3")); + + BooleanRecord7Array|csv:Error bm5br7 = csv:transform([bm5, bm5], {}, BooleanRecord7Array); + test:assertTrue(bm5br7 is csv:Error); + test:assertEquals((bm5br7).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord8Array|csv:Error bm1br8 = csv:transform([bm1, bm1], {}, BooleanRecord8Array); + test:assertTrue(bm1br8 is csv:Error); + test:assertEquals((bm1br8).message(), common:generateErrorMessageForMissingRequiredField("b3")); + + BooleanRecord8Array|csv:Error bm2br8 = csv:transform([bm2, bm2], {}, BooleanRecord8Array); + test:assertTrue(bm2br8 is csv:Error); + test:assertEquals((bm2br8).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord8Array|csv:Error bm3br8 = csv:transform([bm3, bm3], {}, BooleanRecord8Array); + test:assertTrue(bm3br8 is csv:Error); + test:assertEquals((bm3br8).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord8Array|csv:Error bm4br8 = csv:transform([bm4, bm4], {}, BooleanRecord8Array); + test:assertTrue(bm4br8 is csv:Error); + test:assertEquals((bm4br8).message(), common:generateErrorMessageForMissingRequiredField("b3")); + + BooleanRecord8Array|csv:Error bm5br8 = csv:transform([bm5, bm5], {}, BooleanRecord8Array); + test:assertTrue(bm5br8 is csv:Error); + test:assertEquals((bm5br8).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); +} + +@test:Config +function testFromCsvWithTypeForMapAndRecordAsExpectedType3() { + BooleanRecord9Array|csv:Error bm1br9 = csv:transform([bm1, bm1], {}, BooleanRecord9Array); + test:assertTrue(bm1br9 is csv:Error); + test:assertEquals((bm1br9).message(), common:generateErrorMessageForMissingRequiredField("b3")); + + BooleanRecord9Array|csv:Error bm2br9 = csv:transform([bm2, bm2], {}, BooleanRecord9Array); + test:assertEquals(bm2br9, [ + {b1: true, b2: false, b3: (), n1: (), n3: ()}, + {b1: true, b2: false, b3: (), n1: (), n3: ()} + ]); + + BooleanRecord9Array|csv:Error bm3br9 = csv:transform([bm3, bm3], {}, BooleanRecord9Array); + test:assertEquals(bm3br9, [ + {b1: true, b2: false, b3: (), b4: false}, + {b1: true, b2: false, b3: (), b4: false} + ]); + + BooleanRecord9Array|csv:Error bm4br9 = csv:transform([bm4, bm4], {}, BooleanRecord9Array); + test:assertTrue(bm4br9 is csv:Error); + test:assertEquals((bm4br9).message(), common:generateErrorMessageForMissingRequiredField("b3")); + + BooleanRecord9Array|csv:Error bm5br9 = csv:transform([bm5, bm5], {}, BooleanRecord9Array); + test:assertEquals(bm5br9, [ + {b1: true, b2: false, b3: (), b4: true}, + {b1: true, b2: false, b3: (), b4: true} + ]); + + BooleanRecord10Array|csv:Error bm1br10 = csv:transform([bm1, bm1], {}, BooleanRecord10Array); + test:assertEquals(bm1br10, [ + {b1: true, b2: false}, + {b1: true, b2: false} + ]); + + BooleanRecord10Array|csv:Error bm2br10 = csv:transform([bm2, bm2], {}, BooleanRecord10Array); + test:assertEquals(bm2br10, [ + {b1: true, b2: false}, + {b1: true, b2: false} + ]); + + BooleanRecord10Array|csv:Error bm3br10 = csv:transform([bm3, bm3], {}, BooleanRecord10Array); + test:assertEquals(bm3br10, [ + {b1: true, b2: false, b4: false}, + {b1: true, b2: false, b4: false} + ]); + + BooleanRecord10Array|csv:Error bm4br10 = csv:transform([bm4, bm4], {}, BooleanRecord10Array); + test:assertEquals(bm4br10, [ + {}, + {} + ]); + + BooleanRecord10Array|csv:Error bm5br10 = csv:transform([bm5, bm5], {}, BooleanRecord10Array); + test:assertEquals(bm5br10, [ + {b1: true, b2: false, b4: true}, + {b1: true, b2: false, b4: true} + ]); + + BooleanRecord11Array|csv:Error bm1br11 = csv:transform([bm1, bm1], {}, BooleanRecord11Array); + test:assertEquals(bm1br11, [ + {b1: true, b2: false, defaultableField: "", nillableField :null}, + {b1: true, b2: false, defaultableField: "", nillableField :null} + ]); + + BooleanRecord11Array|csv:Error bm2br11 = csv:transform([bm2, bm2], {}, BooleanRecord11Array); + test:assertEquals(bm2br11, [ + {b1: true, b2: false, b3: (), n1: (), n3: (), defaultableField: "", nillableField :null}, + {b1: true, b2: false, b3: (), n1: (), n3: (), defaultableField: "", nillableField :null} + ]); + + BooleanRecord11Array|csv:Error bm3br11 = csv:transform([bm3, bm3], {}, BooleanRecord11Array); + test:assertEquals(bm3br11, [ + {b1: true, b2: false, b3: (), b4: false, defaultableField: "", nillableField :null}, + {b1: true, b2: false, b3: (), b4: false, defaultableField: "", nillableField :null} + ]); + + BooleanRecord11Array|csv:Error bm4br11 = csv:transform([bm4, bm4], {}, BooleanRecord11Array); + test:assertTrue(bm4br11 is csv:Error); + test:assertEquals((bm4br11).message(), common:generateErrorMessageForMissingRequiredField("b1")); + + BooleanRecord11Array|csv:Error bm5br11 = csv:transform([bm5, bm5], {}, BooleanRecord11Array); + test:assertEquals(bm5br11, [ + {b1: true, b2: false, b3: (), b4: true, defaultableField: "", nillableField :null}, + {b1: true, b2: false, b3: (), b4: true, defaultableField: "", nillableField :null} + ]); + + BooleanRecord12Array|csv:Error bm1br12 = csv:transform([bm1, bm1], {}, BooleanRecord12Array); + test:assertTrue(bm1br12 is csv:Error); + test:assertEquals((bm1br12).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord12Array|csv:Error bm2br12 = csv:transform([bm2, bm2], {}, BooleanRecord12Array); + test:assertTrue(bm2br12 is csv:Error); + test:assertEquals((bm2br12).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord12Array|csv:Error bm3br12 = csv:transform([bm3, bm3], {}, BooleanRecord12Array); + test:assertTrue(bm3br12 is csv:Error); + test:assertEquals((bm3br12).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord12Array|csv:Error bm4br12 = csv:transform([bm4, bm4], {}, BooleanRecord12Array); + test:assertTrue(bm4br12 is csv:Error); + test:assertEquals((bm4br12).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord12Array|csv:Error bm5br12 = csv:transform([bm5, bm5], {}, BooleanRecord12Array); + test:assertTrue(bm5br12 is csv:Error); + test:assertEquals((bm5br12).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord13Array|csv:Error bm1br13 = csv:transform([bm1, bm1], {}, BooleanRecord13Array); + test:assertEquals(bm1br13, [ + {b1: true, b2: false, defaultableField: "", nillableField :null}, + {b1: true, b2: false, defaultableField: "", nillableField :null} + ]); + + BooleanRecord13Array|csv:Error bm2br13 = csv:transform([bm2, bm2], {}, BooleanRecord13Array); + test:assertEquals(bm2br13, [ + {b1: true, b2: false, defaultableField: "", nillableField :null}, + {b1: true, b2: false, defaultableField: "", nillableField :null} + ]); + + BooleanRecord13Array|csv:Error bm3br13 = csv:transform([bm3, bm3], {}, BooleanRecord13Array); + test:assertEquals(bm3br13, [ + {b1: true, b2: false, b4: false, defaultableField: "", nillableField :null}, + {b1: true, b2: false, b4: false, defaultableField: "", nillableField :null} + ]); + + BooleanRecord13Array|csv:Error bm4br13 = csv:transform([bm4, bm4], {}, BooleanRecord13Array); + test:assertEquals(bm4br13, [ + {defaultableField: "", nillableField :null}, + {defaultableField: "", nillableField :null} + ]); +} + +@test:Config +function testFromCsvWithTypeForMapAndRecordAsExpectedType4() { + BooleanRecord13Array|csv:Error bm5br13 = csv:transform([bm5, bm5], {}, BooleanRecord13Array); + test:assertEquals(bm5br13, [ + {b1: true, b2: false, b4: true, defaultableField: "", nillableField :null}, + {b1: true, b2: false, b4: true, defaultableField: "", nillableField :null} + ]); + + BooleanRecord14Array|csv:Error bm1br14 = csv:transform([bm1, bm1], {}, BooleanRecord14Array); + test:assertTrue(bm1br14 is csv:Error); + test:assertEquals((bm1br14).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord14Array|csv:Error bm2br14 = csv:transform([bm2, bm2], {}, BooleanRecord14Array); + test:assertTrue(bm2br14 is csv:Error); + test:assertEquals((bm2br14).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord14Array|csv:Error bm3br14 = csv:transform([bm3, bm3], {}, BooleanRecord14Array); + test:assertTrue(bm3br14 is csv:Error); + test:assertEquals((bm3br14).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord14Array|csv:Error bm4br14 = csv:transform([bm4, bm4], {}, BooleanRecord14Array); + test:assertTrue(bm4br14 is csv:Error); + test:assertEquals((bm4br14).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord14Array|csv:Error bm5br14 = csv:transform([bm5, bm5], {}, BooleanRecord14Array); + test:assertTrue(bm5br14 is csv:Error); + test:assertEquals((bm5br14).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord15Array|csv:Error bm1br15 = csv:transform([bm1, bm1], {}, BooleanRecord15Array); + test:assertTrue(bm1br15 is csv:Error); + test:assertEquals((bm1br15).message(), common:generateErrorMessageForInvalidFieldType("true", "b1")); + + BooleanRecord15Array|csv:Error bm3br15 = csv:transform([bm3, bm3], {}, BooleanRecord15Array); + test:assertTrue(bm3br15 is csv:Error); + test:assertEquals((bm3br15).message(), common:generateErrorMessageForInvalidFieldType("true", "b1")); + + BooleanRecord15Array|csv:Error bm4br15 = csv:transform([bm4, bm4], {}, BooleanRecord15Array); + test:assertTrue(bm4br15 is csv:Error); + test:assertEquals((bm4br15).message(), common:generateErrorMessageForMissingRequiredField("b1")); + + BooleanRecord16Array|csv:Error bm1br16 = csv:transform([bm1, bm1], {}, BooleanRecord16Array); + test:assertEquals(bm1br16, [ + {b1: true, b2: false}, + {b1: true, b2: false} + ]); + + BooleanRecord16Array|csv:Error bm2br16 = csv:transform([bm2, bm2], {}, BooleanRecord16Array); + test:assertEquals(bm2br16, [ + {b1: true, b2: false, b3: (), n1: (), n3: ()}, + {b1: true, b2: false, b3: (), n1: (), n3: ()} + ]); + + BooleanRecord16Array|csv:Error bm3br16 = csv:transform([bm3, bm3], {}, BooleanRecord16Array); + test:assertEquals(bm3br16, [ + {b1: true, b2: false, b4: false, b3: ()}, + {b1: true, b2: false, b4: false, b3: ()} + ]); + + BooleanRecord16Array|csv:Error bm4br16 = csv:transform([bm4, bm4], {}, BooleanRecord16Array); + test:assertEquals(bm4br16, [ + {n1: (), n3: ()}, + {n1: (), n3: ()} + ]); + + BooleanRecord16Array|csv:Error bm5br16 = csv:transform([bm5, bm5], {}, BooleanRecord16Array); + test:assertEquals(bm5br16, [ + {b1: true, b2: false, b4: true, b3: ()}, + {b1: true, b2: false, b4: true, b3: ()} + ]); + + BooleanRecord17Array|csv:Error bm1br17 = csv:transform([bm1, bm1], {}, BooleanRecord17Array); + test:assertEquals(bm1br17, [ + {}, + {} + ]); + + BooleanRecord17Array|csv:Error bm2br17 = csv:transform([bm2, bm2], {}, BooleanRecord17Array); + test:assertEquals(bm2br17, [ + {}, + {} + ]); + + BooleanRecord17Array|csv:Error bm3br17 = csv:transform([bm3, bm3], {}, BooleanRecord17Array); + test:assertEquals(bm3br17, [ + {i1: 1}, + {i1: 1} + ]); + + BooleanRecord17Array|csv:Error bm4br17 = csv:transform([bm4, bm4], {}, BooleanRecord17Array); + test:assertEquals(bm4br17, [ + {}, + {} + ]); + + BooleanRecord17Array|csv:Error bm5br17 = csv:transform([bm5, bm5], {}, BooleanRecord17Array); + test:assertEquals(bm5br17, [ + {}, + {} + ]); + + BooleanRecord18Array|csv:Error bm1br18 = csv:transform([bm1, bm1], {}, BooleanRecord18Array); + test:assertEquals(bm1br18, [ + {b2: false}, + {b2: false} + ]); + + BooleanRecord18Array|csv:Error bm2br18 = csv:transform([bm2, bm2], {}, BooleanRecord18Array); + test:assertEquals(bm2br18, [ + {b2: false, b3: (), n1: (), n3: ()}, + {b2: false, b3: (), n1: (), n3: ()} + ]); + + BooleanRecord18Array|csv:Error bm3br18 = csv:transform([bm3, bm3], {}, BooleanRecord18Array); + test:assertEquals(bm3br18, [ + {b2: false, b3: (), i1: 1}, + {b2: false, b3: (), i1: 1} + ]); + + BooleanRecord18Array|csv:Error bm4br18 = csv:transform([bm4, bm4], {}, BooleanRecord18Array); + test:assertTrue(bm4br18 is csv:Error); + test:assertEquals((bm4br18).message(), common:generateErrorMessageForMissingRequiredField("b2")); + + BooleanRecord18Array|csv:Error bm5br18 = csv:transform([bm5, bm5], {}, BooleanRecord18Array); + test:assertEquals(bm5br18, [ + {b2: false, b3: ()}, + {b2: false, b3: ()} + ]); +} + +@test:Config +function testFromCsvWithTypeForMapAndMapAsExpectedType() { + BooleanMapArray|csv:Error bm1bma = csv:transform([bm1, bm1], {}, BooleanMapArray); + test:assertEquals(bm1bma, [ + {b1: true, b2: false}, + {b1: true, b2: false} + ]); + + BooleanMapArray|csv:Error bm2bma = csv:transform([bm2, bm2], {}, BooleanMapArray); + test:assertEquals(bm2bma, [ + {b1: true, b2: false}, + {b1: true, b2: false} + ]); + + BooleanMapArray|csv:Error bm3bma = csv:transform([bm3, bm3], {}, BooleanMapArray); + test:assertEquals(bm3bma, [ + {b1: true, b2: false, b4: false}, + {b1: true, b2: false, b4: false} + ]); + + BooleanMapArray|csv:Error bm4bma = csv:transform([bm4, bm4], {}, BooleanMapArray); + test:assertEquals(bm4bma, [ + {}, + {} + ]); + + BooleanMapArray|csv:Error bm5bma = csv:transform([bm5, bm5], {}, BooleanMapArray); + test:assertEquals(bm5bma, [ + {b1: true, b2: false, b4: true}, + {b1: true, b2: false, b4: true} + ]); + + NillableBooleanMapArray|csv:Error bm1nbma = csv:transform([bm1, bm1], {}, NillableBooleanMapArray); + test:assertEquals(bm1nbma, [ + {b1: true, b2: false}, + {b1: true, b2: false} + ]); + + NillableBooleanMapArray|csv:Error bm2nbma = csv:transform([bm2, bm2], {}, NillableBooleanMapArray); + test:assertEquals(bm2nbma, [ + {b1: true, b2: false, b3:(), n1: (), n3: ()}, + {b1: true, b2: false, b3:(), n1: (), n3: ()} + ]); + + NillableBooleanMapArray|csv:Error bm3nbma = csv:transform([bm3, bm3], {}, NillableBooleanMapArray); + test:assertEquals(bm3nbma, [ + {b1: true, b2: false, b3:(), b4: false}, + {b1: true, b2: false, b3:(), b4: false} + ]); + + NillableBooleanMapArray|csv:Error bm4nbma = csv:transform([bm4, bm4], {}, NillableBooleanMapArray); + test:assertEquals(bm4nbma, [ + {n1: (), n3: ()}, + {n1: (), n3: ()} + ]); + + NillableBooleanMapArray|csv:Error bm5nbma = csv:transform([bm5, bm5], {}, NillableBooleanMapArray); + test:assertEquals(bm5nbma, [ + {b1: true, b2: false, b3: (), b4: true}, + {b1: true, b2: false, b3: (), b4: true} + ]); + + NillableIntUnionBooleanMapArray|csv:Error bm1niubma = csv:transform([bm1, bm1], {}, NillableIntUnionBooleanMapArray); + test:assertEquals(bm1niubma, [ + {b1: true, b2: false}, + {b1: true, b2: false} + ]); + + NillableIntUnionBooleanMapArray|csv:Error bm2niubma = csv:transform([bm2, bm2], {}, NillableIntUnionBooleanMapArray); + test:assertEquals(bm2niubma, [ + {b1: true, b2: false, b3:(), n1: (), n3: ()}, + {b1: true, b2: false, b3:(), n1: (), n3: ()} + ]); + + NillableIntUnionBooleanMapArray|csv:Error bm3niubma = csv:transform([bm3, bm3], {}, NillableIntUnionBooleanMapArray); + test:assertEquals(bm3niubma, [ + {b1: true, b2: false, b3:(), b4: false, i1: 1}, + {b1: true, b2: false, b3:(), b4: false, i1: 1} + ]); + + NillableIntUnionBooleanMapArray|csv:Error bm4niubma = csv:transform([bm4, bm4], {}, NillableIntUnionBooleanMapArray); + test:assertEquals(bm4niubma, [ + {n1: (), n3: ()}, + {n1: (), n3: ()} + ]); + + NillableIntUnionBooleanMapArray|csv:Error bm5niubma = csv:transform([bm5, bm5], {}, NillableIntUnionBooleanMapArray); + test:assertEquals(bm5niubma, [ + {b1: true, b2: false, b3: (), b4: true}, + {b1: true, b2: false, b3: (), b4: true} + ]); + + IntUnionBooleanMapArray|csv:Error bm1iubma = csv:transform([bm1, bm1], {}, IntUnionBooleanMapArray); + test:assertEquals(bm1iubma, [ + {b1: true, b2: false}, + {b1: true, b2: false} + ]); + + IntUnionBooleanMapArray|csv:Error bm2iubma = csv:transform([bm2, bm2], {}, IntUnionBooleanMapArray); + test:assertEquals(bm2iubma, [ + {b1: true, b2: false}, + {b1: true, b2: false} + ]); + + IntUnionBooleanMapArray|csv:Error bm3iubma = csv:transform([bm3, bm3], {}, IntUnionBooleanMapArray); + test:assertEquals(bm3iubma, [ + {b1: true, b2: false, b4: false, i1: 1}, + {b1: true, b2: false, b4: false, i1: 1} + ]); + + IntUnionBooleanMapArray|csv:Error bm4iubma = csv:transform([bm4, bm4], {}, IntUnionBooleanMapArray); + test:assertEquals(bm4iubma, [ + {}, + {} + ]); + + IntUnionBooleanMapArray|csv:Error bm5iubma = csv:transform([bm5, bm5], {}, IntUnionBooleanMapArray); + test:assertEquals(bm5iubma, [ + {b1: true, b2: false, b4: true}, + {b1: true, b2: false, b4: true} + ]); + + NilMapArray|csv:Error bm1nma = csv:transform([bm1, bm1], {}, NilMapArray); + test:assertEquals(bm1nma, [ + {}, + {} + ]); + + NilMapArray|csv:Error bm2nma = csv:transform([bm2, bm2], {}, NilMapArray); + test:assertEquals(bm2nma, [ + {n1: (), n3: (), b3: ()}, + {n1: (), n3: (), b3: ()} + ]); +} + +@test:Config +function testFromCsvWithTypeForMapAndMapAsExpectedType2() { + NilMapArray|csv:Error bm3nma = csv:transform([bm3, bm3], {}, NilMapArray); + test:assertEquals(bm3nma, [ + {b3: ()}, + {b3: ()} + ]); + + NilMapArray|csv:Error bm4nma = csv:transform([bm4, bm4], {}, NilMapArray); + test:assertEquals(bm4nma, [ + {n1: (), n3: ()}, + {n1: (), n3: ()} + ]); + NilMapArray|csv:Error bm5nma = csv:transform([bm5, bm5], {}, NilMapArray); + test:assertEquals(bm5nma, [ + {b3: ()}, + {b3: ()} + ]); + + JsonMapArray|csv:Error bm1jma = csv:transform([bm1, bm1], {}, JsonMapArray); + test:assertEquals(bm1jma, [ + {b1: true, b2: false}, + {b1: true, b2: false} + ]); + + JsonMapArray|csv:Error bm2jma = csv:transform([bm2, bm2], {}, JsonMapArray); + test:assertEquals(bm2jma, [ + {b1: true, b2: false, b3: (), n1: (), n3: ()}, + {b1: true, b2: false, b3: (), n1: (), n3: ()} + ]); + + JsonMapArray|csv:Error bm3jma = csv:transform([bm3, bm3], {}, JsonMapArray); + test:assertEquals(bm3jma, [ + {b1: true, b2: false, b4: false, b3: (), i1: 1}, + {b1: true, b2: false, b4: false, b3: (), i1: 1} + ]); + + JsonMapArray|csv:Error bm4jma = csv:transform([bm4, bm4], {}, JsonMapArray); + test:assertEquals(bm4jma, [ + {n1: (), n3: ()}, + {n1: (), n3: ()} + ]); + + JsonMapArray|csv:Error bm5jma = csv:transform([bm5, bm5], {}, JsonMapArray); + test:assertEquals(bm5jma, [ + {b1: true, b2: false, b4: true, b3: ()}, + {b1: true, b2: false, b4: true, b3: ()} + ]); + + AnydataMapArray|csv:Error bm1anydma = csv:transform([bm1, bm1], {}, AnydataMapArray); + test:assertEquals(bm1anydma, [ + {b1: true, b2: false}, + {b1: true, b2: false} + ]); + + AnydataMapArray|csv:Error bm2anydma = csv:transform([bm2, bm2], {}, AnydataMapArray); + test:assertEquals(bm2anydma, [ + {b1: true, b2: false, b3: (), n1: (), n3: ()}, + {b1: true, b2: false, b3: (), n1: (), n3: ()} + ]); + + AnydataMapArray|csv:Error bm3anydma = csv:transform([bm3, bm3], {}, AnydataMapArray); + test:assertEquals(bm3anydma, [ + {b1: true, b2: false, b4: false, b3: (), i1: 1}, + {b1: true, b2: false, b4: false, b3: (), i1: 1} + ]); + + AnydataMapArray|csv:Error bm4anydma = csv:transform([bm4, bm4], {}, AnydataMapArray); + test:assertEquals(bm4anydma, [ + {n1: (), n3: ()}, + {n1: (), n3: ()} + ]); + + AnydataMapArray|csv:Error bm5anydma = csv:transform([bm5, bm5], {}, AnydataMapArray); + test:assertEquals(bm5anydma, [ + {b1: true, b2: false, b4: true, b3: ()}, + {b1: true, b2: false, b4: true, b3: ()} + ]); + + CustomMapArray|csv:Error bm1cma = csv:transform([bm1, bm1], {}, CustomMapArray); + test:assertEquals(bm1cma, [ + {}, + {} + ]); + + CustomMapArray|csv:Error bm2cma = csv:transform([bm2, bm2], {}, CustomMapArray); + test:assertEquals(bm2cma, [ + {}, + {} + ]); + + CustomMapArray|csv:Error bm3cma = csv:transform([bm3, bm3], {}, CustomMapArray); + test:assertEquals(bm3cma, [ + {i1: 1}, + {i1: 1} + ]); + + CustomMapArray|csv:Error bm4cma = csv:transform([bm4, bm4], {}, CustomMapArray); + test:assertEquals(bm4cma, [ + {}, + {} + ]); + + CustomMapArray|csv:Error bm5cma = csv:transform([bm5, bm5], {}, CustomMapArray); + test:assertEquals(bm5cma, [ + {}, + {} + ]); + + StringMapArray|csv:Error bm1sma = csv:transform([bm1, bm1], {}, StringMapArray); + test:assertEquals(bm1sma, [ + {}, + {} + ]); + + StringMapArray|csv:Error bm2sma = csv:transform([bm2, bm2], {}, StringMapArray); + test:assertEquals(bm2sma, [ + {}, + {} + ]); + + StringMapArray|csv:Error bm3sma = csv:transform([bm3, bm3], {}, StringMapArray); + test:assertEquals(bm3sma, [ + {}, + {} + ]); + + StringMapArray|csv:Error bm4sma = csv:transform([bm4, bm4], {}, StringMapArray); + test:assertEquals(bm4sma, [ + {}, + {} + ]); + + StringMapArray|csv:Error bm5sma = csv:transform([bm5, bm5], {}, StringMapArray); + test:assertEquals(bm5sma, [ + {}, + {} + ]); +} + +@test:Config +function testArrayIndexesInRecords() { + map[] csv = [{a: 1, b: 2}, {a: 3, b: 4}, {a: 5, b: 6}, {a: 7, b: 8}]; + + record {}[2]|csv:Error rec = csv:transform(csv); + test:assertEquals(rec, [ + {a: 1, b: 2}, + {a: 3, b: 4} + ]); + + record {|int a;|}[2]|csv:Error rec2 = csv:transform(csv, {skipLines: [2]}); + test:assertEquals(rec2, [ + {a: 1}, + {a: 5} + ]); + + record {|int a;|}[5]|csv:Error rec2_2 = csv:transform(csv, {skipLines: [2]}); + test:assertTrue(rec2_2 is csv:Error); +} \ No newline at end of file diff --git a/ballerina-tests/parse-record-types-tests/tests/test_data_values.bal b/ballerina-tests/parse-record-types-tests/tests/test_data_values.bal new file mode 100644 index 0000000..547a3cb --- /dev/null +++ b/ballerina-tests/parse-record-types-tests/tests/test_data_values.bal @@ -0,0 +1,31 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +final boolean b1 = true; +final false b2 = false; +final boolean? b3 = (); +final boolean|int b4 = false; + +final () n1 = (); +final int? n2 = (); +final () n3 = null; + +final int i1 = 1; +final map bm1 = {b1, b2}; +final map bm2 = {b1, b2, b3, n1, n3}; +final map bm3 = {b1, b2, b3, b4, i1}; +final map<()> bm4 = {n1, n3}; +final map bm5 = {b1, b2, b3, b4:true}; diff --git a/ballerina-tests/parse-record-types-tests/tests/types.bal b/ballerina-tests/parse-record-types-tests/tests/types.bal new file mode 100644 index 0000000..1d0a8ba --- /dev/null +++ b/ballerina-tests/parse-record-types-tests/tests/types.bal @@ -0,0 +1,377 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +type BooleanRecord1 record { + boolean b1; + boolean|string b2; + boolean|string? b3; + boolean b4; +}; + +type BooleanRecord2 record {| + boolean b1; + boolean|string b2; + boolean|string? b3; + boolean b4; +|}; + +type BooleanRecord3 record {| + boolean b1; + boolean? b3; +|}; + +type BooleanRecord4 record { + boolean b1; + boolean? b3; +}; + +type BooleanRecord5 record { + boolean b1; + boolean? b3; + string defaultableField = ""; + string? nillableField = (); +}; + +type BooleanRecord6 record {| + boolean b1; + boolean? b3; + string defaultableField = ""; + string? nillableField = (); +|}; + +type BooleanRecord7 record { + boolean b1; + boolean? b3; + string defaultableField = ""; + string? nillableField = (); + string requiredField; +}; + +type BooleanRecord8 record {| + boolean b1; + boolean? b3; + string defaultableField = ""; + string? nillableField = (); + string requiredField; +|}; + +type BooleanRecord9 record {| + boolean b1; + boolean? b3; + boolean?...; +|}; + +type BooleanRecord10 record {| + boolean...; +|}; + +type BooleanRecord11 record {| + boolean b1; + string defaultableField = ""; + string? nillableField = (); + boolean?|string...; +|}; + +type BooleanRecord12 record {| + boolean b1; + string defaultableField = ""; + string? nillableField = (); + string requiredField; + boolean...; +|}; + +type BooleanRecord13 record {| + string defaultableField = ""; + string? nillableField = (); + string|boolean...; +|}; + +type BooleanRecord14 record {| + string defaultableField = ""; + string? nillableField = (); + string requiredField; + boolean...; +|}; + +type BooleanRecord15 record {| + int b1; + string defaultableField = ""; + string? nillableField = (); + boolean?...; +|}; + +type BooleanRecord16 record {| + boolean?...; +|}; + +type BooleanRecord17 record {| + int...; +|}; + +type BooleanRecord18 record {| + boolean b2; + int?...; +|}; + +type BooleanTuple1 [boolean, boolean, boolean, boolean]; + +type BooleanTuple2 [boolean, boolean]; + +type BooleanTuple3 [boolean, boolean...]; + +type BooleanTuple4 [boolean...]; + +type NillableBooleanTuple5 [boolean?, boolean?, boolean?, boolean?, boolean?]; + +type NillableBooleanTuple6 [boolean?, boolean?]; + +type NillableBooleanTuple7 [boolean?, boolean?, boolean?...]; + +type NillableBooleanTuple8 [boolean?...]; + +type NillableIntBooleanTuple9 [int|boolean?, int|boolean?...]; + +type NilTuple3 [(), ()...]; + +type StringTuple3 [string, string...]; + +type AnydataTuple3 [anydata, anydata...]; + +type JsonTuple3 [json, json...]; + +type StringArray string[]; + +type StringArray1 string[]; + +type BooleanArray boolean[]; + +type NillableBooleanArray boolean?[]; + +type NillableIntOrUnionBooleanArray (int|boolean?)[]; + +type JsonArray1 json[]; + +type AnydataArray1 anydata[]; + +type IntegerMap map; + +type StringMap map; + +type NillableIntUnionStringMap map; + +type IntUnionStringMap map; + +type DecimalMap map; + +type FloatMap map; + +type BooleanMap map; + +type NillableBooleanMap map; + +type NillableIntUnionBooleanMap map; + +type IntUnionBooleanMap map; + +type NilMap map<()>; + +type JsonMap map; + +type AnydataMap map; + +type CustomMap map; + +type BooleanRecord1Array BooleanRecord1[]; + +type ClosedBooleanRecord1Array BooleanRecord1[3]; + +type BooleanRecord2Array BooleanRecord2[]; + +type ClosedBooleanRecord2Array BooleanRecord2[3]; + +type BooleanRecord3Array BooleanRecord3[]; + +type ClosedBooleanRecord3Array BooleanRecord3[3]; + +type BooleanRecord4Array BooleanRecord4[]; + +type ClosedBooleanRecord4Array BooleanRecord4[3]; + +type BooleanRecord5Array BooleanRecord5[]; + +type ClosedBooleanRecord5Array BooleanRecord5[3]; + +type BooleanRecord6Array BooleanRecord6[]; + +type ClosedBooleanRecord6Array BooleanRecord6[3]; + +type BooleanRecord7Array BooleanRecord7[]; + +type ClosedBooleanRecord7Array BooleanRecord7[3]; + +type BooleanRecord8Array BooleanRecord8[]; + +type ClosedBooleanRecord8Array BooleanRecord8[3]; + +type BooleanRecord9Array BooleanRecord9[]; + +type ClosedBooleanRecord9Array BooleanRecord9[3]; + +type BooleanRecord10Array BooleanRecord10[]; + +type ClosedBooleanRecord10Array BooleanRecord10[3]; + +type BooleanRecord11Array BooleanRecord11[]; + +type ClosedBooleanRecord11Array BooleanRecord11[3]; + +type BooleanRecord12Array BooleanRecord12[]; + +type ClosedBooleanRecord12Array BooleanRecord12[3]; + +type BooleanRecord13Array BooleanRecord13[]; + +type ClosedBooleanRecord13Array BooleanRecord13[3]; + +type BooleanRecord14Array BooleanRecord14[]; + +type ClosedBooleanRecord14Array BooleanRecord14[3]; + +type BooleanRecord15Array BooleanRecord15[]; + +type ClosedBooleanRecord15Array BooleanRecord15[3]; + +type BooleanRecord16Array BooleanRecord16[]; + +type ClosedBooleanRecord16Array BooleanRecord16[3]; + +type BooleanRecord17Array BooleanRecord17[]; + +type ClosedBooleanRecord17Array BooleanRecord17[3]; + +type BooleanRecord18Array BooleanRecord18[]; + +type ClosedBooleanRecord18Array BooleanRecord18[3]; + +type BooleanTuple1Array BooleanTuple1[]; + +type ClosedBooleanTuple1Array BooleanTuple1[3]; + +type BooleanTuple2Array BooleanTuple2[]; + +type ClosedBooleanTuple2Array BooleanTuple2[3]; + +type BooleanTuple3Array BooleanTuple3[]; + +type ClosedBooleanTuple3Array BooleanTuple3[3]; + +type BooleanTuple4Array BooleanTuple4[]; + +type ClosedBooleanTuple4Array BooleanTuple4[3]; + +type NillableBooleanTuple5Array NillableBooleanTuple5[]; + +type NillableBooleanTuple6Array NillableBooleanTuple6[]; + +type NillableBooleanTuple7Array NillableBooleanTuple7[]; + +type NillableBooleanTuple8Array NillableBooleanTuple8[]; + +type NillableIntBooleanTuple9Array NillableIntBooleanTuple9[]; + +type NilTuple3Array NilTuple3[]; + +type ClosedNilTuple3Array NilTuple3[3]; + +type StringTuple3Array StringTuple3[]; + +type ClosedStringTuple3Array StringTuple3[3]; + +type AnydataTuple3Array AnydataTuple3[]; + +type ClosedAnydataTuple3Array AnydataTuple3[3]; + +type JsonTuple3Array JsonTuple3[]; + +type ClosedJsonTuple3Array JsonTuple3[3]; + +type StringArray1Array StringArray1[]; + +type StringArrayArray StringArray[]; + +type ClosedStringArray1Array StringArray1[3]; + +type BooleanArrayArray BooleanArray[]; + +type ClosedBooleanArrayArray BooleanArray[3]; + +type NillableBooleanArrayArray NillableBooleanArray[]; + +type NillableIntOrUnionBooleanArrayArray NillableIntOrUnionBooleanArray[]; + +type JsonArray1Array JsonArray1[]; + +type ClosedJsonArray1Array JsonArray1[3]; + +type AnydataArray1Array AnydataArray1[]; + +type ClosedAnydataArray1Array AnydataArray1[3]; + +type IntegerMapArray IntegerMap[]; + +type ClosedIntegerMapArray IntegerMap[3]; + +type StringMapArray StringMap[]; + +type NillableIntUnionStringMapArray NillableIntUnionStringMap[]; + +type IntUnionStringMapArray IntUnionStringMap[]; + +type ClosedStringMapArray StringMap[3]; + +type DecimalMapArray DecimalMap[]; + +type ClosedDecimalMapArray DecimalMap[3]; + +type FloatMapArray FloatMap[]; + +type ClosedFloatMapArray FloatMap[3]; + +type BooleanMapArray BooleanMap[]; + +type NillableBooleanMapArray NillableBooleanMap[]; + +type NillableIntUnionBooleanMapArray NillableIntUnionBooleanMap[]; + +type IntUnionBooleanMapArray IntUnionBooleanMap[]; + +type ClosedBooleanMapArray BooleanMap[3]; + +type NilMapArray NilMap[]; + +type ClosedNilMapArray NilMap[3]; + +type JsonMapArray JsonMap[]; + +type ClosedJsonMapArray JsonMap[3]; + +type AnydataMapArray AnydataMap[]; + +type ClosedAnydataMapArray AnydataMap[3]; + +type CustomMapArray CustomMap[]; + +type ClosedCustomMapArray CustomMap[3]; diff --git a/ballerina-tests/parse-string-array-types-tests/Ballerina.toml b/ballerina-tests/parse-string-array-types-tests/Ballerina.toml new file mode 100644 index 0000000..d1d42fe --- /dev/null +++ b/ballerina-tests/parse-string-array-types-tests/Ballerina.toml @@ -0,0 +1,16 @@ +[package] +org = "ballerina" +name = "parse_string_array_types_tests" +version = "0.1.0" + +[[dependency]] +org = "ballerina" +name = "csv_commons" +repository = "local" +version = "0.1.0" + +[platform.java17] +graalvmCompatible = true + +[build-options] +graalvmBuildOptions = "-H:+IncludeAllLocales" diff --git a/ballerina-tests/parse-string-array-types-tests/Dependencies.toml b/ballerina-tests/parse-string-array-types-tests/Dependencies.toml new file mode 100644 index 0000000..e07bf86 --- /dev/null +++ b/ballerina-tests/parse-string-array-types-tests/Dependencies.toml @@ -0,0 +1,98 @@ +# AUTO-GENERATED FILE. DO NOT MODIFY. + +# This file is auto-generated by Ballerina for managing dependency versions. +# It should not be modified by hand. + +[ballerina] +dependencies-toml-version = "2" +distribution-version = "2201.10.0-20240801-104200-87df251c" + +[[package]] +org = "ballerina" +name = "csv_commons" +version = "0.1.0" +scope = "testOnly" +modules = [ + {org = "ballerina", packageName = "csv_commons", moduleName = "csv_commons"} +] + +[[package]] +org = "ballerina" +name = "data.csv" +version = "0.1.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] +modules = [ + {org = "ballerina", packageName = "data.csv", moduleName = "data.csv"} +] + +[[package]] +org = "ballerina" +name = "jballerina.java" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "lang.__internal" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.object"} +] + +[[package]] +org = "ballerina" +name = "lang.array" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.__internal"} +] + +[[package]] +org = "ballerina" +name = "lang.error" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "lang.object" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "parse_string_array_types_tests" +version = "0.1.0" +dependencies = [ + {org = "ballerina", name = "csv_commons"}, + {org = "ballerina", name = "data.csv"}, + {org = "ballerina", name = "test"} +] +modules = [ + {org = "ballerina", packageName = "parse_string_array_types_tests", moduleName = "parse_string_array_types_tests"} +] + +[[package]] +org = "ballerina" +name = "test" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.array"}, + {org = "ballerina", name = "lang.error"} +] +modules = [ + {org = "ballerina", packageName = "test", moduleName = "test"} +] + diff --git a/ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_array_test.bal b/ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_array_test.bal new file mode 100644 index 0000000..96612e2 --- /dev/null +++ b/ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_array_test.bal @@ -0,0 +1,480 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/csv_commons as common; +import ballerina/data.csv; +import ballerina/test; + +@test:Config +function testFromCsvStringWithTypeForStringAndArrayAsExpectedType() { + BooleanArrayArray|csv:Error cv1baa = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cv1baa, [ + [true, false, true, false], + [true, false, true, false], + [true, false, true, false] + ]); + + BooleanArrayArray|csv:Error cv2baa = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cv2baa, [ + [true, false, true, false, true], + [true, false, true, false, true] + ]); + + BooleanArrayArray|csv:Error cv3baa = csv:parseString(csvStringWithBooleanValues3); + test:assertTrue(cv3baa is csv:Error); + test:assertEquals((cv3baa).message(), common:generateErrorMessageForInvalidCast("()", "boolean")); + + BooleanArrayArray|csv:Error cv4baa = csv:parseString(csvStringWithBooleanValues4); + test:assertTrue(cv4baa is csv:Error); + test:assertEquals((cv4baa).message(), common:generateErrorMessageForInvalidCast("()", "boolean")); + + BooleanArrayArray|csv:Error cv5baa = csv:parseString(csvStringWithBooleanValues5); + test:assertTrue(cv5baa is csv:Error); + test:assertEquals((cv5baa).message(), common:generateErrorMessageForInvalidCast("2", "boolean")); + + BooleanArrayArray|csv:Error cv6baa = csv:parseString(csvStringWithBooleanValues6); + test:assertTrue(cv6baa is csv:Error); + test:assertEquals((cv6baa).message(), common:generateErrorMessageForInvalidCast("()", "boolean")); + + BooleanArrayArray|csv:Error cv7baa = csv:parseString(csvStringWithBooleanValues7); + test:assertTrue(cv7baa is csv:Error); + test:assertEquals((cv7baa).message(), common:generateErrorMessageForInvalidCast("()", "boolean")); + + NillableBooleanArrayArray|csv:Error cv1nbaa = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cv1nbaa, [ + [true, false, true, false], + [true, false, true, false], + [true, false, true, false] + ]); + + NillableBooleanArrayArray|csv:Error cv2nbaa = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cv2nbaa, [ + [true, false, true, false, true], + [true, false, true, false, true] + ]); + + NillableBooleanArrayArray|csv:Error cv3nbaa = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(cv3nbaa, [ + [true, false, true], + [true, false, ()], + [true, true, false] + ]); + + NillableBooleanArrayArray|csv:Error cv4nbaa = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(cv4nbaa, [ + [true, (), (), false], + [true, (), (), false] + ]); + + NillableBooleanArrayArray|csv:Error cv5nbaa = csv:parseString(csvStringWithBooleanValues5); + test:assertTrue(cv5nbaa is csv:Error); + test:assertEquals((cv5nbaa).message(), common:generateErrorMessageForInvalidCast("2", "boolean?")); + + NillableBooleanArrayArray|csv:Error cv6nbaa = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(cv6nbaa, [ + [(), ()] + ]); + + NillableBooleanArrayArray|csv:Error cv7nbaa = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(cv7nbaa, [ + [b1, b2, (), b4] + ]); + + NillableIntOrUnionBooleanArrayArray|csv:Error cv1niubaa = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cv1niubaa, [ + [true, false, true, false], + [true, false, true, false], + [true, false, true, false] + ]); + + NillableIntOrUnionBooleanArrayArray|csv:Error cv2niubaa = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cv2niubaa, [ + [true, false, true, false, true], + [true, false, true, false, true] + ]); + + NillableIntOrUnionBooleanArrayArray|csv:Error cv3niubaa = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(cv3niubaa, [ + [true, false, true], + [true, false, ()], + [true, true, false] + ]); + + NillableIntOrUnionBooleanArrayArray|csv:Error cv4niubaa = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(cv4niubaa, [ + [true, (), (), false], + [true, (), (), false] + ]); + + NillableIntOrUnionBooleanArrayArray|csv:Error cv5niubaa = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(cv5niubaa, [ + [true, false, true, 2], + [true, false, true, 3] + ]); +} + +@test:Config +function testFromCsvStringWithTypeForStringAndArrayAsExpectedType2() { + NillableIntOrUnionBooleanArrayArray|csv:Error cv6niubaa = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(cv6niubaa, [ + [(), ()] + ]); + + NillableIntOrUnionBooleanArrayArray|csv:Error cv7niubaa = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(cv7niubaa, [ + [b1, b2, (), b4] + ]); + + StringArray1Array|csv:Error cv1saa = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cv1saa, [ + ["true", "false", "true", "false"], + ["true", "false", "true", "false"], + ["true", "false", "true", "false"] + ]); + + StringArray1Array|csv:Error cv2saa = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cv2saa, [ + ["true", "false", "true", "false", "true"], + ["true", "false", "true", "false", "true"] + ]); + + StringArray1Array|csv:Error cv3saa = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(cv3saa, [ + ["true", "false", "true"], + ["TRUE", "FALSE", "()"], + ["true", "true", "FALSE"] + ]); + + StringArray1Array|csv:Error cv4saa = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(cv4saa, [ + ["true", "()", "()", "false"], + ["true", "()", "null", "false"] + ]); + + StringArray1Array|csv:Error cv5saa = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(cv5saa, [ + ["true", "false", "true", "2"], + ["true", "false", "true", "3"] + ]); + + StringArray1Array|csv:Error cv6saa = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(cv6saa, [ + ["()", "()"] + ]); + + StringArray1Array|csv:Error cv7saa = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(cv7saa, [ + ["true", "false", "()", "false"] + ]); + + StringArray2Array|csv:Error cv1s2aa = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cv1s2aa, [ + ["true", "false"], + ["true", "false"], + ["true", "false"] + ]); + + StringArray2Array|csv:Error cv2s2aa = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cv2s2aa, [ + ["true", "false"], + ["true", "false"] + ]); + + StringArray2Array|csv:Error cv3s2aa = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(cv3s2aa, [ + ["true", "false"], + ["TRUE", "FALSE"], + ["true", "true"] + ]); + + StringArray2Array|csv:Error cv4s2aa = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(cv4s2aa, [ + ["true", "()"], + ["true", "()"] + ]); + + StringArray2Array|csv:Error cv5s2aa = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(cv5s2aa, [ + ["true", "false"], + ["true", "false"] + ]); + + StringArray2Array|csv:Error cv6s2aa = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(cv6s2aa, [ + ["()", "()"] + ]); + + StringArray2Array|csv:Error cv7s2aa = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(cv7s2aa, [ + ["true", "false"] + ]); + + JsonArray1Array|csv:Error cv1jaa = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cv1jaa, [ + [true, false, true, false], + [true, false, true, false], + [true, false, true, false] + ]); +} + +@test:Config +function testFromCsvStringWithTypeForStringAndArrayAsExpectedType3() { + JsonArray1Array|csv:Error cv2jaa = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cv2jaa, [ + [true, false, true, false, true], + [true, false, true, false, true] + ]); + + JsonArray1Array|csv:Error cv3jaa = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(cv3jaa, [ + [true, false, true], + [true, false, ()], + [true, true, false] + ]); + + JsonArray1Array|csv:Error cv4jaa = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(cv4jaa, [ + [true, (), (), false], + [true, (), (), false] + ]); + + JsonArray1Array|csv:Error cv5jaa = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(cv5jaa, [ + [true, false, true, 2], + [true, false, true, 3] + ]); + + JsonArray1Array|csv:Error cv6jaa = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(cv6jaa, [ + [(), ()] + ]); + + JsonArray1Array|csv:Error cv7jaa = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(cv7jaa, [ + [b1, b2, (), b4] + ]); + + AnydataArray1Array|csv:Error cv1anydaa = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cv1anydaa, [ + [true, false, true, false], + [true, false, true, false], + [true, false, true, false] + ]); + + AnydataArray1Array|csv:Error cv2anydaa = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cv2anydaa, [ + [true, false, true, false, true], + [true, false, true, false, true] + ]); + + AnydataArray1Array|csv:Error cv3anydaa = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(cv3anydaa, [ + [true, false, true], + [true, false, ()], + [true, true, false] + ]); + + AnydataArray1Array|csv:Error cv4anydaa = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(cv4anydaa, [ + [true, (), (), false], + [true, (), (), false] + ]); + + AnydataArray1Array|csv:Error cv5anydaa = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(cv5anydaa, [ + [true, false, true, 2], + [true, false, true, 3] + ]); + + AnydataArray1Array|csv:Error cv6anydaa = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(cv6anydaa, [ + [(), ()] + ]); + + AnydataArray1Array|csv:Error cv7anydaa = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(cv7anydaa, [ + [b1, b2, (), b4] + ]); + + DecimalArray1Array|csv:Error cv1daa = csv:parseString(csvStringWithBooleanValues1); + test:assertTrue(cv1daa is csv:Error); + test:assertEquals((cv1daa).message(), common:generateErrorMessageForInvalidCast("true", "decimal")); + + DecimalArray1Array|csv:Error cv3daa = csv:parseString(csvStringWithBooleanValues3); + test:assertTrue(cv3daa is csv:Error); + test:assertEquals((cv3daa).message(), common:generateErrorMessageForInvalidCast("true", "decimal")); + + DecimalArray1Array|csv:Error cv6daa = csv:parseString(csvStringWithBooleanValues6); + test:assertTrue(cv6daa is csv:Error); + test:assertEquals((cv6daa).message(), common:generateErrorMessageForInvalidCast("()", "decimal")); + + DecimalArray1Array|csv:Error cv7daa = csv:parseString(csvStringWithBooleanValues7); + test:assertTrue(cv7daa is csv:Error); + test:assertEquals((cv7daa).message(), common:generateErrorMessageForInvalidCast("true", "decimal")); +} + +@test:Config +function testArrayIndexes() { + string csv = string `a, b + 1, 2 + 3, 4 + 5, 6 + 7, 8`; + + string csv2 = string `a, b + 1, 2, 3 + 3, 4, 5 + 5, 6, 7 + 7, 8, 9`; + + record {}[2]|csv:Error rec = csv:parseString(csv); + test:assertEquals(rec, [ + {a: 1, b: 2}, + {a: 3, b: 4} + ]); + + map[2]|csv:Error rec_2 = csv:parseString(csv); + test:assertEquals(rec_2, [ + {a: 1, b: 2}, + {a: 3, b: 4} + ]); + + record {|int a;|}[2]|csv:Error rec2 = csv:parseString(csv, {skipLines: [2]}); + test:assertEquals(rec2, [ + {a: 1}, + {a: 5} + ]); + + record {|int a;|}[5]|csv:Error rec2_2 = csv:parseString(csv, {skipLines: [2]}); + test:assertTrue(rec2_2 is csv:Error); + + int[][2]|csv:Error rec3 = csv:parseString(csv2); + test:assertEquals(rec3, [ + [1, 2], + [3, 4], + [5, 6], + [7, 8] + ]); + + [int, int][2]|csv:Error rec3_2 = csv:parseString(csv2); + test:assertEquals(rec3_2, [ + [1, 2], + [3, 4] + ]); + + [int...][2]|csv:Error rec3_3 = csv:parseString(csv2); + test:assertEquals(rec3_3, [ + [1, 2, 3], + [3, 4, 5] + ]); + + int[1][2]|csv:Error rec4 = csv:parseString(csv2, {skipLines: [2]}); + test:assertEquals(rec4, [ + [1, 2] + ]); + + int[2][]|csv:Error rec5 = csv:parseString(csv2); + test:assertEquals(rec5, [ + [1, 2, 3], + [3, 4, 5] + ]); +} + +@test:Config +function testParseStringArrayAsExpectedTypeWithOutputHeaders() { + BooleanArrayArray|csv:Error cv1baa = csv:parseString(csvStringWithBooleanValues1, {outputWithHeaders: true}); + test:assertTrue(cv1baa is csv:Error); + test:assertEquals((cv1baa).message(), common:generateErrorMessageForInvalidCast("b1", "boolean")); + + string[][]|csv:Error cv1baa_2 = csv:parseString(csvStringWithBooleanValues1, {outputWithHeaders: true}); + test:assertEquals(cv1baa_2, [ + ["b1", "b2", "b3", "b4"], + ["true", "false", "true", "false"], + ["true", "false", "true", "false"], + ["true", "false", "true", "false"] + ]); + + (boolean|string)[][]|csv:Error cv2baa = csv:parseString(csvStringWithBooleanValues2, {outputWithHeaders: true}); + test:assertEquals(cv2baa, [ + ["b1", "b2", "b3", "b4", "b5"], + [true, false, true, false, true], + [true, false, true, false, true] + ]); + + (string|boolean)[][]|csv:Error cv2baa_2 = csv:parseString(csvStringWithBooleanValues2, {outputWithHeaders: true}); + test:assertEquals(cv2baa_2, [ + ["b1", "b2", "b3", "b4", "b5"], + ["true", "false", "true", "false", "true"], + ["true", "false", "true", "false", "true"] + ]); + + [string...][]|csv:Error cv2baa_2_2 = csv:parseString(csvStringWithBooleanValues2, {outputWithHeaders: true}); + test:assertEquals(cv2baa_2_2, [ + ["b1", "b2", "b3", "b4", "b5"], + ["true", "false", "true", "false", "true"], + ["true", "false", "true", "false", "true"] + ]); + + [boolean|string...][]|csv:Error cv2baa_3 = csv:parseString(csvStringWithBooleanValues2, {outputWithHeaders: true}); + test:assertEquals(cv2baa_3, [ + ["b1", "b2", "b3", "b4", "b5"], + [true, false, true, false, true], + [true, false, true, false, true] + ]); + + string[][]|csv:Error cv1baa_4 = csv:parseString(csvStringWithBooleanValues1, {outputWithHeaders: true}); + test:assertEquals(cv1baa_4, [ + ["b1", "b2", "b3", "b4"], + ["true", "false", "true", "false"], + ["true", "false", "true", "false"], + ["true", "false", "true", "false"] + ]); + + string[][]|csv:Error cv1baa_5 = csv:parseString(csvStringWithBooleanValues1, {outputWithHeaders: true, header: 2}); + test:assertTrue(cv1baa_5 is csv:Error); + test:assertEquals((cv1baa_5).message(), "duplicate header found: 'true'"); + + string[][]|csv:Error cv1baa_6 = csv:parseString(csvStringWithBooleanValues8, {outputWithHeaders: false, header: 2}); + test:assertEquals(cv1baa_6, [ + ["true", "false", "true1", "false1"] + ]); + + [string, string, string, string, string][]|csv:Error cv2baa_7 = csv:parseString(csvStringWithBooleanValues2, {outputWithHeaders: true}); + test:assertEquals(cv2baa_7, [ + ["b1", "b2", "b3", "b4", "b5"], + ["true", "false", "true", "false", "true"], + ["true", "false", "true", "false", "true"] + ]); + + [boolean|string, boolean|string...][]|csv:Error cv2baa_8 = csv:parseString(csvStringWithBooleanValues2, {outputWithHeaders: true}); + test:assertEquals(cv2baa_8, [ + ["b1", "b2", "b3", "b4", "b5"], + [true, false, true, false, true], + [true, false, true, false, true] + ]); + + string[2][2]|csv:Error cv1baa_9 = csv:parseString(csvStringWithBooleanValues1, {outputWithHeaders: true, header: ()}); + test:assertEquals(cv1baa_9, [ + ["b1", "b2"], + ["true", "false"] + ]); + + cv1baa_9 = csv:parseString(csvStringWithBooleanValues1, {outputWithHeaders: true, header: null}); + test:assertEquals(cv1baa_9, [ + ["b1", "b2"], + ["true", "false"] + ]); +} diff --git a/ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_tuple_test.bal b/ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_tuple_test.bal new file mode 100644 index 0000000..a43009d --- /dev/null +++ b/ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_tuple_test.bal @@ -0,0 +1,488 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/csv_commons as common; +import ballerina/data.csv; +import ballerina/test; + +@test:Config +function testFromCsvStringWithTypeForStringAndTupleAsExpectedType() { + BooleanTuple1Array|csv:Error cbv1bt1 = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cbv1bt1, [ + [true, false, true, false], + [true, false, true, false], + [true, false, true, false] + ]); + + BooleanTuple1Array|csv:Error cbv2bt1 = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cbv2bt1, [ + [true, false, true, false], + [true, false, true, false] + ]); + + BooleanTuple1Array|csv:Error cbv3bt1 = csv:parseString(csvStringWithBooleanValues3); + test:assertTrue(cbv3bt1 is csv:Error); + test:assertEquals((cbv3bt1).message(), common:generateErrorMessageForInvalidCast("()", "boolean")); + + BooleanTuple1Array|csv:Error cbv5bt1 = csv:parseString(csvStringWithBooleanValues5); + test:assertTrue(cbv5bt1 is csv:Error); + test:assertEquals((cbv5bt1).message(), common:generateErrorMessageForInvalidCast("2", "boolean")); + + BooleanTuple1Array|csv:Error cbv7bt1 = csv:parseString(csvStringWithBooleanValues7); + test:assertTrue(cbv7bt1 is csv:Error); + test:assertEquals((cbv7bt1).message(), common:generateErrorMessageForInvalidCast("()", "boolean")); + + BooleanTuple2Array|csv:Error cbv1bt2 = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cbv1bt2, [ + [true, false], + [true, false], + [true, false] + ]); + + BooleanTuple2Array|csv:Error cbv2bt2 = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cbv2bt2, [ + [true, false], + [true, false] + ]); + + BooleanTuple2Array|csv:Error cbv3bt2 = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(cbv3bt2, [ + [true, false], + [true, false], + [true, true] + ]); + + BooleanTuple2Array|csv:Error cbv4bt2 = csv:parseString(csvStringWithBooleanValues4); + test:assertTrue(cbv4bt2 is csv:Error); + test:assertEquals((cbv4bt2).message(), common:generateErrorMessageForInvalidCast("()", "boolean")); + + BooleanTuple3Array|csv:Error cbv1bt3 = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cbv1bt3, [ + [true, false, true, false], + [true, false, true, false], + [true, false, true, false] + ]); + + BooleanTuple3Array|csv:Error cbv2bt3 = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cbv2bt3, [ + [true, false, true, false, true], + [true, false, true, false, true] + ]); + + BooleanTuple3Array|csv:Error cbv3bt3 = csv:parseString(csvStringWithBooleanValues3); + test:assertTrue(cbv3bt3 is csv:Error); + test:assertEquals((cbv3bt3).message(), common:generateErrorMessageForInvalidCast("()", "boolean")); + + BooleanTuple3Array|csv:Error cbv5bt3 = csv:parseString(csvStringWithBooleanValues5); + test:assertTrue(cbv5bt3 is csv:Error); + test:assertEquals((cbv5bt3).message(), common:generateErrorMessageForInvalidCast("2", "boolean")); + + BooleanTuple3Array|csv:Error cbv7bt3 = csv:parseString(csvStringWithBooleanValues7); + test:assertTrue(cbv7bt3 is csv:Error); + test:assertEquals((cbv7bt3).message(), common:generateErrorMessageForInvalidCast("()", "boolean")); + + BooleanTuple4Array|csv:Error cbv1bt4 = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cbv1bt4, [ + [true, false, true, false], + [true, false, true, false], + [true, false, true, false] + ]); + + BooleanTuple4Array|csv:Error cbv2bt4 = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cbv2bt4, [ + [true, false, true, false, true], + [true, false, true, false, true] + ]); + + BooleanTuple4Array|csv:Error cbv3bt4 = csv:parseString(csvStringWithBooleanValues3); + test:assertTrue(cbv3bt4 is csv:Error); + test:assertEquals((cbv3bt4).message(), common:generateErrorMessageForInvalidCast("()", "boolean")); + + BooleanTuple4Array|csv:Error cbv4bt4 = csv:parseString(csvStringWithBooleanValues4); + test:assertTrue(cbv4bt4 is csv:Error); + test:assertEquals((cbv4bt4).message(), common:generateErrorMessageForInvalidCast("()", "boolean")); + + BooleanTuple4Array|csv:Error cbv5bt4 = csv:parseString(csvStringWithBooleanValues5); + test:assertTrue(cbv5bt4 is csv:Error); + test:assertEquals((cbv5bt4).message(), common:generateErrorMessageForInvalidCast("2", "boolean")); + + BooleanTuple4Array|csv:Error cbv6bt4 = csv:parseString(csvStringWithBooleanValues6); + test:assertTrue(cbv6bt4 is csv:Error); + test:assertEquals((cbv6bt4).message(), common:generateErrorMessageForInvalidCast("()", "boolean")); + +} + +@test:Config +function testFromCsvStringWithTypeForStringAndTupleAsExpectedType2() { + BooleanTuple4Array|csv:Error cbv7bt4 = csv:parseString(csvStringWithBooleanValues7); + test:assertTrue(cbv7bt4 is csv:Error); + test:assertEquals((cbv7bt4).message(), common:generateErrorMessageForInvalidCast("()", "boolean")); + + NillableBooleanTuple5Array|csv:Error cbv1bt5 = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cbv1bt5, [ + [true, false, true, false, null], + [true, false, true, false, null], + [true, false, true, false, null] + ]); + + NillableBooleanTuple5Array|csv:Error cbv2bt5 = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cbv2bt5, [ + [true, false, true, false, true], + [true, false, true, false, true] + ]); + + NillableBooleanTuple5Array|csv:Error cbv3bt5 = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(cbv3bt5, [ + [true, false, true, null, null], + [true, false, (), null, null], + [true, true, false, null, null] + ]); + + NillableBooleanTuple5Array|csv:Error cbv4bt5 = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(cbv4bt5, [ + [true, (), (), false, null], + [true, (), (), false, null] + ]); + + NillableBooleanTuple5Array|csv:Error cbv5bt5 = csv:parseString(csvStringWithBooleanValues5); + test:assertTrue(cbv5bt5 is csv:Error); + test:assertEquals((cbv5bt5).message(), common:generateErrorMessageForInvalidCast("2", "boolean?")); + + NillableBooleanTuple5Array|csv:Error cbv6bt5 = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(cbv6bt5, [ + [(), (), null, null, null] + ]); + + NillableBooleanTuple5Array|csv:Error cbv7bt5 = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(cbv7bt5, [ + [b1, b2, (), b4, null] + ]); + + NillableBooleanTuple6Array|csv:Error cbv1bt6 = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cbv1bt6, [ + [true, false], + [true, false], + [true, false] + ]); + + NillableBooleanTuple6Array|csv:Error cbv2bt6 = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cbv2bt6, [ + [true, false], + [true, false] + ]); + + NillableBooleanTuple6Array|csv:Error cbv3bt6 = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(cbv3bt6, [ + [true, false], + [true, false], + [true, true] + ]); + + NillableBooleanTuple6Array|csv:Error cbv4bt6 = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(cbv4bt6, [ + [true, ()], + [true, ()] + ]); + + NillableBooleanTuple6Array|csv:Error cbv6bt6 = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(cbv6bt6, [ + [(), null] + ]); + + NillableBooleanTuple6Array|csv:Error cbv7bt6 = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(cbv7bt6, [ + [b1, b2] + ]); + + NillableBooleanTuple7Array|csv:Error cbv1bt7 = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cbv1bt7, [ + [true, false, true, false], + [true, false, true, false], + [true, false, true, false] + ]); + + NillableBooleanTuple7Array|csv:Error cbv2bt7 = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cbv2bt7, [ + [true, false, true, false, true], + [true, false, true, false, true] + ]); + + NillableBooleanTuple7Array|csv:Error cbv3bt7 = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(cbv3bt7, [ + [true, false, true], + [true, false, ()], + [true, true, false] + ]); + + NillableBooleanTuple7Array|csv:Error cbv4bt7 = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(cbv4bt7, [ + [true, (), (), false], + [true, (), (), false] + ]); + +} +@test:Config +function testFromCsvStringWithTypeForStringAndTupleAsExpectedType3() { + NillableBooleanTuple7Array|csv:Error cbv5bt7 = csv:parseString(csvStringWithBooleanValues5); + test:assertTrue(cbv5bt7 is csv:Error); + test:assertEquals((cbv5bt7).message(), common:generateErrorMessageForInvalidCast("2", "boolean?")); + + NillableBooleanTuple7Array|csv:Error cbv6bt7 = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(cbv6bt7, [ + [(), ()] + ]); + + NillableBooleanTuple7Array|csv:Error cbv7bt7 = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(cbv7bt7, [ + [b1, b2, (), false] + ]); + + NillableBooleanTuple8Array|csv:Error cbv1bt8 = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cbv1bt8, [ + [true, false, true, false], + [true, false, true, false], + [true, false, true, false] + ]); + + NillableBooleanTuple8Array|csv:Error cbv2bt8 = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cbv2bt8, [ + [true, false, true, false, true], + [true, false, true, false, true] + ]); + + NillableBooleanTuple8Array|csv:Error cbv3bt8 = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(cbv3bt8, [ + [true, false, true], + [true, false, ()], + [true, true, false] + ]); + + NillableBooleanTuple8Array|csv:Error cbv4bt8 = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(cbv4bt8, [ + [true, (), (), false], + [true, (), (), false] + ]); + + NillableBooleanTuple8Array|csv:Error cbv5bt8 = csv:parseString(csvStringWithBooleanValues5); + test:assertTrue(cbv5bt8 is csv:Error); + test:assertEquals((cbv5bt8).message(), common:generateErrorMessageForInvalidCast("2", "boolean?")); + + NillableBooleanTuple8Array|csv:Error cbv6bt8 = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(cbv6bt8, [ + [(), ()] + ]); + + NillableBooleanTuple8Array|csv:Error cbv7bt8 = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(cbv7bt8, [ + [b1, b2, (), false] + ]); + + NillableIntBooleanTuple9Array|csv:Error cbv1bt9 = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cbv1bt9, [ + [true, false, true, false], + [true, false, true, false], + [true, false, true, false] + ]); + + NillableIntBooleanTuple9Array|csv:Error cbv2bt9 = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cbv2bt9, [ + [true, false, true, false, true], + [true, false, true, false, true] + ]); + + NillableIntBooleanTuple9Array|csv:Error cbv3bt9 = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(cbv3bt9, [ + [true, false, true], + [true, false, ()], + [true, true, false] + ]); + + NillableIntBooleanTuple9Array|csv:Error cbv4bt9 = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(cbv4bt9, [ + [true, (), (), false], + [true, (), (), false] + ]); + + NillableIntBooleanTuple9Array|csv:Error cbv5bt9 = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(cbv5bt9, [ + [true, false, true, 2], + [true, false, true, 3] + ]); + + NillableIntBooleanTuple9Array|csv:Error cbv6bt9 = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(cbv6bt9, [ + [(), ()] + ]); + + NillableIntBooleanTuple9Array|csv:Error cbv7bt9 = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(cbv7bt9, [ + [b1, b2, (), false] + ]); + + NilTuple3Array|csv:Error cbv1nt3 = csv:parseString(csvStringWithBooleanValues1); + test:assertTrue(cbv1nt3 is csv:Error); + test:assertEquals((cbv1nt3).message(), common:generateErrorMessageForInvalidCast("true", "()")); + + NilTuple3Array|csv:Error cbv3nt3 = csv:parseString(csvStringWithBooleanValues3); + test:assertTrue(cbv3nt3 is csv:Error); + test:assertEquals((cbv3nt3).message(), common:generateErrorMessageForInvalidCast("true", "()")); + + NilTuple3Array|csv:Error cbv6nt3 = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(cbv6nt3, [ + [(), ()] + ]); + + AnydataTuple3Array|csv:Error cbv1anyd3 = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cbv1anyd3, [ + [true, false, true, false], + [true, false, true, false], + [true, false, true, false] + ]); + + AnydataTuple3Array|csv:Error cbv2anyd3 = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cbv2anyd3, [ + [true, false, true, false, true], + [true, false, true, false, true] + ]); + + AnydataTuple3Array|csv:Error cbv3anyd3 = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(cbv3anyd3, [ + [true, false, true], + [true, false, ()], + [true, true, false] + ]); + + AnydataTuple3Array|csv:Error cbv4anyd3 = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(cbv4anyd3, [ + [true, (), (), false], + [true, (), (), false] + ]); + + AnydataTuple3Array|csv:Error cbv5anyd3 = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(cbv5anyd3, [ + [true, false, true, 2], + [true, false, true, 3] + ]); + + AnydataTuple3Array|csv:Error cbv6anyd3 = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(cbv6anyd3, [ + [(), ()] + ]); + + AnydataTuple3Array|csv:Error cbv7anyd3 = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(cbv7anyd3, [ + [b1, b2, (), false] + ]); +} + +@test:Config +function testFromCsvStringWithTypeForStringAndTupleAsExpectedType4() { + JsonTuple3Array|csv:Error cbv1j3 = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cbv1j3, [ + [true, false, true, false], + [true, false, true, false], + [true, false, true, false] + ]); + + JsonTuple3Array|csv:Error cbv2j3 = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cbv2j3, [ + [true, false, true, false, true], + [true, false, true, false, true] + ]); + + JsonTuple3Array|csv:Error cbv3j3 = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(cbv3j3, [ + [true, false, true], + [true, false, ()], + [true, true, false] + ]); + + JsonTuple3Array|csv:Error cbv4j3 = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(cbv4j3, [ + [true, (), (), false], + [true, (), (), false] + ]); + + JsonTuple3Array|csv:Error cbv5j3 = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(cbv5j3, [ + [true, false, true, 2], + [true, false, true, 3] + ]); + + JsonTuple3Array|csv:Error cbv6j3 = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(cbv6j3, [ + [(), ()] + ]); + + JsonTuple3Array|csv:Error cbv7j3 = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(cbv7j3, [ + [b1, b2, (), false] + ]); + + StringTuple3Array|csv:Error cbv1s3 = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(cbv1s3, [ + ["true", "false", "true", "false"], + ["true", "false", "true", "false"], + ["true", "false", "true", "false"] + ]); + + StringTuple3Array|csv:Error cbv2s3 = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(cbv2s3, [ + ["true", "false", "true", "false", "true"], + ["true", "false", "true", "false", "true"] + ]); + + StringTuple3Array|csv:Error cbv3s3 = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(cbv3s3, [ + ["true", "false", "true"], + ["TRUE", "FALSE", "()"], + ["true", "true", "FALSE"] + ]); + + StringTuple3Array|csv:Error cbv4s3 = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(cbv4s3, [ + ["true", "()", "()", "false"], + ["true", "()", "null", "false"] + ]); + + StringTuple3Array|csv:Error cbv5s3 = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(cbv5s3, [ + ["true", "false", "true", "2"], + ["true", "false", "true", "3"] + ]); + + StringTuple3Array|csv:Error cbv6s3 = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(cbv6s3, [ + ["()", "()"] + ]); + + StringTuple3Array|csv:Error cbv7s3 = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(cbv7s3, [ + ["true", "false", "()", "false"] + ]); + + DecimalTuple3Array|csv:Error cbv1dt3 = csv:parseString(csvStringWithBooleanValues1); + test:assertTrue(cbv1dt3 is csv:Error); + test:assertEquals((cbv1dt3).message(), common:generateErrorMessageForInvalidCast("true", "decimal")); + + DecimalTuple3Array|csv:Error cbv3dt3 = csv:parseString(csvStringWithBooleanValues3); + test:assertTrue(cbv3dt3 is csv:Error); + test:assertEquals((cbv3dt3).message(), common:generateErrorMessageForInvalidCast("true", "decimal")); + + DecimalTuple3Array|csv:Error cbv6dt3 = csv:parseString(csvStringWithBooleanValues6); + test:assertTrue(cbv6dt3 is csv:Error); + test:assertEquals((cbv6dt3).message(), common:generateErrorMessageForInvalidCast("()", "decimal")); +} diff --git a/ballerina-tests/parse-string-array-types-tests/tests/test_data_values.bal b/ballerina-tests/parse-string-array-types-tests/tests/test_data_values.bal new file mode 100644 index 0000000..b39e526 --- /dev/null +++ b/ballerina-tests/parse-string-array-types-tests/tests/test_data_values.bal @@ -0,0 +1,68 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +final boolean b1 = true; +final false b2 = false; +final boolean|int b4 = false; + +final string csvStringWithBooleanValues1 = string `b1,b2,b3,b4 +true,false,true,false +true,false, true,false +true,false,true,false +`; + +final string csvStringWithBooleanValues2 = string `b1,b2,b3,b4,b5 +true,false, true,false,true +true,false, true,false,true`; + +final string csvStringWithBooleanValues3 = string `b1,b2,b3 +${" "}${"\t"} +true, false,true +${" "} + TRUE, FALSE,() +${" "} + +true, true,FALSE + +`; + +final string csvStringWithBooleanValues4 = string `b1,b2,b3,b4 + true,(), (),false + true,(), null,false + +`; + +final string csvStringWithBooleanValues5 = string `b1,b2,b3,b4 + +true,false,true,2 + +true,false,true,3 +`; + +final string csvStringWithBooleanValues6 = string `b2,b3 +(),() + +`; + +final string csvStringWithBooleanValues7 = string `b1,b2,b3,b4 +${b1},${b2},(),${b4} +`; + +final string csvStringWithBooleanValues8 = string `b1,b2,b3,b4 +true,false,true1,false1 +true,false, true1,false1 +true,false,true1,false1 +`; diff --git a/ballerina-tests/parse-string-array-types-tests/tests/types.bal b/ballerina-tests/parse-string-array-types-tests/tests/types.bal new file mode 100644 index 0000000..411b2b0 --- /dev/null +++ b/ballerina-tests/parse-string-array-types-tests/tests/types.bal @@ -0,0 +1,63 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +type BooleanArray boolean[]; +type BooleanArrayArray BooleanArray[]; +type NillableIntOrUnionBooleanArray (int|boolean?)[]; +type StringArray1 string[]; +type StringArray2 string[2]; +type JsonArray1 json[]; +type AnydataArray1 anydata[]; +type BooleanTuple1 [boolean, boolean, boolean, boolean]; +type BooleanTuple2 [boolean, boolean]; +type BooleanTuple3 [boolean, boolean...]; +type BooleanTuple4 [boolean...]; +type NillableBooleanTuple5 [boolean?, boolean?, boolean?, boolean?, boolean?]; +type NillableBooleanTuple6 [boolean?, boolean?]; +type NillableBooleanTuple7 [boolean?, boolean?, boolean?...]; +type NillableBooleanTuple8 [boolean?...]; +type NillableIntBooleanTuple9 [int|boolean?, int|boolean?...]; +type NilTuple3 [(), ()...]; +type DecimalTuple3 [decimal, decimal...]; +type StringTuple3 [string, string...]; +type AnydataTuple3 [anydata, anydata...]; +type JsonTuple3 [json, json...]; +type DecimalArray1 decimal[]; + +type NilTuple3Array NilTuple3[]; +type DecimalTuple3Array DecimalTuple3[]; +type StringTuple3Array StringTuple3[]; +type AnydataTuple3Array AnydataTuple3[]; +type JsonTuple3Array JsonTuple3[]; + +type NillableBooleanArray boolean?[]; +type NillableBooleanArrayArray NillableBooleanArray[]; +type NillableIntOrUnionBooleanArrayArray NillableIntOrUnionBooleanArray[]; +type StringArray1Array StringArray1[]; +type StringArray2Array StringArray2[]; +type JsonArray1Array JsonArray1[]; +type AnydataArray1Array AnydataArray1[]; + +type BooleanTuple1Array BooleanTuple1[]; +type BooleanTuple2Array BooleanTuple2[]; +type BooleanTuple3Array BooleanTuple3[]; +type BooleanTuple4Array BooleanTuple4[]; +type NillableBooleanTuple5Array NillableBooleanTuple5[]; +type NillableBooleanTuple6Array NillableBooleanTuple6[]; +type NillableBooleanTuple7Array NillableBooleanTuple7[]; +type NillableBooleanTuple8Array NillableBooleanTuple8[]; +type NillableIntBooleanTuple9Array NillableIntBooleanTuple9[]; +type DecimalArray1Array DecimalArray1[]; diff --git a/ballerina-tests/parse-string-record-types-tests/Ballerina.toml b/ballerina-tests/parse-string-record-types-tests/Ballerina.toml new file mode 100644 index 0000000..795eb35 --- /dev/null +++ b/ballerina-tests/parse-string-record-types-tests/Ballerina.toml @@ -0,0 +1,16 @@ +[package] +org = "ballerina" +name = "parse_string_record_types_tests" +version = "0.1.0" + +[[dependency]] +org = "ballerina" +name = "csv_commons" +repository = "local" +version = "0.1.0" + +[platform.java17] +graalvmCompatible = true + +[build-options] +graalvmBuildOptions = "-H:+IncludeAllLocales" diff --git a/ballerina-tests/parse-string-record-types-tests/Dependencies.toml b/ballerina-tests/parse-string-record-types-tests/Dependencies.toml new file mode 100644 index 0000000..cf6d15e --- /dev/null +++ b/ballerina-tests/parse-string-record-types-tests/Dependencies.toml @@ -0,0 +1,98 @@ +# AUTO-GENERATED FILE. DO NOT MODIFY. + +# This file is auto-generated by Ballerina for managing dependency versions. +# It should not be modified by hand. + +[ballerina] +dependencies-toml-version = "2" +distribution-version = "2201.10.0-20240801-104200-87df251c" + +[[package]] +org = "ballerina" +name = "csv_commons" +version = "0.1.0" +scope = "testOnly" +modules = [ + {org = "ballerina", packageName = "csv_commons", moduleName = "csv_commons"} +] + +[[package]] +org = "ballerina" +name = "data.csv" +version = "0.1.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] +modules = [ + {org = "ballerina", packageName = "data.csv", moduleName = "data.csv"} +] + +[[package]] +org = "ballerina" +name = "jballerina.java" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "lang.__internal" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.object"} +] + +[[package]] +org = "ballerina" +name = "lang.array" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.__internal"} +] + +[[package]] +org = "ballerina" +name = "lang.error" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "lang.object" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "parse_string_record_types_tests" +version = "0.1.0" +dependencies = [ + {org = "ballerina", name = "csv_commons"}, + {org = "ballerina", name = "data.csv"}, + {org = "ballerina", name = "test"} +] +modules = [ + {org = "ballerina", packageName = "parse_string_record_types_tests", moduleName = "parse_string_record_types_tests"} +] + +[[package]] +org = "ballerina" +name = "test" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.array"}, + {org = "ballerina", name = "lang.error"} +] +modules = [ + {org = "ballerina", packageName = "test", moduleName = "test"} +] + diff --git a/ballerina-tests/parse-string-record-types-tests/tests/parse_string_to_map_tests.bal b/ballerina-tests/parse-string-record-types-tests/tests/parse_string_to_map_tests.bal new file mode 100644 index 0000000..7a831e9 --- /dev/null +++ b/ballerina-tests/parse-string-record-types-tests/tests/parse_string_to_map_tests.bal @@ -0,0 +1,425 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/data.csv; +import ballerina/test; + +@test:Config +function testFromCsvStringWithTypeForStringAndMapAsExpectedType() { + BooleanMapArray|csv:Error bv1bma = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(bv1bma, [ + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false} + ]); + + BooleanMapArray|csv:Error bv2bma = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(bv2bma, [ + {b1: true, b2: false, b3: true, b4: false, b5: true}, + {b1: true, b2: false, b3: true, b4: false, b5: true} + ]); + + BooleanMapArray|csv:Error bv3bma = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(bv3bma, [ + {b1: true, b2: false, b3: true}, + {b1: true, b2: false}, + {b1: true, b2: true, b3: false} + ]); + + BooleanMapArray|csv:Error bv4bma = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(bv4bma, [ + {b1: true, b4: false}, + {b1: true, b4: false} + ]); + + BooleanMapArray|csv:Error bv5bma = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(bv5bma, [ + {b1: true, b2: false, b3: true}, + {b1: true, b2: false, b3: true} + ]); + + BooleanMapArray|csv:Error bv6bma = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(bv6bma, [ + {} + ]); + + BooleanMapArray|csv:Error bv7bma = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(bv7bma, [ + {b1, b2, b4} + ]); + + NillableBooleanMapArray|csv:Error bv1bnbma = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(bv1bnbma, [ + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false} + ]); + + NillableBooleanMapArray|csv:Error bv2bnbma = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(bv2bnbma, [ + {b1: true, b2: false, b3: true, b4: false, b5: true}, + {b1: true, b2: false, b3: true, b4: false, b5: true} + ]); + + NillableBooleanMapArray|csv:Error bv3bnbma = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(bv3bnbma, [ + {b1: true, b2: false, b3: true}, + {b1: true, b2: false, b3: null}, + {b1: true, b2: true, b3: false} + ]); + + NillableBooleanMapArray|csv:Error bv4bnbma = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(bv4bnbma, [ + {b1: true, b2: (), b3: (), b4: false}, + {b1: true, b2: (), b3: (), b4: false} + ]); + + NillableBooleanMapArray|csv:Error bv5bnbma = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(bv5bnbma, [ + {b1: true, b2: false, b3: true}, + {b1: true, b2: false, b3: true} + ]); + + NillableBooleanMapArray|csv:Error bv6bnbma = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(bv6bnbma, [ + {b2: (), b3: ()} + ]); + + NillableBooleanMapArray|csv:Error bv7bnbma = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(bv7bnbma, [ + {b1, b2, b3, b4} + ]); + + NillableIntUnionBooleanMapArray|csv:Error bv1bniubma = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(bv1bniubma, [ + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false} + ]); + + NillableIntUnionBooleanMapArray|csv:Error bv2bniubma = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(bv2bniubma, [ + {b1: true, b2: false, b3: true, b4: false, b5: true}, + {b1: true, b2: false, b3: true, b4: false, b5: true} + ]); + +} + +@test:Config +function testFromCsvStringWithTypeForStringAndMapAsExpectedType2() { + NillableIntUnionBooleanMapArray|csv:Error bv3bniubma = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(bv3bniubma, [ + {b1: true, b2: false, b3: true}, + {b1: true, b2: false, b3: null}, + {b1: true, b2: true, b3: false} + ]); + + NillableIntUnionBooleanMapArray|csv:Error bv4bniubma = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(bv4bniubma, [ + {b1: true, b2: (), b3: (), b4: false}, + {b1: true, b2: (), b3: (), b4: false} + ]); + + NillableIntUnionBooleanMapArray|csv:Error bv5bniubma = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(bv5bniubma, [ + {b1: true, b2: false, b3: true, b4: 2}, + {b1: true, b2: false, b3: true, b4: 3} + ]); + + NillableIntUnionBooleanMapArray|csv:Error bv6bniubma = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(bv6bniubma, [ + {b2: (), b3: ()} + ]); + + NillableIntUnionBooleanMapArray|csv:Error bv7bniubma = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(bv7bniubma, [ + {b1, b2, b3, b4} + ]); + + IntUnionBooleanMapArray|csv:Error bv1biubma = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(bv1biubma, [ + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false} + ]); + + IntUnionBooleanMapArray|csv:Error bv2biubma = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(bv2biubma, [ + {b1: true, b2: false, b3: true, b4: false, b5: true}, + {b1: true, b2: false, b3: true, b4: false, b5: true} + ]); + + IntUnionBooleanMapArray|csv:Error bv3biubma = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(bv3biubma, [ + {b1: true, b2: false, b3: true}, + {b1: true, b2: false}, + {b1: true, b2: true, b3: false} + ]); + + IntUnionBooleanMapArray|csv:Error bv4biubma = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(bv4biubma, [ + {b1: true, b4: false}, + {b1: true, b4: false} + ]); + + IntUnionBooleanMapArray|csv:Error bv5biubma = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(bv5biubma, [ + {b1: true, b2: false, b3: true, b4: 2}, + {b1: true, b2: false, b3: true, b4: 3} + ]); + + IntUnionBooleanMapArray|csv:Error bv6biubma = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(bv6biubma, [ + {} + ]); + + IntUnionBooleanMapArray|csv:Error bv7biubma = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(bv7biubma, [ + {b1, b2, b4} + ]); + + NilMapArray|csv:Error bv1bnma = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(bv1bnma, [ + {}, + {}, + {} + ]); + + NilMapArray|csv:Error bv2bnma = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(bv2bnma, [ + {}, + {} + ]); + + NilMapArray|csv:Error bv3bnma = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(bv3bnma, [ + {}, + {b3: ()}, + {} + ]); + + NilMapArray|csv:Error bv4bnma = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(bv4bnma, [ + {b2: (), b3: ()}, + {b2: (), b3: ()} + ]); + + NilMapArray|csv:Error bv5bnma = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(bv5bnma, [ + {}, + {} + ]); +} + +@test:Config +function testFromCsvStringWithTypeForStringAndMapAsExpectedType3() { + NilMapArray|csv:Error bv6bnma = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(bv6bnma, [ + {b2: (), b3: ()} + ]); + + NilMapArray|csv:Error bv7bnma = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(bv7bnma, [ + {b3} + ]); + + JsonMapArray|csv:Error bv1bjma = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(bv1bjma, [ + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false} + ]); + + JsonMapArray|csv:Error bv2bjma = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(bv2bjma, [ + {b1: true, b2: false, b3: true, b4: false, b5: true}, + {b1: true, b2: false, b3: true, b4: false, b5: true} + ]); + + JsonMapArray|csv:Error bv3bjma = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(bv3bjma, [ + {b1: true, b2: false, b3: true}, + {b1: true, b2: false, b3: null}, + {b1: true, b2: true, b3: false} + ]); + + JsonMapArray|csv:Error bv4bjma = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(bv4bjma, [ + {b1: true, b2: (), b3: (), b4: false}, + {b1: true, b2: (), b3: (), b4: false} + ]); + + JsonMapArray|csv:Error bv5bjma = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(bv5bjma, [ + {b1: true, b2: false, b3: true, b4: 2}, + {b1: true, b2: false, b3: true, b4: 3} + ]); + + JsonMapArray|csv:Error bv6bjma = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(bv6bjma, [ + {b2: (), b3: ()} + ]); + + JsonMapArray|csv:Error bv7bjma = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(bv7bjma, [ + {b1, b2, b3, b4} + ]); + + AnydataMapArray|csv:Error bv1banydma = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(bv1banydma, [ + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false} + ]); + + AnydataMapArray|csv:Error bv2banydma = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(bv2banydma, [ + {b1: true, b2: false, b3: true, b4: false, b5: true}, + {b1: true, b2: false, b3: true, b4: false, b5: true} + ]); + + AnydataMapArray|csv:Error bv3banydma = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(bv3banydma, [ + {b1: true, b2: false, b3: true}, + {b1: true, b2: false, b3: null}, + {b1: true, b2: true, b3: false} + ]); + + AnydataMapArray|csv:Error bv4banydma = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(bv4banydma, [ + {b1: true, b2: (), b3: (), b4: false}, + {b1: true, b2: (), b3: (), b4: false} + ]); + + AnydataMapArray|csv:Error bv5banydma = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(bv5banydma, [ + {b1: true, b2: false, b3: true, b4: 2}, + {b1: true, b2: false, b3: true, b4: 3} + ]); + + AnydataMapArray|csv:Error bv6banydma = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(bv6banydma, [ + {b2: (), b3: ()} + ]); + + AnydataMapArray|csv:Error bv7banydma = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(bv7banydma, [ + {b1, b2, b3, b4} + ]); + + CustomMapArray|csv:Error bv1bcma = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(bv1bcma, [ + {b1: "true", b2: "false", b3: "true", b4: "false"}, + {b1: "true", b2: "false", b3: "true", b4: "false"}, + {b1: "true", b2: "false", b3: "true", b4: "false"} + ]); +} + +@test:Config +function testFromCsvStringWithTypeForStringAndMapAsExpectedType4() { + CustomMapArray|csv:Error bv2bcma = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(bv2bcma, [ + {b1: "true", b2: "false", b3: "true", b4: "false", b5: "true"}, + {b1: "true", b2: "false", b3: "true", b4: "false", b5: "true"} + ]); + + CustomMapArray|csv:Error bv3bcma = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(bv3bcma, [ + {b1: "true", b2: "false", b3: "true"}, + {b1: "TRUE", b2: "FALSE", b3: "()"}, + {b1: "true", b2: "true", b3: "FALSE"} + ]); + + CustomMapArray|csv:Error bv4bcma = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(bv4bcma, [ + {b1: "true", b2: "()", b3: "()", b4: "false"}, + {b1: "true", b2: "()", b3: "null", b4: "false"} + ]); + + CustomMapArray|csv:Error bv5bcma = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(bv5bcma, [ + {b1: "true", b2: "false", b3: "true", b4: 2}, + {b1: "true", b2: "false", b3: "true", b4: 3} + ]); + + CustomMapArray|csv:Error bv6bcma = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(bv6bcma, [ + {b2: "()", b3: "()"} + ]); + + CustomMapArray|csv:Error bv7bcma = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(bv7bcma, [ + {b1: "true", b2: "false", b3: "()", b4: "false"} + ]); + + StringMapArray|csv:Error bv1bsma = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(bv1bsma, [ + {b1: "true", b2: "false", b3: "true", b4: "false"}, + {b1: "true", b2: "false", b3: "true", b4: "false"}, + {b1: "true", b2: "false", b3: "true", b4: "false"} + ]); + + StringMapArray|csv:Error bv2bsma = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(bv2bsma, [ + {b1: "true", b2: "false", b3: "true", b4: "false", b5: "true"}, + {b1: "true", b2: "false", b3: "true", b4: "false", b5: "true"} + ]); + + StringMapArray|csv:Error bv3bsma = csv:parseString(csvStringWithBooleanValues3); + test:assertEquals(bv3bsma, [ + {b1: "true", b2: "false", b3: "true"}, + {b1: "TRUE", b2: "FALSE", b3: "()"}, + {b1: "true", b2: "true", b3: "FALSE"} + ]); + + StringMapArray|csv:Error bv4bsma = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(bv4bsma, [ + {b1: "true", b2: "()", b3: "()", b4: "false"}, + {b1: "true", b2: "()", b3: "null", b4: "false"} + ]); + + StringMapArray|csv:Error bv5bsma = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(bv5bsma, [ + {b1: "true", b2: "false", b3: "true", b4: "2"}, + {b1: "true", b2: "false", b3: "true", b4: "3"} + ]); + + StringMapArray|csv:Error bv6bsma = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(bv6bsma, [ + {b2: "()", b3: "()"} + ]); + + StringMapArray|csv:Error bv7bsma = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(bv7bsma, [ + {b1: "true", b2: "false", b3: "()", b4: "false"} + ]); + + DecimalMapArray|csv:Error bv1dsma = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(bv1dsma, [ + {}, + {}, + {} + ]); + DecimalMapArray|csv:Error bv6dsma = csv:parseString(csvStringWithBooleanValues6); + test:assertEquals(bv6dsma, [ + {} + ]); + + DecimalMapArray|csv:Error bv7dsma = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(bv7dsma, [ + {} + ]); +} diff --git a/ballerina-tests/parse-string-record-types-tests/tests/parse_string_to_record_tests.bal b/ballerina-tests/parse-string-record-types-tests/tests/parse_string_to_record_tests.bal new file mode 100644 index 0000000..1a6b01a --- /dev/null +++ b/ballerina-tests/parse-string-record-types-tests/tests/parse_string_to_record_tests.bal @@ -0,0 +1,519 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/csv_commons as common; +import ballerina/data.csv; +import ballerina/test; + +@test:Config +function testFromCsvStringWithTypeForStringAndRecordAsExpectedType() { + BooleanRecord1Array|csv:Error csvb1br1 = csv:parseString(csvStringWithBooleanValues1); + test:assertEquals(csvb1br1, [ + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false} + ]); + + BooleanRecord1Array|csv:Error csvb2br1 = csv:parseString(csvStringWithBooleanValues2, {}); + test:assertEquals(csvb2br1, [ + {b1: true, b2: false, b3: true, b4: false, b5: true}, + {b1: true, b2: false, b3: true, b4: false, b5: true} + ]); + + BooleanRecord1Array|csv:Error csvb3br1 = csv:parseString(csvStringWithBooleanValues3, {}); + test:assertTrue(csvb3br1 is csv:Error); + test:assertEquals((csvb3br1).message(), common:generateErrorMessageForMissingRequiredField("b4")); + + BooleanRecord1Array|csv:Error csvb4br1 = csv:parseString(csvStringWithBooleanValues4, {}); + test:assertEquals(csvb4br1, [ + {b1: true, b2: "()", b3: (), b4: false}, + {b1: true, b2: "()", b3: (), b4: false} + ]); + + BooleanRecord1Array|csv:Error csvb5br1 = csv:parseString(csvStringWithBooleanValues5, {}); + test:assertTrue(csvb5br1 is csv:Error); + test:assertEquals((csvb5br1).message(), common:generateErrorMessageForInvalidCast("2", "boolean")); + + BooleanRecord1Array|csv:Error csvb6br1 = csv:parseString(csvStringWithBooleanValues6, {}); + test:assertTrue(csvb6br1 is csv:Error); + test:assertEquals((csvb6br1).message(), common:generateErrorMessageForMissingRequiredField("b4")); + + BooleanRecord1Array|csv:Error csvb7br1 = csv:parseString(csvStringWithBooleanValues7, {}); + test:assertEquals(csvb7br1, [ + {b1, b2, b3: (), b4} + ]); + + BooleanRecord2Array|csv:Error csvb1br2 = csv:parseString(csvStringWithBooleanValues1, {}); + test:assertEquals(csvb1br2, [ + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false} + ]); + + BooleanRecord2Array|csv:Error csvb2br2 = csv:parseString(csvStringWithBooleanValues2, {}); + test:assertEquals(csvb2br2, [ + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false} + ]); + + BooleanRecord2Array|csv:Error csvb3br2 = csv:parseString(csvStringWithBooleanValues3); + test:assertTrue(csvb3br2 is csv:Error); + test:assertEquals((csvb3br2).message(), common:generateErrorMessageForMissingRequiredField("b4")); + + BooleanRecord2Array|csv:Error csvb4br2 = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(csvb4br2, [ + {b1: true, b2: "()", b3: (), b4: false}, + {b1: true, b2: "()", b3: (), b4: false} + ]); + + BooleanRecord2Array|csv:Error csvb5br2 = csv:parseString(csvStringWithBooleanValues5, {}); + test:assertTrue(csvb5br2 is csv:Error); + test:assertEquals((csvb5br2).message(), common:generateErrorMessageForInvalidCast("2", "boolean")); + + BooleanRecord2Array|csv:Error csvb7br2 = csv:parseString(csvStringWithBooleanValues7, {}); + test:assertEquals(csvb7br2, [ + {b1, b2, b3: (), b4} + ]); + + BooleanRecord3Array|csv:Error csvb1br3 = csv:parseString(csvStringWithBooleanValues1, {}); + test:assertEquals(csvb1br3, [ + {b1: true, b3: true}, + {b1: true, b3: true}, + {b1: true, b3: true} + ]); + + BooleanRecord3Array|csv:Error csvb2br3 = csv:parseString(csvStringWithBooleanValues2, {}); + test:assertEquals(csvb2br3, [ + {b1: true, b3: true}, + {b1: true, b3: true} + ]); + + BooleanRecord3Array|csv:Error csvb3br3 = csv:parseString(csvStringWithBooleanValues3, {}); + test:assertEquals(csvb3br3, [ + {b1: true, b3: true}, + {b1: true, b3: ()}, + {b1: true, b3: false} + ]); + + BooleanRecord3Array|csv:Error csvb4br3 = csv:parseString(csvStringWithBooleanValues4, {}); + test:assertEquals(csvb4br3, [ + {b1: true, b3: ()}, + {b1: true, b3: ()} + ]); + + BooleanRecord3Array|csv:Error csvb5br3 = csv:parseString(csvStringWithBooleanValues5, {}); + test:assertEquals(csvb5br3, [ + {b1: true, b3: true}, + {b1: true, b3: true} + ]); +} + +@test:Config +function testFromCsvStringWithTypeForStringAndRecordAsExpectedType2() { + BooleanRecord3Array|csv:Error csvb7br3 = csv:parseString(csvStringWithBooleanValues7, {}); + test:assertEquals(csvb7br3, [ + {b1, b3: ()} + ]); + + BooleanRecord4Array|csv:Error csvb1br4 = csv:parseString(csvStringWithBooleanValues1, {}); + test:assertEquals(csvb1br4, [ + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false} + ]); + + BooleanRecord4Array|csv:Error csvb2br4 = csv:parseString(csvStringWithBooleanValues2, {}); + test:assertEquals(csvb2br4, [ + {b1: true, b2: false, b3: true, b4: false, b5: true}, + {b1: true, b2: false, b3: true, b4: false, b5: true} + ]); + + BooleanRecord4Array|csv:Error csvb3br4 = csv:parseString(csvStringWithBooleanValues3, {}); + test:assertEquals(csvb3br4, [ + {b1: true, b2: false, b3: true}, + {b1: true, b2: false, b3: ()}, + {b1: true, b2: true, b3: false} + ]); + + BooleanRecord4Array|csv:Error csvb4br4 = csv:parseString(csvStringWithBooleanValues4, {}); + test:assertEquals(csvb4br4, [ + {b1: true, b2: (), b3: (), b4: false}, + {b1: true, b2: (), b3: (), b4: false} + ]); + + BooleanRecord4Array|csv:Error csvb5br4 = csv:parseString(csvStringWithBooleanValues5, {}); + test:assertEquals(csvb5br4, [ + {b1: true, b2: false, b3: true, b4: 2}, + {b1: true, b2: false, b3: true, b4: 3} + ]); + + BooleanRecord4Array|csv:Error csvb7br4 = csv:parseString(csvStringWithBooleanValues7, {}); + test:assertEquals(csvb7br4, [ + {b1, b2, b3: (), b4} + ]); + + BooleanRecord5Array|csv:Error csvb1br5 = csv:parseString(csvStringWithBooleanValues1, {}); + test:assertEquals(csvb1br5, [ + {b1: true, b2: false, b3: true, b4: false, defaultableField: "", nillableField: ()}, + {b1: true, b2: false, b3: true, b4: false, defaultableField: "", nillableField: ()}, + {b1: true, b2: false, b3: true, b4: false, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord5Array|csv:Error csvb2br5 = csv:parseString(csvStringWithBooleanValues2, {}); + test:assertEquals(csvb2br5, [ + {b1: true, b2: false, b3: true, b4: false, b5: true, defaultableField: "", nillableField: ()}, + {b1: true, b2: false, b3: true, b4: false, b5: true, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord5Array|csv:Error csvb3br5 = csv:parseString(csvStringWithBooleanValues3, {}); + test:assertEquals(csvb3br5, [ + {b1: true, b2: false, b3: true, defaultableField: "", nillableField: ()}, + {b1: true, b2: false, b3: (), defaultableField: "", nillableField: ()}, + {b1: true, b2: true, b3: false, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord5Array|csv:Error csvb4br5 = csv:parseString(csvStringWithBooleanValues4, {}); + test:assertEquals(csvb4br5, [ + {b1: true, b2: (), b3: (), b4: false, defaultableField: "", nillableField: ()}, + {b1: true, b2: (), b3: (), b4: false, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord5Array|csv:Error csvb5br5 = csv:parseString(csvStringWithBooleanValues5, {}); + test:assertEquals(csvb5br5, [ + {b1: true, b2: false, b3: true, b4: 2, defaultableField: "", nillableField: ()}, + {b1: true, b2: false, b3: true, b4: 3, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord5Array|csv:Error csvb7br5 = csv:parseString(csvStringWithBooleanValues7, {}); + test:assertEquals(csvb7br5, [ + {b1, b2, b3: (), b4, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord6Array|csv:Error csvb1br6 = csv:parseString(csvStringWithBooleanValues1, {}); + test:assertEquals(csvb1br6, [ + {b1: true, b3: true, defaultableField: "", nillableField: ()}, + {b1: true, b3: true, defaultableField: "", nillableField: ()}, + {b1: true, b3: true, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord6Array|csv:Error csvb2br6 = csv:parseString(csvStringWithBooleanValues2, {}); + test:assertEquals(csvb2br6, [ + {b1: true, b3: true, defaultableField: "", nillableField: ()}, + {b1: true, b3: true, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord6Array|csv:Error csvb3br6 = csv:parseString(csvStringWithBooleanValues3, {}); + test:assertEquals(csvb3br6, [ + {b1: true, b3: true, defaultableField: "", nillableField: ()}, + {b1: true, b3: (), defaultableField: "", nillableField: ()}, + {b1: true, b3: false, defaultableField: "", nillableField: ()} + ]); +} + +@test:Config +function testFromCsvStringWithTypeForStringAndRecordAsExpectedType3() { + BooleanRecord6Array|csv:Error csvb4br6 = csv:parseString(csvStringWithBooleanValues4, {}); + test:assertEquals(csvb4br6, [ + {b1: true, b3: (), defaultableField: "", nillableField: ()}, + {b1: true, b3: (), defaultableField: "", nillableField: ()} + ]); + + BooleanRecord6Array|csv:Error csvb5br6 = csv:parseString(csvStringWithBooleanValues5, {}); + test:assertEquals(csvb5br6, [ + {b1: true, b3: true, defaultableField: "", nillableField: ()}, + {b1: true, b3: true, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord6Array|csv:Error csvb7br6 = csv:parseString(csvStringWithBooleanValues7, {}); + test:assertEquals(csvb7br6, [ + {b1, b3: (), defaultableField: "", nillableField: ()} + ]); + + BooleanRecord7Array|csv:Error csvb1br7 = csv:parseString(csvStringWithBooleanValues1, {}); + test:assertTrue(csvb1br7 is csv:Error); + test:assertEquals((csvb1br7).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord8Array|csv:Error csvb1br8 = csv:parseString(csvStringWithBooleanValues1, {}); + test:assertTrue(csvb1br8 is csv:Error); + test:assertEquals((csvb1br8).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord9Array|csv:Error csvb1br9 = csv:parseString(csvStringWithBooleanValues1, {}); + test:assertEquals(csvb1br9, [ + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false} + ]); + + BooleanRecord9Array|csv:Error csvb2br9 = csv:parseString(csvStringWithBooleanValues2, {}); + test:assertEquals(csvb2br9, [ + {b1: true, b2: false, b3: true, b4: false, b5: true}, + {b1: true, b2: false, b3: true, b4: false, b5: true} + ]); + + BooleanRecord9Array|csv:Error csvb3br9 = csv:parseString(csvStringWithBooleanValues3, {}); + test:assertEquals(csvb3br9, [ + {b1: true, b2: false, b3: true}, + {b1: true, b2: false, b3: ()}, + {b1: true, b2: true, b3: false} + ]); + + BooleanRecord9Array|csv:Error csvb4br9 = csv:parseString(csvStringWithBooleanValues4, {}); + test:assertEquals(csvb4br9, [ + {b1: true, b2: (), b3: (), b4: false}, + {b1: true, b2: (), b3: (), b4: false} + ]); + + BooleanRecord9Array|csv:Error csvb5br9 = csv:parseString(csvStringWithBooleanValues5, {}); + test:assertEquals(csvb5br9, [ + {b1: true, b2: false, b3: true}, + {b1: true, b2: false, b3: true} + ]); + + BooleanRecord9Array|csv:Error csvb6br9 = csv:parseString(csvStringWithBooleanValues6, {}); + test:assertTrue(csvb6br9 is csv:Error); + test:assertEquals((csvb6br9).message(), common:generateErrorMessageForMissingRequiredField("b1")); + + BooleanRecord9Array|csv:Error csvb7br9 = csv:parseString(csvStringWithBooleanValues7, {}); + test:assertEquals(csvb7br9, [ + {b1, b2, b3: (), b4} + ]); + + BooleanRecord10Array|csv:Error csvb1br10 = csv:parseString(csvStringWithBooleanValues1, {}); + test:assertEquals(csvb1br10, [ + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false} + ]); + + BooleanRecord10Array|csv:Error csvb2br10 = csv:parseString(csvStringWithBooleanValues2, {}); + test:assertEquals(csvb2br10, [ + {b1: true, b2: false, b3: true, b4: false, b5: true}, + {b1: true, b2: false, b3: true, b4: false, b5: true} + ]); + + BooleanRecord10Array|csv:Error csvb3br10 = csv:parseString(csvStringWithBooleanValues3, {}); + test:assertEquals(csvb3br10, [ + {b1: true, b2: false, b3: true}, + {b1: true, b2: false}, + {b1: true, b2: true, b3: false} + ]); + + BooleanRecord10Array|csv:Error csvb4br10 = csv:parseString(csvStringWithBooleanValues4, {}); + test:assertEquals(csvb4br10, [ + {b1: true, b4: false}, + {b1: true, b4: false} + ]); + + BooleanRecord10Array|csv:Error csvb5br10 = csv:parseString(csvStringWithBooleanValues5, {}); + test:assertEquals(csvb5br10, [ + {b1: true, b2: false, b3: true}, + {b1: true, b2: false, b3: true} + ]); + + BooleanRecord10Array|csv:Error csvb6br10 = csv:parseString(csvStringWithBooleanValues6, {}); + test:assertEquals(csvb6br10, [ + {} + ]); + + BooleanRecord10Array|csv:Error csvb7br10 = csv:parseString(csvStringWithBooleanValues7, {}); + test:assertEquals(csvb7br10, [ + {b1, b2, b4} + ]); + + BooleanRecord11Array|csv:Error csvb1br11 = csv:parseString(csvStringWithBooleanValues1, {}); + test:assertEquals(csvb1br11, [ + {b1: true, b2: false, b3: true, b4: false, defaultableField: "", nillableField: ()}, + {b1: true, b2: false, b3: true, b4: false, defaultableField: "", nillableField: ()}, + {b1: true, b2: false, b3: true, b4: false, defaultableField: "", nillableField: ()} + ]); +} + +@test:Config +function testFromCsvStringWithTypeForStringAndRecordAsExpectedType4() { + BooleanRecord11Array|csv:Error csvb2br11 = csv:parseString(csvStringWithBooleanValues2); + test:assertEquals(csvb2br11, [ + {b1: true, b2: false, b3: true, b4: false, b5: true, defaultableField: "", nillableField: ()}, + {b1: true, b2: false, b3: true, b4: false, b5: true, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord11Array|csv:Error csvb3br11 = csv:parseString(csvStringWithBooleanValues3, {}); + test:assertEquals(csvb3br11, [ + {b1: true, b2: false, b3: true, defaultableField: "", nillableField: ()}, + {b1: true, b2: false, b3: (), defaultableField: "", nillableField: ()}, + {b1: true, b2: true, b3: false, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord11Array|csv:Error csvb4br11 = csv:parseString(csvStringWithBooleanValues4, {}); + test:assertEquals(csvb4br11, [ + {b1: true, b2: (), b3: (), b4: false, defaultableField: "", nillableField: ()}, + {b1: true, b2: (), b3: (), b4: false, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord11Array|csv:Error csvb5br11 = csv:parseString(csvStringWithBooleanValues5); + test:assertEquals(csvb5br11, [ + {b1: true, b2: false, b3: true, b4: "2", defaultableField: "", nillableField: ()}, + {b1: true, b2: false, b3: true, b4: "3", defaultableField: "", nillableField: ()} + ]); + + BooleanRecord11Array|csv:Error csvb6br11 = csv:parseString(csvStringWithBooleanValues6, {}); + test:assertTrue(csvb6br11 is csv:Error); + test:assertEquals((csvb6br11).message(), common:generateErrorMessageForMissingRequiredField("b1")); + + BooleanRecord11Array|csv:Error csvb7br11 = csv:parseString(csvStringWithBooleanValues7, {}); + test:assertEquals(csvb7br11, [ + {b1, b2, b3, b4, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord12Array|csv:Error csvb1br12 = csv:parseString(csvStringWithBooleanValues1); + test:assertTrue(csvb1br12 is csv:Error); + test:assertEquals((csvb1br12).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); + + BooleanRecord13_2[]|csv:Error csvb1br13_2 = csv:parseString(csvStringWithBooleanValues1, {}); + test:assertEquals(csvb1br13_2, [ + {b1: "true", b2: "false", b3: "true", b4: "false", defaultableField: "", nillableField: ()}, + {b1: "true", b2: "false", b3: "true", b4: "false", defaultableField: "", nillableField: ()}, + {b1: "true", b2: "false", b3: "true", b4: "false", defaultableField: "", nillableField: ()} + ]); + + BooleanRecord13Array|csv:Error csvb1br13 = csv:parseString(csvStringWithBooleanValues1, {}); + test:assertEquals(csvb1br13, [ + {b1: true, b2: false, b3: true, b4: false, defaultableField: "", nillableField: ()}, + {b1: true, b2: false, b3: true, b4: false, defaultableField: "", nillableField: ()}, + {b1: true, b2: false, b3: true, b4: false, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord13Array|csv:Error csvb2br13 = csv:parseString(csvStringWithBooleanValues2, {}); + test:assertEquals(csvb2br13, [ + {b1: true, b2: false, b3: true, b4: false, b5: true, defaultableField: "", nillableField: ()}, + {b1: true, b2: false, b3: true, b4: false, b5: true, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord13Array|csv:Error csvb3br13 = csv:parseString(csvStringWithBooleanValues3, {}); + test:assertEquals(csvb3br13, [ + {b1: true, b2: false, b3: true, defaultableField: "", nillableField: ()}, + {b1: true, b2: false, b3: "()", defaultableField: "", nillableField: ()}, + {b1: true, b2: true, b3: false, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord13Array|csv:Error csvb4br13 = csv:parseString(csvStringWithBooleanValues4); + test:assertEquals(csvb4br13, [ + {b1: true, b2: "()", b3: "()", b4: false, defaultableField: "", nillableField: ()}, + {b1: true, b2: "()", b3: "null", b4: false, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord13Array|csv:Error csvb5br13 = csv:parseString(csvStringWithBooleanValues5, {}); + test:assertEquals(csvb5br13, [ + {b1: true, b2: false, b3: true, b4: "2", defaultableField: "", nillableField: ()}, + {b1: true, b2: false, b3: true, b4: "3", defaultableField: "", nillableField: ()} + ]); + + BooleanRecord13Array|csv:Error csvb6br13 = csv:parseString(csvStringWithBooleanValues6, {}); + test:assertEquals(csvb6br13, [ + {b2: "()", b3: "()", defaultableField: "", nillableField: ()} + ]); + + BooleanRecord13Array|csv:Error csvb7br13 = csv:parseString(csvStringWithBooleanValues7); + test:assertEquals(csvb7br13, [ + {b1, b2, b3: "()", b4, defaultableField: "", nillableField: ()} + ]); + + BooleanRecord14Array|csv:Error csvb7br14 = csv:parseString(csvStringWithBooleanValues6, {}); + test:assertTrue(csvb7br14 is csv:Error); + test:assertEquals((csvb7br14).message(), common:generateErrorMessageForMissingRequiredField("requiredField")); +} + +@test:Config +function testFromCsvStringWithTypeForStringAndRecordAsExpectedType5() { + BooleanRecord15Array|csv:Error csvb1br15 = csv:parseString(csvStringWithBooleanValues1, {}); + test:assertTrue(csvb1br15 is csv:Error); + test:assertEquals((csvb1br15).message(), common:generateErrorMessageForInvalidCast("true", "int")); + + BooleanRecord15Array|csv:Error csvb6br15 = csv:parseString(csvStringWithBooleanValues6, {}); + test:assertTrue(csvb6br15 is csv:Error); + test:assertEquals((csvb6br15).message(), common:generateErrorMessageForMissingRequiredField("b1")); + + BooleanRecord15Array|csv:Error csvb7br15 = csv:parseString(csvStringWithBooleanValues7, {}); + test:assertTrue(csvb7br15 is csv:Error); + test:assertEquals((csvb7br15).message(), common:generateErrorMessageForInvalidCast("true", "int")); + + BooleanRecord16Array|csv:Error csvb1br16 = csv:parseString(csvStringWithBooleanValues1, {}); + test:assertEquals(csvb1br16, [ + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false}, + {b1: true, b2: false, b3: true, b4: false} + ]); + + BooleanRecord16Array|csv:Error csvb2br16 = csv:parseString(csvStringWithBooleanValues2, {}); + test:assertEquals(csvb2br16, [ + {b1: true, b2: false, b3: true, b4: false, b5: true}, + {b1: true, b2: false, b3: true, b4: false, b5: true} + ]); + + BooleanRecord16Array|csv:Error csvb3br16 = csv:parseString(csvStringWithBooleanValues3, {}); + test:assertEquals(csvb3br16, [ + {b1: true, b2: false, b3: true}, + {b1: true, b2: false, b3: ()}, + {b1: true, b2: true, b3: false} + ]); + + BooleanRecord16Array|csv:Error csvb4br16 = csv:parseString(csvStringWithBooleanValues4, {}); + test:assertEquals(csvb4br16, [ + {b1: true, b2: (), b3: (), b4: false}, + {b1: true, b2: (), b3: (), b4: false} + ]); + + BooleanRecord16Array|csv:Error csvb5br16 = csv:parseString(csvStringWithBooleanValues5, {}); + test:assertEquals(csvb5br16, [ + {b1: true, b2: false, b3: true}, + {b1: true, b2: false, b3: true} + ]); + + BooleanRecord16Array|csv:Error csvb6br16 = csv:parseString(csvStringWithBooleanValues6, {}); + test:assertEquals(csvb6br16, [ + {b2: (), b3: ()} + ]); + + BooleanRecord16Array|csv:Error csvb7br16 = csv:parseString(csvStringWithBooleanValues7, {}); + test:assertEquals(csvb7br16, [ + {b1, b2, b3: (), b4} + ]); + + BooleanRecord17Array|csv:Error csvb1br17 = csv:parseString(csvStringWithBooleanValues1, {}); + test:assertEquals(csvb1br17, [{}, {}, {}]); + + BooleanRecord17Array|csv:Error csvb4br17 = csv:parseString(csvStringWithBooleanValues4, {}); + test:assertEquals(csvb4br17, [{}, {}]); + + BooleanRecord17Array|csv:Error csvb6br17 = csv:parseString(csvStringWithBooleanValues6, {}); + test:assertEquals(csvb6br17, [{}]); + + BooleanRecord17Array|csv:Error csvb7br17 = csv:parseString(csvStringWithBooleanValues7, {}); + test:assertEquals(csvb7br17, [{}]); + + BooleanRecord18Array|csv:Error csvb1br18 = csv:parseString(csvStringWithBooleanValues1, {}); + test:assertEquals(csvb1br18, [{b2: false}, {b2: false}, {b2: false}]); + + BooleanRecord18Array|csv:Error csvb4br18 = csv:parseString(csvStringWithBooleanValues4, {}); + test:assertTrue(csvb4br18 is csv:Error); + test:assertEquals((csvb4br18).message(), common:generateErrorMessageForInvalidCast("()", "boolean")); + + BooleanRecord18Array|csv:Error csvb6br18 = csv:parseString(csvStringWithBooleanValues6, {}); + test:assertTrue(csvb6br18 is csv:Error); + test:assertEquals((csvb6br18).message(), common:generateErrorMessageForInvalidCast("()", "boolean")); + + BooleanRecord18Array|csv:Error csvb7br18 = csv:parseString(csvStringWithBooleanValues7, {}); + test:assertEquals(csvb7br18, [{b2, b3: ()}]); +} diff --git a/ballerina-tests/parse-string-record-types-tests/tests/test_data_values.bal b/ballerina-tests/parse-string-record-types-tests/tests/test_data_values.bal new file mode 100644 index 0000000..db0814f --- /dev/null +++ b/ballerina-tests/parse-string-record-types-tests/tests/test_data_values.bal @@ -0,0 +1,63 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +final boolean b1 = true; +final false b2 = false; +final boolean? b3 = (); +final boolean|int b4 = false; + +final string csvStringWithBooleanValues1 = string `b1,b2,b3,b4 +true,false,true,false +true,false, true,false +true,false,true,false +`; + +final string csvStringWithBooleanValues2 = string `b1,b2,b3,b4,b5 +true,false, true,false,true +true,false, true,false,true`; + +final string csvStringWithBooleanValues3 = string `b1,b2,b3 +${" "}${"\t"} +true, false,true +${" "} + TRUE, FALSE,() +${" "} + +true, true,FALSE + +`; + +final string csvStringWithBooleanValues4 = string `b1,b2,b3,b4 + true,(), (),false + true,(), null,false + +`; + +final string csvStringWithBooleanValues5 = string `b1,b2,b3,b4 + +true,false,true,2 + +true,false,true,3 +`; + +final string csvStringWithBooleanValues6 = string `b2,b3 +(),() + +`; + +final string csvStringWithBooleanValues7 = string `b1,b2,b3,b4 +${b1},${b2},(),${b4} +`; diff --git a/ballerina-tests/parse-string-record-types-tests/tests/types.bal b/ballerina-tests/parse-string-record-types-tests/tests/types.bal new file mode 100644 index 0000000..0a189a3 --- /dev/null +++ b/ballerina-tests/parse-string-record-types-tests/tests/types.bal @@ -0,0 +1,255 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +type BooleanRecord1 record { + boolean b1; + boolean|string b2; + boolean|string? b3; + boolean b4; +}; + +type BooleanRecord2 record {| + boolean b1; + boolean|string b2; + boolean|string? b3; + boolean b4; +|}; + +type BooleanRecord3 record {| + boolean b1; + boolean? b3; +|}; + +type BooleanRecord4 record { + boolean b1; + boolean? b3; +}; + +type BooleanRecord5 record { + boolean b1; + boolean? b3; + string defaultableField = ""; + string? nillableField = (); +}; + +type BooleanRecord6 record {| + boolean b1; + boolean? b3; + string defaultableField = ""; + string? nillableField = (); +|}; + +type BooleanRecord7 record { + boolean b1; + boolean? b3; + string defaultableField = ""; + string? nillableField = (); + string requiredField; +}; + +type BooleanRecord8 record {| + boolean b1; + boolean? b3; + string defaultableField = ""; + string? nillableField = (); + string requiredField; +|}; + +type BooleanRecord9 record {| + boolean b1; + boolean? b3; + boolean?...; +|}; + +type BooleanRecord10 record {| + boolean...; +|}; + +type BooleanRecord11 record {| + boolean b1; + string defaultableField = ""; + string? nillableField = (); + boolean?|string...; +|}; + +type BooleanRecord12 record {| + boolean b1; + string defaultableField = ""; + string? nillableField = (); + string requiredField; + boolean...; +|}; + +type BooleanRecord13 record {| + string defaultableField = ""; + string? nillableField = (); + boolean|string...; +|}; + +type BooleanRecord13_2 record {| + string defaultableField = ""; + string? nillableField = (); + string|boolean...; +|}; + +type BooleanRecord14 record {| + string defaultableField = ""; + string? nillableField = (); + string requiredField; + boolean...; +|}; + +type BooleanRecord15 record {| + int b1; + string defaultableField = ""; + string? nillableField = (); + boolean?...; +|}; + +type BooleanRecord16 record {| + boolean?...; +|}; + +type BooleanRecord17 record {| + int...; +|}; + +type BooleanRecord18 record {| + boolean b2; + int?...; +|}; + +type StringMap map; + +type DecimalMap map; + +type BooleanMap map; + +type NillableBooleanMap map; + +type NillableIntUnionBooleanMap map; + +type IntUnionBooleanMap map; + +type NilMap map<()>; + +type JsonMap map; + +type AnydataMap map; + +type CustomMap map; + +type BooleanRecord1Array BooleanRecord1[]; + +type ClosedBooleanRecord1Array BooleanRecord1[3]; + +type BooleanRecord2Array BooleanRecord2[]; + +type ClosedBooleanRecord2Array BooleanRecord2[3]; + +type BooleanRecord3Array BooleanRecord3[]; + +type ClosedBooleanRecord3Array BooleanRecord3[3]; + +type BooleanRecord4Array BooleanRecord4[]; + +type ClosedBooleanRecord4Array BooleanRecord4[3]; + +type BooleanRecord5Array BooleanRecord5[]; + +type ClosedBooleanRecord5Array BooleanRecord5[3]; + +type BooleanRecord6Array BooleanRecord6[]; + +type ClosedBooleanRecord6Array BooleanRecord6[3]; + +type BooleanRecord7Array BooleanRecord7[]; + +type ClosedBooleanRecord7Array BooleanRecord7[3]; + +type BooleanRecord8Array BooleanRecord8[]; + +type ClosedBooleanRecord8Array BooleanRecord8[3]; + +type BooleanRecord9Array BooleanRecord9[]; + +type ClosedBooleanRecord9Array BooleanRecord9[3]; + +type BooleanRecord10Array BooleanRecord10[]; + +type ClosedBooleanRecord10Array BooleanRecord10[3]; + +type BooleanRecord11Array BooleanRecord11[]; + +type ClosedBooleanRecord11Array BooleanRecord11[3]; + +type BooleanRecord12Array BooleanRecord12[]; + +type ClosedBooleanRecord12Array BooleanRecord12[3]; + +type BooleanRecord13Array BooleanRecord13[]; + +type ClosedBooleanRecord13Array BooleanRecord13[3]; + +type BooleanRecord14Array BooleanRecord14[]; + +type ClosedBooleanRecord14Array BooleanRecord14[3]; + +type BooleanRecord15Array BooleanRecord15[]; + +type ClosedBooleanRecord15Array BooleanRecord15[3]; + +type BooleanRecord16Array BooleanRecord16[]; + +type ClosedBooleanRecord16Array BooleanRecord16[3]; + +type BooleanRecord17Array BooleanRecord17[]; + +type ClosedBooleanRecord17Array BooleanRecord17[3]; + +type BooleanRecord18Array BooleanRecord18[]; + +type ClosedBooleanRecord18Array BooleanRecord18[3]; + +type BooleanMapArray BooleanMap[]; + +type NillableBooleanMapArray NillableBooleanMap[]; + +type NillableIntUnionBooleanMapArray NillableIntUnionBooleanMap[]; + +type IntUnionBooleanMapArray IntUnionBooleanMap[]; + +type ClosedBooleanMapArray BooleanMap[3]; + +type NilMapArray NilMap[]; + +type DecimalMapArray DecimalMap[]; + +type StringMapArray StringMap[]; + +type ClosedNilMapArray NilMap[3]; + +type JsonMapArray JsonMap[]; + +type ClosedJsonMapArray JsonMap[3]; + +type AnydataMapArray AnydataMap[]; + +type ClosedAnydataMapArray AnydataMap[3]; + +type CustomMapArray CustomMap[]; + +type ClosedCustomMapArray CustomMap[3]; diff --git a/ballerina-tests/type-compatible-tests/Ballerina.toml b/ballerina-tests/type-compatible-tests/Ballerina.toml new file mode 100644 index 0000000..d7661d9 --- /dev/null +++ b/ballerina-tests/type-compatible-tests/Ballerina.toml @@ -0,0 +1,16 @@ +[package] +org = "ballerina" +name = "type_compatible_tests" +version = "0.1.0" + +[[dependency]] +org = "ballerina" +name = "csv_commons" +repository = "local" +version = "0.1.0" + +[platform.java17] +graalvmCompatible = true + +[build-options] +graalvmBuildOptions = "-H:+IncludeAllLocales" diff --git a/ballerina-tests/type-compatible-tests/Dependencies.toml b/ballerina-tests/type-compatible-tests/Dependencies.toml new file mode 100644 index 0000000..5cc76ac --- /dev/null +++ b/ballerina-tests/type-compatible-tests/Dependencies.toml @@ -0,0 +1,121 @@ +# AUTO-GENERATED FILE. DO NOT MODIFY. + +# This file is auto-generated by Ballerina for managing dependency versions. +# It should not be modified by hand. + +[ballerina] +dependencies-toml-version = "2" +distribution-version = "2201.9.0" + +[[package]] +org = "ballerina" +name = "csv_commons" +version = "0.1.0" +scope = "testOnly" +modules = [ + {org = "ballerina", packageName = "csv_commons", moduleName = "csv_commons"} +] + +[[package]] +org = "ballerina" +name = "data.csv" +version = "0.1.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] +modules = [ + {org = "ballerina", packageName = "data.csv", moduleName = "data.csv"} +] + +[[package]] +org = "ballerina" +name = "io" +version = "1.6.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.value"} +] +modules = [ + {org = "ballerina", packageName = "io", moduleName = "io"} +] + +[[package]] +org = "ballerina" +name = "jballerina.java" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "lang.__internal" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.object"} +] + +[[package]] +org = "ballerina" +name = "lang.array" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.__internal"} +] + +[[package]] +org = "ballerina" +name = "lang.error" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "lang.object" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "lang.value" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "test" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.array"}, + {org = "ballerina", name = "lang.error"} +] +modules = [ + {org = "ballerina", packageName = "test", moduleName = "test"} +] + +[[package]] +org = "ballerina" +name = "type_compatible_tests" +version = "0.1.0" +dependencies = [ + {org = "ballerina", name = "csv_commons"}, + {org = "ballerina", name = "data.csv"}, + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "test"} +] +modules = [ + {org = "ballerina", packageName = "type_compatible_tests", moduleName = "type_compatible_tests"} +] + diff --git a/ballerina-tests/type-compatible-tests/tests/csv_content.txt b/ballerina-tests/type-compatible-tests/tests/csv_content.txt new file mode 100644 index 0000000..2917073 --- /dev/null +++ b/ballerina-tests/type-compatible-tests/tests/csv_content.txt @@ -0,0 +1,4 @@ +a, b, c d, e +"Hello World", \"Hello World\", Hello World, 2 +"Hello World", \"Hello World\", Hello World, 2 +"Hello World", \"Hello World\", Hello World, 2 \ No newline at end of file diff --git a/ballerina-tests/type-compatible-tests/tests/csv_content_2.txt b/ballerina-tests/type-compatible-tests/tests/csv_content_2.txt new file mode 100644 index 0000000..c2d88ba --- /dev/null +++ b/ballerina-tests/type-compatible-tests/tests/csv_content_2.txt @@ -0,0 +1,4 @@ +a, b, c d, e +"Hello World1", \"Hello World2\", Hello World3, 21 +"Hello World1", \"Hello World2\", Hello World3, 22 +"Hello World1", \"Hello World2\", Hello World3, 23 \ No newline at end of file diff --git a/ballerina-tests/type-compatible-tests/tests/csv_error_content.txt b/ballerina-tests/type-compatible-tests/tests/csv_error_content.txt new file mode 100644 index 0000000..d2335d3 --- /dev/null +++ b/ballerina-tests/type-compatible-tests/tests/csv_error_content.txt @@ -0,0 +1,2 @@ +a, b, c d, e +"Hello W \ No newline at end of file diff --git a/ballerina-tests/type-compatible-tests/tests/nill_type_test.bal b/ballerina-tests/type-compatible-tests/tests/nill_type_test.bal new file mode 100644 index 0000000..49a00e6 --- /dev/null +++ b/ballerina-tests/type-compatible-tests/tests/nill_type_test.bal @@ -0,0 +1,58 @@ +import ballerina/data.csv; +import ballerina/test; + +type Book1 record { + ()|string name; + ()|string author; + ()|string year; +}; + +@test:Config +function testEmptyStringWithNilConfig() { + string csvString1 = string `name,author,year + ,b,c + a,,c + a,b,`; + + Book1[]|error books1 = csv:parseString(csvString1, {nilValue: ""}); + test:assertEquals(books1, [ + {name: null, author: "b", year: "c"}, + {name: "a", author: null, year: "c"}, + {name: "a", author: "b", year: null} + ]); + + Book1[]|error books2 = csv:parseString(csvString1); + test:assertEquals(books2, [ + {name: "", author: "b", year: "c"}, + {name: "a", author: "", year: "c"}, + {name: "a", author: "b", year: ""} + ]); + + (boolean|()|string|int)[][]|error arrbooks1 = csv:parseString(csvString1, {nilValue: ""}); + test:assertEquals(arrbooks1, [ + [null, "b", "c"], + ["a", null, "c"], + ["a", "b", null] + ]); + + (boolean|()|string|int)[][2]|error arrbooks2 = csv:parseString(csvString1, {nilValue: ""}); + test:assertEquals(arrbooks2, [ + [null, "b"], + ["a", null], + ["a", "b"] + ]); + + (boolean|()|string|int)[][]|error arrbooks3 = csv:parseString(csvString1); + test:assertEquals(arrbooks3, [ + ["", "b", "c"], + ["a", "", "c"], + ["a", "b", ""] + ]); + + (boolean|()|string|int)[][2]|error arrbooks4 = csv:parseString(csvString1); + test:assertEquals(arrbooks4, [ + ["", "b"], + ["a", ""], + ["a", "b"] + ]); +} diff --git a/ballerina-tests/type-compatible-tests/tests/parse_string_compatibality_test.bal b/ballerina-tests/type-compatible-tests/tests/parse_string_compatibality_test.bal new file mode 100644 index 0000000..6199175 --- /dev/null +++ b/ballerina-tests/type-compatible-tests/tests/parse_string_compatibality_test.bal @@ -0,0 +1,361 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/csv_commons as common; +import ballerina/data.csv; +import ballerina/io; +import ballerina/test; + +const string filepath = "tests/csv_content.txt"; +const string filepath2 = "tests/csv_content_2.txt"; +const string errorFilepath = "tests/csv_error_content.txt"; + +@test:Config +function testFromCsvStringWithTypeCompatibility() { + string value = string `i1,i2,s1,s2, b1,b2,n1,n2,f1,f2, d1,d2,j1,a1,j2,a2 + ${i1},${i2},${s1},${s2},${b1},${b2},(),(),${f1}, ${f2},${d1},${d2},${b1},${d1},${b2},${d2} + ${i1},${i2},${s1},${s2},${b1},${b2},(),(), ${f1},${f2},${d1},${d2},${b1},${d1},${b2},${d2} + `; + string value2 = string `i1, s1, b1, n1, f1, d1, j1, a1, s2, s3, j2, a2 + ${i1}, ${s1},${b1}, null,${f1}, ${d1},${b1}, ${d1},${s2}, ${s3},${b2}, ${d2} + `; + string value3 = string `i1, s1, b1, n1, f1, d1, j1, a1, s2, s3 + ${i1}, ${s1},${b1}, null,${f1}, ${d1},${b1}, ${d1},${s2}, ${s3} + `; + + AnydataArray1Array|csv:Error v3anyd1a = csv:parseString(value); + test:assertEquals(v3anyd1a, [ + [i1, i2, s1, s2, b1, b2, n1, n2, 2.234, -3.21, 2.234, -3.21, b1, 2.234, b2, -3.21], + [i1, i2, s1, s2, b1, b2, n1, n2, 2.234, -3.21, 2.234, -3.21, b1, 2.234, b2, -3.21] + ]); + + CustomRecord27Array|csv:Error vcr27a = csv:parseString(value, {}, CustomRecord27Array); + test:assertEquals(vcr27a, [ + {i1, s1, b1, n1, f1, d1, a1: 2.234, j1: b1, defaultableField: "", nillableField: null, i2: "-2", s2: "", b2: "false", n2: "()", f2: "-3.21", d2: "-3.21", j2: "false", a2: "-3.21"}, + {i1, s1, b1, n1, f1, d1, a1: 2.234, j1: b1, defaultableField: "", nillableField: null, i2: "-2", s2: "", b2: "false", n2: "()", f2: "-3.21", d2: "-3.21", j2: "false", a2: "-3.21"} + ]); + + CustomRecord27Array|csv:Error v2cr27a = csv:parseString(value2, {}, CustomRecord27Array); + test:assertEquals(v2cr27a, [ + {i1, s1, b1, n1, f1, d1, a1: 2.234, j1: b1, defaultableField: "", nillableField: null, s2, j2: "false", a2: "-3.21", s3} + ]); + + CustomRecord27Array|csv:Error v3cr27a = csv:parseString(value3, {}); + test:assertEquals(v3cr27a, [ + {i1, s1, b1, n1, f1, d1, a1: 2.234, j1: b1, defaultableField: "", nillableField: null, s2, s3} + ]); + + AnydataMapArray|csv:Error vanydma = csv:parseString(value); + test:assertEquals(vanydma, [ + {i1, i2, s1, s2, b1, b2, n1, n2, f1: 2.234, f2: -3.21, d1: 2.234, d2: -3.21, j1: b1, a1: 2.234, j2: false, a2: -3.21}, + {i1, i2, s1, s2, b1, b2, n1, n2, f1: 2.234, f2: -3.21, d1: 2.234, d2: -3.21, j1: b1, a1: 2.234, j2: false, a2: -3.21} + ]); + + JsonMapArray|csv:Error vjma = csv:parseString(value); + test:assertEquals(vjma, [ + {i1, i2, s1, s2, b1, b2, n1, n2, f1: 2.234, f2: -3.21, d1: 2.234, d2: -3.21, j1: b1, a1: 2.234, j2: false, a2: -3.21}, + {i1, i2, s1, s2, b1, b2, n1, n2, f1: 2.234, f2: -3.21, d1: 2.234, d2: -3.21, j1: b1, a1: 2.234, j2: false, a2: -3.21} + ]); + + StringMapArray|csv:Error vsma = csv:parseString(value); + test:assertEquals(vsma, [ + {i1: "1", s1: "string", b1: "true", n1: "()", f1: "2.234", d1: "2.234", a1: "2.234", j1: "true", i2: "-2", s2: "", b2: "false", n2: "()", f2: "-3.21", d2: "-3.21", j2: "false", a2: "-3.21"}, + {i1: "1", s1: "string", b1: "true", n1: "()", f1: "2.234", d1: "2.234", a1: "2.234", j1: "true", i2: "-2", s2: "", b2: "false", n2: "()", f2: "-3.21", d2: "-3.21", j2: "false", a2: "-3.21"} + ]); + + CustomTuple7Array|csv:Error v2ct7a = csv:parseString(value2); + test:assertEquals(v2ct7a, [ + [i1, s1, b1, n1, 2.234, 2.234, b1, 2.234, s2, s3, "false", "-3.21"] + ]); + + CustomTuple7Array|csv:Error v3ct7a = csv:parseString(value3); + test:assertEquals(v3ct7a, [ + [i1, s1, b1, n1, 2.234, 2.234, b1, 2.234, s2, s3] + ]); + + [float, decimal, string][]|csv:Error mrrta = csv:parseString(string `a, b,c + 1.23, 1.23, 1.23 + 0,0,0 + 0.0,0.0,0.0 + -1.2,-1.2,-1.2`); + test:assertEquals(mrrta, [ + [1.23, 1.23, "1.23"], + [0, 0, "0"], + [0, 0, "0.0"], + [-1.2, -1.2, "-1.2"] + ]); + + [float, decimal, string, int][]|csv:Error m2rrta = csv:parseString(string `a, b,c,d + 1, 1, 1,1 + 0,0,0,0 + -1,-1,-1,-1`); + test:assertEquals(m2rrta, [ + [1, 1, "1", 1], + [0, 0, "0", 0], + [-1, -1, "-1", -1] + ]); + + [int...][]|csv:Error m3rrta = csv:parseString(string `a, b,c,d + 1.2, abc, true,1.0`); + test:assertTrue(m3rrta is csv:Error); + test:assertEquals((m3rrta).message(), common:generateErrorMessageForInvalidCast("1.2", "int")); + + [boolean|int, int|boolean][]|csv:Error m4rrta = csv:parseString(string `a, b + 1, 1 + 0,0`); + test:assertEquals(m4rrta, [ + [1, 1], + [0, 0] + ]); + + record {|int...;|}[]|csv:Error irrma = csv:parseString(string ` + a, b, c + 1, a, 2.3 + 1, -2, true + hello, -2, hello`, {header: 1}); + test:assertEquals(irrma, [ + {a: 1}, + {a: i1, b: i2}, + {b: i2} + ]); + + record {|()...;|}[]|csv:Error nrrma = csv:parseString(string ` + a, b, c + 1, a, () + 1, null, () + hello, -2, hello`, {header: 1}); + test:assertEquals(nrrma, [ + {c: ()}, + {b: (), c: ()}, + {} + ]); + + record {|decimal...;|}[]|csv:Error drra = csv:parseString(string `a, b, c + 2.234, invalid , 1 + ${f2}, 0, 2.3d + invalid, ${d2}, ${f3}`); + test:assertTrue(drra is record {|decimal...;|}[]); + test:assertEquals(drra, [ + {a: d1, c: 1.0}, + {a: f2, b: d3}, + {b: -3.21d, c: f3} + ]); + + record {|string...;|}[]|csv:Error srra = csv:parseString(string `a, b, c + 1, a, 2.3 + 1, -2, true + hello, -2, hello`); + test:assertTrue(srra is record {|string...;|}[]); + test:assertEquals(srra, [ + {a: "1", b: "a", c: "2.3"}, + {a: "1", b: "-2", c: "true"}, + {a: "hello", b: "-2", c: "hello"} + ]); + + record {|float...;|}[]|csv:Error frra = csv:parseString(string `a, b, c + 1.2, invalid , 1 + ${d2}, ${d3}, true + ${d4}, ${f2}, 0.0`); + test:assertEquals(frra, [ + {a: 1.2, c: 1.0}, + {a: d2, b: d3}, + {a: d4, b: f2, c: 0.0} + ]); + + record {float a; decimal b; string c;}[]|csv:Error mrra = csv:parseString(string `a, b,c + 1.23, 1.23, 1.23 + 0,0,0 + 0.0,0.0,0.0 + -1.2,-1.2,-1.2`); + test:assertEquals(mrra, [ + {a: 1.23, b: 1.23, c: "1.23"}, + {a: 0, b: 0, c: "0"}, + {a: 0, b: 0, c: "0.0"}, + {a: -1.2, b: -1.2, c: "-1.2"} + ]); + + record {|float a; decimal b; string c; int d;|}[]|csv:Error m2rra = csv:parseString(string `a, b,c,d + 1, 1, 1,1 + 0,0,0,0 + -1,-1,-1,-1`); + test:assertEquals(m2rra, [ + {a: 1, b: 1, c: "1", d: 1}, + {a: 0, b: 0, c: "0", d: 0}, + {a: -1, b: -1, c: "-1", d: -1} + ]); + + record {int d;}[]|csv:Error m3rra = csv:parseString(string `a, b,c,d + 1.2, abc, true,1.0`); + test:assertTrue(m3rra is csv:Error); + test:assertEquals((m3rra).message(), common:generateErrorMessageForInvalidCast("1.0", "int")); +} + +@test:Config +function testSpaceBetweendData() { + string csv = string `a b, b d e, f + "Hello world", " Hi I am ", \" Hi I am \"`; + + record {|string...;|}[]|csv:Error rec = csv:parseString(csv); + test:assertEquals(rec, [ + {"a b":"Hello world","b d e":" Hi I am ","f":"\"Hi I am \""}]); +} + +@test:Config +function testParseBytes() returns error? { + byte[] csvBytes = check io:fileReadBytes(filepath); + byte[] csvBytes2 = check io:fileReadBytes(filepath2); + + record {}[]|csv:Error rec = csv:parseBytes(csvBytes, {}); + test:assertEquals(rec, [ + {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2}, + {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2}, + {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2}] + ); + + rec = csv:parseBytes(csvBytes); + test:assertEquals(rec, [ + {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2}, + {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2}, + {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2}] + ); + + string[][]|csv:Error rec2 = csv:parseBytes(csvBytes, {}); + test:assertEquals(rec2, [ + ["Hello World", "\"Hello World\"", "Hello World", "2"], + ["Hello World", "\"Hello World\"", "Hello World", "2"], + ["Hello World", "\"Hello World\"", "Hello World", "2"] + ]); + + rec2 = csv:parseBytes(csvBytes, {outputWithHeaders: true}); + test:assertEquals(rec2, [ + ["a", "b", "c d", "e"], + ["Hello World", "\"Hello World\"", "Hello World", "2"], + ["Hello World", "\"Hello World\"", "Hello World", "2"], + ["Hello World", "\"Hello World\"", "Hello World", "2"] + ]); + + rec2 = csv:parseBytes(csvBytes2, {outputWithHeaders: true, header: 1}); + test:assertEquals(rec2, [ + ["Hello World1", "\"Hello World2\"", "Hello World3", "21"], + ["Hello World1", "\"Hello World2\"", "Hello World3", "22"], + ["Hello World1", "\"Hello World2\"", "Hello World3", "23"] + ]); + + rec2 = csv:parseBytes(csvBytes2, { header: 1}); + test:assertEquals(rec2, [ + ["Hello World1", "\"Hello World2\"", "Hello World3", "22"], + ["Hello World1", "\"Hello World2\"", "Hello World3", "23"] + ]); + + int[][]|csv:Error rec3 = csv:parseBytes(csvBytes, {}); + test:assertTrue(rec3 is csv:Error); + test:assertEquals(( rec3).message(), common:generateErrorMessageForInvalidCast("Hello World", "int")); + + record {int a;}[]|csv:Error rec4 = csv:parseBytes(csvBytes, {}); + test:assertTrue(rec4 is csv:Error); + test:assertEquals(( rec4).message(), common:generateErrorMessageForInvalidCast("Hello World", "int")); +} + +@test:Config +function testParseStream() returns error? { + stream csvByteStream = check io:fileReadBlocksAsStream(filepath); + stream csvByteStream2 = check io:fileReadBlocksAsStream(filepath2); + + record {}[]|csv:Error rec = csv:parseStream(csvByteStream, {}); + test:assertEquals(rec, [ + {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2}, + {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2}, + {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2}] + ); + + csvByteStream = check io:fileReadBlocksAsStream(filepath); + rec = csv:parseStream(csvByteStream); + test:assertEquals(rec, [ + {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2}, + {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2}, + {"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2}] + ); + + csvByteStream = check io:fileReadBlocksAsStream(filepath); + string[][]|csv:Error rec2 = csv:parseStream(csvByteStream, {}); + test:assertEquals(rec2, [ + ["Hello World", "\"Hello World\"", "Hello World", "2"], + ["Hello World", "\"Hello World\"", "Hello World", "2"], + ["Hello World", "\"Hello World\"", "Hello World", "2"] + ]); + + rec2 = csv:parseStream(csvByteStream2, {header: 1, outputWithHeaders: true}); + test:assertEquals(rec2, [ + ["Hello World1", "\"Hello World2\"", "Hello World3", "21"], + ["Hello World1", "\"Hello World2\"", "Hello World3", "22"], + ["Hello World1", "\"Hello World2\"", "Hello World3", "23"] + ]); + + csvByteStream = check io:fileReadBlocksAsStream(filepath); + rec2 = csv:parseStream(csvByteStream, {outputWithHeaders: true}); + test:assertEquals(rec2, [ + ["a", "b", "c d", "e"], + ["Hello World", "\"Hello World\"", "Hello World", "2"], + ["Hello World", "\"Hello World\"", "Hello World", "2"], + ["Hello World", "\"Hello World\"", "Hello World", "2"] + ]); + + csvByteStream = check io:fileReadBlocksAsStream(filepath2); + rec2 = csv:parseStream(csvByteStream, {header: 1}); + test:assertEquals(rec2, [ + ["Hello World1", "\"Hello World2\"", "Hello World3", "22"], + ["Hello World1", "\"Hello World2\"", "Hello World3", "23"] + ]); + + csvByteStream = check io:fileReadBlocksAsStream(filepath); + record {int a;}[]|csv:Error rec3 = csv:parseStream(csvByteStream, {}); + test:assertTrue(rec3 is csv:Error); + test:assertEquals(( rec3).message(), "Error occurred while reading the stream: " + + common:generateErrorMessageForInvalidCast("Hello World", "int")); + + csvByteStream = check io:fileReadBlocksAsStream(filepath); + int[][]|csv:Error rec4 = csv:parseStream(csvByteStream, {}); + test:assertTrue(rec4 is csv:Error); + test:assertEquals(( rec4).message(), "Error occurred while reading the stream: " + + common:generateErrorMessageForInvalidCast("Hello World", "int")); +} + +@test:Config +function testErrorParseBytes() returns error? { + byte[] csvBytes = check io:fileReadBytes(errorFilepath); + + int[][]|csv:Error rec3 = csv:parseBytes(csvBytes, {}); + test:assertTrue(rec3 is csv:Error); + test:assertTrue(( rec3).message().includes("cannot be cast into")); + + record {int a;}[]|csv:Error rec4 = csv:parseBytes(csvBytes, {}); + test:assertTrue(rec4 is csv:Error); + test:assertTrue(( rec3).message().includes("cannot be cast into")); +} + +@test:Config +function testErrorParseStream() returns error? { + stream csvByteStream = check io:fileReadBlocksAsStream(errorFilepath); + + record {int a;}[]|csv:Error rec3 = csv:parseStream(csvByteStream, {}); + test:assertTrue(rec3 is csv:Error); + test:assertTrue(( rec3).message().includes("cannot be cast into")); + + csvByteStream = check io:fileReadBlocksAsStream(errorFilepath); + int[][]|csv:Error rec4 = csv:parseStream(csvByteStream, {}); + test:assertTrue(rec4 is csv:Error); + test:assertTrue(( rec4).message().includes("cannot be cast into")); +} diff --git a/ballerina-tests/type-compatible-tests/tests/parse_type_compatibility_test.bal b/ballerina-tests/type-compatible-tests/tests/parse_type_compatibility_test.bal new file mode 100644 index 0000000..1cf12ca --- /dev/null +++ b/ballerina-tests/type-compatible-tests/tests/parse_type_compatibility_test.bal @@ -0,0 +1,510 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/csv_commons as common; +import ballerina/data.csv; +import ballerina/test; + +@test:Config +function testFromCsvWithTypeFunctionWithTypeCompatibility() { + var value = {i1, i2, s1, s2, b1, b2, n1, n2, f1, f2, d1, d2, j1: b1, a1: d1, j2: b2, a2: d2}; + + CustomRecord27Array|csv:Error vcr27a = csv:transform([value, value, value], {}, CustomRecord27Array); + test:assertEquals(vcr27a, [ + {i1, s1, s2, b1, n1, f1, d1, j1: b1, a1: d1, defaultableField: "", nillableField: ()}, + {i1, s1, s2, b1, n1, f1, d1, j1: b1, a1: d1, defaultableField: "", nillableField: ()}, + {i1, s1, s2, b1, n1, f1, d1, j1: b1, a1: d1, defaultableField: "", nillableField: ()} + ]); + + AnydataMapArray|csv:Error vanydma = csv:transform([value, value, value], {}, AnydataMapArray); + test:assertEquals(vanydma, [ + value, + value, + value + ]); + + JsonMapArray|csv:Error vjma = csv:transform([value, value, value], {}, JsonMapArray); + test:assertEquals(vjma, [ + value, + value, + value + ]); + + record {|int...;|}[]|csv:Error irrma = csv:transform([{"a": 1}, {"a": i1, "b": i2}, {"a": i1, "b": i2, "c": s1}]); + test:assertEquals(irrma, [ + {a: 1}, + {a: i1, b: i2}, + {a: i1, b: i2} + ]); + + record {|decimal...;|}[]|csv:Error drra = csv:transform([{"a": d1}, {"a": d2, "b": d3}, {"a": d4, "b": f2, "c": s1, "d": f3}]); + test:assertTrue(drra is record {|decimal...;|}[]); + test:assertEquals(drra, [ + {a: d1}, + {a: d2, b: d3}, + {a: d4, b: -3.21d, d: f3} + ]); + + record {|string...;|}[]|csv:Error srra = csv:transform([{"a": "string"}, {"c": 1, "a": s1, "b": s2}, {"a": b1, "b": s3, "c": d1}]); + test:assertEquals(srra, [ + {a: "string"}, + {a: s1, b: s2}, + {b: s3} + ]); + + record {|float...;|}[]|csv:Error frra = csv:transform([{"a": 1.2, "b": 1.2f}, {"a": d2, "b": d3}, {"a": d4, "b": f2, "c": s1}]); + test:assertEquals(frra, [ + {a: 1.2, b: 1.2}, + {a: d2, b: d3}, + {a: d4, b: f2} + ]); + + record {|float a; decimal b;|}[]|csv:Error fdc1a = csv:transform([{"a": d1, "b": d2}, {"a": f1, "b": f2}, {"a": d2, "b": f2}, {"a": f2, "b": d2}]); + test:assertEquals(fdc1a, [ + {a: d1, b: d2}, + {a: f1, b: f2}, + {a: d2, b: f2}, + {a: f2, b: d2} + ]); + + record {|float a; decimal...;|}[]|csv:Error fdc2a = csv:transform([{"a": d1, "b": d2}, {"a": f1, "b": f2}, {"a": d2, "b": f2}, {"a": f2, "b": d2}]); + test:assertEquals(fdc2a, [ + {a: d1, b: d2}, + {a: f1, b: f2}, + {a: d2, b: f2}, + {a: f2, b: d2} + ]); + + record {|decimal b; float...;|}[]|csv:Error fdc3a = csv:transform([{"a": d1, "b": d2}, {"a": f1, "b": f2}, {"a": d2, "b": f2}, {"a": f2, "b": d2}]); + test:assertEquals(fdc3a, [ + {a: d1, b: d2}, + {a: f1, b: f2}, + {a: d2, b: f2}, + {a: f2, b: d2} + ]); +} + +type A decimal|int; + +type A2 int|decimal; + +type B readonly & A|C; + +type B2 readonly & C|readonly & A2; + +type C string|boolean; + +type C2 boolean|string; + +@test:Config +function testFromCsvWithIntersectionTypeCompatibility2() { + record { + readonly & int a; + readonly & string b; + (readonly & boolean)|(readonly & decimal) c; + }[]|csv:Error r1a = csv:parseString(string `a,b,c + 1,string,true + 2,string2,false + 3,string3,true`); + + test:assertEquals(r1a, [ + {a: 1, b: "string", c: true}, + {a: 2, b: "string2", c: false}, + {a: 3, b: "string3", c: true} + ]); + + record {A a; B b; C c;}[]|csv:Error r2a = csv:parseString(string `a,b,c + 1,string,true + 2,string2,false + 3,string3,true`); + + test:assertEquals(r2a, [ + {a: 1d, b: "string", c: "true"}, + {a: 2d, b: "string2", c: "false"}, + {a: 3d, b: "string3", c: "true"} + ]); + + record {A2 a; B2 b; C2 c;}[]|csv:Error r3a = csv:parseString(string `a,b,c + 1,string,true + 2,string2,false + 3,string3,true`); + + test:assertEquals(r3a, [ + {a: 1, b: "string", c: true}, + {a: 2, b: "string2", c: false}, + {a: 3, b: "string3", c: true} + ]); + + record {|A2 a; B2 b; C2...;|}[]|csv:Error r4a = csv:parseString(string ` + a,b,c,d + 1,string,true,string + 2,string2,false,string2 + 3,string3,true,string3`, {header: 1}); + + test:assertEquals(r4a, [ + {a: 1, b: "string", c: true, d: "string"}, + {a: 2, b: "string2", c: false, d: "string2"}, + {a: 3, b: "string3", c: true, d: "string3"} + ]); + + record {|C2...;|}[]|csv:Error r5a = csv:parseString(string `a,b,c,d + 1,string,true,string + 2,string2,false,string2 + 3,string3,true,string3`); + + test:assertEquals(r5a, [ + {a: "1", b: "string", c: true, d: "string"}, + {a: "2", b: "string2", c: false, d: "string2"}, + {a: "3", b: "string3", c: true, d: "string3"} + ]); + + [readonly & int, readonly & string, (readonly & boolean)|(readonly & decimal)][]|csv:Error r16a = + csv:parseString(string `a,b,c + 1,string,true + 2,string2,false + 3,string3,true`); + + test:assertEquals(r16a, [ + [1, "string", true], + [2, "string2", false], + [3, "string3", true] + ]); + + [A, B, C][]|csv:Error r17a = csv:parseString( + string `a,b,c + 1,string,true + 2,string2,false + 3,string3,true`); + + test:assertEquals(r17a, [ + [1d, "string", "true"], + [2d, "string2", "false"], + [3d, "string3", "true"] + ]); + + [A2, B2, C2][]|csv:Error r18a = csv:parseString( + string `a,b,c + 1,string,true + 2,string2,false + 3,string3,true`); + + test:assertEquals(r18a, [ + [1, "string", true], + [2, "string2", false], + [3, "string3", true] + ]); + + [A2, B2, C2...][]|csv:Error r19a = csv:parseString( + string `a,b,c,d + 1,string,true,string + 2,string2,false,string2 + 3,string3,true,string3`); + + test:assertEquals(r19a, [ + [1, "string", true, "string"], + [2, "string2", false, "string2"], + [3, "string3", true, "string3"] + ]); + + [C2...][]|csv:Error r20a = csv:parseString( + string `a,b,c,d + 1,string,true,string + 2,string2,false,string2 + 3,string3,true,string3`); + + test:assertEquals(r20a, [ + ["1", "string", true, "string"], + ["2", "string2", false, "string2"], + ["3", "string3", true, "string3"] + ]); + + record {A a; B b; C c;}[]|csv:Error rt2a = csv:transform( + [{"a": 1, "b": "string", "c": true}, {"a": 2, "b": "string2", "c": false}, {"a": 3, "b": "string3", "c": true}]); + + test:assertEquals(rt2a, [ + {a: 1, b: "string", c: true}, + {a: 2, b: "string2", c: false}, + {a: 3, b: "string3", c: true} + ]); + + record { + readonly & int a; + readonly & string b; + (readonly & boolean)|(readonly & decimal) c; + }[]|csv:Error rt1a = csv:transform( + [{"a": 1, "b": "string", "c": true}, {"a": 2, "b": "string2", "c": false}, {"a": 3, "b": "string3", "c": true}]); + + test:assertEquals(rt1a, [ + {a: 1, b: "string", c: true}, + {a: 2, b: "string2", c: false}, + {a: 3, b: "string3", c: true} + ]); + + record {A2 a; B2 b; C2 c;}[]|csv:Error rt3a = csv:transform( + [{"a": 1, "b": "string", "c": true}, {"a": 2, "b": "string2", "c": false}, {"a": 3, "b": "string3", "c": true}]); + + test:assertEquals(rt3a, [ + {a: 1, b: "string", c: true}, + {a: 2, b: "string2", c: false}, + {a: 3, b: "string3", c: true} + ]); + + record {|A2 a; B2 b; C2...;|}[]|csv:Error rt4a = csv:transform( + [{"a": 1, "b": "string", "c": true, "d": "string"}, {"a": 2, "b": "string2", "c": false, "d": "string2"}, {"a": 3, "b": "string3", "c": true, "d": "string3"}]); + + test:assertEquals(rt4a, [ + {a: 1, b: "string", c: true, d: "string"}, + {a: 2, b: "string2", c: false, d: "string2"}, + {a: 3, b: "string3", c: true, d: "string3"} + ]); + + record {|C2...;|}[]|csv:Error rt5a = csv:transform( + [{"a": 1, "b": "string", "c": true, "d": "string"}, {"a": 2, "b": "string2", "c": false, "d": "string2"}, {"a": 3, "b": "string3", "c": true, "d": "string3"}]); + + test:assertEquals(rt5a, [ + {b: "string", c: true, d: "string"}, + {b: "string2", c: false, d: "string2"}, + {b: "string3", c: true, d: "string3"} + ]); + + [readonly & int, readonly & string, (readonly & boolean)|(readonly & decimal)][]|csv:Error rt6a = + csv:transform( + [ + {"a": 1, "b": "string", "c": true}, + {"a": 2, "b": "string2", "c": false}, + {"a": 3, "b": "string3", "c": true} + ], {headerOrder: ["a", "b", "c"]}); + + test:assertEquals(rt6a, [ + [1, "string", true], + [2, "string2", false], + [3, "string3", true] + ]); + + [A, B, C][]|csv:Error rt7a = csv:transform( + [{"a": 1, "b": "string", "c": true}, {"a": 2, "b": "string2", "c": false}, {"a": 3, "b": "string3", "c": true}] + , {headerOrder: ["a", "b", "c"]}); + + test:assertEquals(rt7a, [ + [1d, "string", true], + [2d, "string2", false], + [3d, "string3", true] + ]); + + [A2, B2, C2][]|csv:Error rt8a = csv:transform( + [{"a": 1, "b": "string", "c": true}, {"a": 2, "b": "string2", "c": false}, {"a": 3, "b": "string3", "c": true}] + , {headerOrder: ["a", "b", "c"]}); + + test:assertEquals(rt8a, [ + [1, "string", true], + [2, "string2", false], + [3, "string3", true] + ]); + + [A2, B2, C2...][]|csv:Error rt9a = csv:transform( + [{"a": 1, "b": "string", "c": true, "d": "string"}, {"a": 2, "b": "string2", "c": false, "d": "string2"}, {"a": 3, "b": "string3", "c": true, "d": "string3"}] + , {headerOrder: ["a", "b", "c", "d"]}); + + test:assertEquals(rt9a, [ + [1, "string", true, "string"], + [2, "string2", false, "string2"], + [3, "string3", true, "string3"] + ]); + + [C2...][]|csv:Error rt10a = csv:transform( + [{"a": 1, "b": "string", "c": true, "d": "string"}, {"a": 2, "b": "string2", "c": false, "d": "string2"}, {"a": 3, "b": "string3", "c": true, "d": "string3"}] + , {headerOrder: ["a", "b", "c", "d"]}); + + test:assertTrue(rt10a is csv:Error); + test:assertEquals((rt10a).message(), common:generateErrorMessageForInvalidValueForArrayType("1", "0", "type_compatible_tests:C2")); + + record { + readonly & int a; + readonly & string b; + (readonly & boolean)|(readonly & decimal) c; + }[]|csv:Error rt11a = csv:parseList( + [["1", "string", "true"], ["2", "string2", "false"], ["3", "string3", "true"]], {customHeaders: ["a", "b", "c"]}); + + test:assertEquals(rt11a, [ + {a: 1, b: "string", c: true}, + {a: 2, b: "string2", c: false}, + {a: 3, b: "string3", c: true} + ]); + + record {A a; B b; C c;}[]|csv:Error rt12a = csv:parseList( + [["1", "string", "true"], ["2", "string2", "false"], ["3", "string3", "true"]], {customHeaders: ["a", "b", "c"]}); + + test:assertEquals(rt12a, [ + {a: 1d, b: "string", c: "true"}, + {a: 2d, b: "string2", c: "false"}, + {a: 3d, b: "string3", c: "true"} + ]); + + record {string|decimal a; B b; C c;}[]|csv:Error rt12a_3 = csv:parseList( + [["1", "string", "true"], ["2", "string2", "false"], ["3", "string3", "true"]], {customHeaders: ["a", "b", "c"]}); + + test:assertEquals(rt12a_3, [ + {a: "1", b: "string", c: "true"}, + {a: "2", b: "string2", c: "false"}, + {a: "3", b: "string3", c: "true"} + ]); + + record {decimal|string a; B b; C c;}[]|csv:Error rt12a_4 = csv:parseList( + [["1", "string", "true"], ["2", "string2", "false"], ["3", "string3", "true"]], {customHeaders: ["a", "b", "c"]}); + + test:assertEquals(rt12a_4, [ + {a: 1, b: "string", c: "true"}, + {a: 2, b: "string2", c: "false"}, + {a: 3, b: "string3", c: "true"} + ]); + + record {A2 a; B2 b; C2 c;}[]|csv:Error rt13a = csv:parseList( + [["1", "string", "true"], ["2", "string2", "false"], ["3", "string3", "true"]], {customHeaders: ["a", "b", "c"]}); + + test:assertEquals(rt13a, [ + {a: 1, b: "string", c: true}, + {a: 2, b: "string2", c: false}, + {a: 3, b: "string3", c: true} + ]); + + record {|A2 a; B2 b; C2...;|}[]|csv:Error rt14a = csv:parseList( + [["1", "string", "true", "string"], ["2", "string2", "false", "string2"], ["3", "string3", "true", "string3"]] + , {customHeaders: ["a", "b", "c", "d"]}); + + test:assertEquals(rt14a, [ + {a: 1, b: "string", c: true, d: "string"}, + {a: 2, b: "string2", c: false, d: "string2"}, + {a: 3, b: "string3", c: true, d: "string3"} + ]); + + record {|C2...;|}[]|csv:Error rt15a = csv:parseList( + [["1", "string", "true", "string"], ["2", "string2", "false", "string2"], ["3", "string3", "true", "string3"]] + , {customHeaders: ["a", "b", "c", "d"]}); + + test:assertEquals(rt15a, [ + {a: "1", b: "string", c: true, d: "string"}, + {a: "2", b: "string2", c: false, d: "string2"}, + {a: "3", b: "string3", c: true, d: "string3"} + ]); + + [readonly & int, readonly & string, (readonly & boolean)|(readonly & decimal)][]|csv:Error rt16a = + csv:parseList( + [ + ["1", "string", "true"], + ["2", "string2", "false"], + ["3", "string3", "true"] + ]); + + test:assertEquals(rt16a, [ + [1, "string", true], + [2, "string2", false], + [3, "string3", true] + ]); + + [A, B, C][]|csv:Error rt17a = csv:parseList( + [["1", "string", "true"], ["2", "string2", "false"], ["3", "string3", "true"]]); + + test:assertEquals(rt17a, [ + [1d, "string", "true"], + [2d, "string2", "false"], + [3d, "string3", "true"] + ]); + + [A2, B2, C2][]|csv:Error rt18a = csv:parseList( + [["1", "string", "true"], ["2", "string2", "false"], ["3", "string3", "true"]]); + + test:assertEquals(rt18a, [ + [1, "string", true], + [2, "string2", false], + [3, "string3", true] + ]); + + [A2, B2, C2...][]|csv:Error rt19a = csv:parseList( + [["1", "string", "true", "string"], ["2", "string2", "false", "string2"], ["3", "string3", "true", "string3"]]); + + test:assertEquals(rt19a, [ + [1, "string", true, "string"], + [2, "string2", false, "string2"], + [3, "string3", true, "string3"] + ]); + + [C2...][]|csv:Error rt20a = csv:parseList( + [["1", "string", "true", "string"], ["2", "string2", "false", "string2"], ["3", "string3", "true", "string3"]]); + + test:assertEquals(rt20a, [ + ["1", "string", true, "string"], + ["2", "string2", false, "string2"], + ["3", "string3", true, "string3"] + ]); +} + +@test:Config +function testSliceOperation() { + string[][] v = [["1", "2"], ["3", "4"], ["a", "b"]]; + var v2 = [{a: 1, b: 2}, {a: 3, b: 4}, {a: "a", b: "b"}]; + string v3 = string `a,b + 1,2 + 3,4 + a,b`; + + int[2][]|error c = csv:parseList(v); + test:assertEquals(c, [[1, 2], [3, 4]]); + + record {|int...;|}[2]|error c2 = csv:parseList(v, {customHeaders: ["a", "b"]}); + test:assertEquals(c2, [{a: 1, b: 2}, {a: 3, b: 4}]); + + int[2][]|error c3 = csv:transform(v2, {headerOrder: ["a", "b"]}); + test:assertEquals(c3, [[1, 2], [3, 4]]); + + record {|int...;|}[2]|error c4 = csv:transform(v2); + test:assertEquals(c4, [{a: 1, b: 2}, {a: 3, b: 4}]); + + int[2][]|error c5 = csv:parseString(v3); + test:assertEquals(c5, [[1, 2], [3, 4]]); + + record {|int...;|}[2]|error c6 = csv:parseString(v3); + test:assertEquals(c6, [{a: 1, b: 2}, {a: 3, b: 4}]); +} + +@test:Config +function testSliceOperation2() { + string[][] v = [["c", "c", "c"], ["1", "2", "a"], ["c", "c", "c"], ["3", "4", "a"], ["a", "b", "a"]]; + var v2 = [{a: "c", b: "c", c: "c"}, {a: 1, b: 2, c: "c"}, {a: "c", b: "c", c: "c"}, {a: 3, b: 4, c: "c"}, {a: "a", b: "b", c: "c"}]; + string v3 = string `a,b, c + c,c,c + 1,2,c + c,c,c + 3,4,c + a,b,c`; + + int[2][2]|error c = csv:parseList(v, {skipLines: [1, 3]}); + test:assertEquals(c, [[1, 2], [3, 4]]); + + record {|int...;|}[2]|error c2 = csv:parseList(v, {customHeaders: ["a", "b", "c"], skipLines: [1, 3]}); + test:assertEquals(c2, [{a: 1, b: 2}, {a: 3, b: 4}]); + + int[2][2]|error c3 = csv:transform(v2, {headerOrder: ["a", "b", "c"], skipLines: [1, 3]}); + test:assertEquals(c3, [[1, 2], [3, 4]]); + + record {|int...;|}[2]|error c4 = csv:transform(v2, {skipLines: [1, 3]}); + test:assertEquals(c4, [{a: 1, b: 2}, {a: 3, b: 4}]); + + int[2][2]|error c5 = csv:parseString(v3, {skipLines: [1, 3]}); + test:assertEquals(c5, [[1, 2], [3, 4]]); + + record {|int...;|}[2]|error c6 = csv:parseString(v3, {skipLines: [1, 3]}); + test:assertEquals(c6, [{a: 1, b: 2}, {a: 3, b: 4}]); +} \ No newline at end of file diff --git a/ballerina-tests/type-compatible-tests/tests/test_data_values.bal b/ballerina-tests/type-compatible-tests/tests/test_data_values.bal new file mode 100644 index 0000000..5838ec6 --- /dev/null +++ b/ballerina-tests/type-compatible-tests/tests/test_data_values.bal @@ -0,0 +1,54 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +final boolean b1 = true; +final false b2 = false; +final boolean? b3 = (); +final boolean|int b4 = false; + +final () n1 = (); +final int? n2 = (); +final () n3 = null; + +final int i1 = 1; +final int i2 = -2; +final int i3 = int:MAX_VALUE; +final int i4 = int:MIN_VALUE; +final int i5 = 0; +final 2 i6 = 2; +final int? i7 = (); +final int|string i8 = 100; + +final float f1 = 2.234; +final float f2 = -3.21f; +final float f3 = 0; +final float f4 = float:Infinity; +final float f5 = -float:Infinity; +final float f6 = float:NaN; +final 2.3f f7 = 2.3; +final float? f8 = (); +final float|decimal f9 = 1.21; + +final decimal d1 = 2.234; +final decimal d2 = -3.21d; +final decimal d3 = 0; +final 2.3d d4 = 2.3; +final decimal? d5 = (); +final decimal|int d6 = 1.21; + +final string s1 = "string"; +final string s2 = ""; +final string:Char s3 = "a"; diff --git a/ballerina-tests/type-compatible-tests/tests/types.bal b/ballerina-tests/type-compatible-tests/tests/types.bal new file mode 100644 index 0000000..2d6ed41 --- /dev/null +++ b/ballerina-tests/type-compatible-tests/tests/types.bal @@ -0,0 +1,41 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +type AnydataArray1 anydata[]; +type CustomRecord27 record {| + int i1; + string s1; + boolean b1; + () n1; + float f1; + decimal d1; + anydata a1; + json j1; + anydata defaultableField = ""; + anydata? nillableField = (); + string...; +|}; +type AnydataMap map; +type JsonMap map; +type StringMap map; +type CustomTuple7 [int, string, boolean, (), float, decimal, json, anydata, string...]; + +type AnydataArray1Array AnydataArray1[]; +type CustomRecord27Array CustomRecord27[]; +type AnydataMapArray AnydataMap[]; +type JsonMapArray JsonMap[]; +type StringMapArray StringMap[]; +type CustomTuple7Array CustomTuple7[]; diff --git a/ballerina-tests/unicode-tests/Ballerina.toml b/ballerina-tests/unicode-tests/Ballerina.toml new file mode 100644 index 0000000..a2cadc0 --- /dev/null +++ b/ballerina-tests/unicode-tests/Ballerina.toml @@ -0,0 +1,16 @@ +[package] +org = "ballerina" +name = "unicode_tests" +version = "0.1.0" + +[[dependency]] +org = "ballerina" +name = "csv_commons" +repository = "local" +version = "0.1.0" + +[platform.java17] +graalvmCompatible = true + +[build-options] +graalvmBuildOptions = "-H:+IncludeAllLocales" diff --git a/ballerina-tests/unicode-tests/Dependencies.toml b/ballerina-tests/unicode-tests/Dependencies.toml new file mode 100644 index 0000000..9497389 --- /dev/null +++ b/ballerina-tests/unicode-tests/Dependencies.toml @@ -0,0 +1,88 @@ +# AUTO-GENERATED FILE. DO NOT MODIFY. + +# This file is auto-generated by Ballerina for managing dependency versions. +# It should not be modified by hand. + +[ballerina] +dependencies-toml-version = "2" +distribution-version = "2201.9.0" + +[[package]] +org = "ballerina" +name = "data.csv" +version = "0.1.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] +modules = [ + {org = "ballerina", packageName = "data.csv", moduleName = "data.csv"} +] + +[[package]] +org = "ballerina" +name = "jballerina.java" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "lang.__internal" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.object"} +] + +[[package]] +org = "ballerina" +name = "lang.array" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.__internal"} +] + +[[package]] +org = "ballerina" +name = "lang.error" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "lang.object" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "test" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.array"}, + {org = "ballerina", name = "lang.error"} +] +modules = [ + {org = "ballerina", packageName = "test", moduleName = "test"} +] + +[[package]] +org = "ballerina" +name = "unicode_tests" +version = "0.1.0" +dependencies = [ + {org = "ballerina", name = "data.csv"}, + {org = "ballerina", name = "test"} +] +modules = [ + {org = "ballerina", packageName = "unicode_tests", moduleName = "unicode_tests"} +] + diff --git a/ballerina-tests/unicode-tests/tests/escape_character_test.bal b/ballerina-tests/unicode-tests/tests/escape_character_test.bal new file mode 100644 index 0000000..5ed18d1 --- /dev/null +++ b/ballerina-tests/unicode-tests/tests/escape_character_test.bal @@ -0,0 +1,71 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/data.csv; +import ballerina/test; + +@test:Config +function testEscapedCharactres() returns error? { + string csvString = string `a, b + quote\"\"quoted\"quote, 1 + backslash\\backslash, 2 + newline\nnewline, 3 + tab\ttab, 5 + unicode\u0061unicode, 6 + slash\/slash, 9 + quoted string \\'abc\\', 10`; + + record {string a; int b;}[]|csv:Error rec = csv:parseString(csvString); + test:assertEquals(rec, [ + {a: string `quote""quoted"quote`, b: 1}, + {a: string `backslash${"\\"}backslash`, b: 2}, + {a: string `newline${"\n"}newline`, b: 3}, + {a: string `tab${"\t"}tab`, b: 5}, + {a: string `unicodeaunicode`, b: 6}, + {a: string `slash/slash`, b: 9}, + {a: string `quoted string \'abc\'`, b: 10} + ]); +} + +@test:Config +function testEscapedCharactres2() returns error? { + string csvString = string `a, b + backspace\bbackspace, 7`; + + record {string a; int b;}[]|csv:Error rec = csv:parseString(csvString); + test:assertTrue(rec is record {string a; int b;}[]); +} + +@test:Config +function testEscapedCharactres3() returns error? { + string csvString = string ` a c, b + carriage return\r carriage return, 4`; + + record {}[]|csv:Error rec = csv:parseString(csvString); + test:assertEquals(rec, [ + {"a c": string `carriage return${"\r"} carriage return`, b: 4} + ]); +} + +@test:Config +function testEscapedCharactres4() returns error? { + string csvString = string `a, b + form feed\f form feed, 8`; + + record {string a; int b;}[]|csv:Error rec = csv:parseString(csvString); + test:assertTrue(rec is record {string a; int b;}[]); + // TODO: Add tests after supports \f by Ballerina +} diff --git a/ballerina-tests/union-type-tests/Ballerina.toml b/ballerina-tests/union-type-tests/Ballerina.toml new file mode 100644 index 0000000..66da3f7 --- /dev/null +++ b/ballerina-tests/union-type-tests/Ballerina.toml @@ -0,0 +1,16 @@ +[package] +org = "ballerina" +name = "union_type_tests" +version = "0.1.0" + +[[dependency]] +org = "ballerina" +name = "csv_commons" +repository = "local" +version = "0.1.0" + +[platform.java17] +graalvmCompatible = true + +[build-options] +graalvmBuildOptions = "-H:+IncludeAllLocales" diff --git a/ballerina-tests/union-type-tests/Dependencies.toml b/ballerina-tests/union-type-tests/Dependencies.toml new file mode 100644 index 0000000..3eb4395 --- /dev/null +++ b/ballerina-tests/union-type-tests/Dependencies.toml @@ -0,0 +1,98 @@ +# AUTO-GENERATED FILE. DO NOT MODIFY. + +# This file is auto-generated by Ballerina for managing dependency versions. +# It should not be modified by hand. + +[ballerina] +dependencies-toml-version = "2" +distribution-version = "2201.9.0" + +[[package]] +org = "ballerina" +name = "csv_commons" +version = "0.1.0" +scope = "testOnly" +modules = [ + {org = "ballerina", packageName = "csv_commons", moduleName = "csv_commons"} +] + +[[package]] +org = "ballerina" +name = "data.csv" +version = "0.1.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] +modules = [ + {org = "ballerina", packageName = "data.csv", moduleName = "data.csv"} +] + +[[package]] +org = "ballerina" +name = "jballerina.java" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "lang.__internal" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.object"} +] + +[[package]] +org = "ballerina" +name = "lang.array" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.__internal"} +] + +[[package]] +org = "ballerina" +name = "lang.error" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "lang.object" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "test" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.array"}, + {org = "ballerina", name = "lang.error"} +] +modules = [ + {org = "ballerina", packageName = "test", moduleName = "test"} +] + +[[package]] +org = "ballerina" +name = "union_type_tests" +version = "0.1.0" +dependencies = [ + {org = "ballerina", name = "csv_commons"}, + {org = "ballerina", name = "data.csv"}, + {org = "ballerina", name = "test"} +] +modules = [ + {org = "ballerina", packageName = "union_type_tests", moduleName = "union_type_tests"} +] + diff --git a/ballerina-tests/union-type-tests/tests/test_data_values.bal b/ballerina-tests/union-type-tests/tests/test_data_values.bal new file mode 100644 index 0000000..23c0be9 --- /dev/null +++ b/ballerina-tests/union-type-tests/tests/test_data_values.bal @@ -0,0 +1,23 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +final string csvStringData1 = string ` + a, b, c, d, e, f + 1, string1, true, 2.234, 2.234, () + 2, string2, false, 0, 0, null + 3, string3, false, 1.23, 1.23, () + 4, string4, true, -6.51, -6.51, () + 5, string5, true, 3, 3, ()`; diff --git a/ballerina-tests/union-type-tests/tests/test_union_types_with_headers.bal b/ballerina-tests/union-type-tests/tests/test_union_types_with_headers.bal new file mode 100644 index 0000000..85a4f8c --- /dev/null +++ b/ballerina-tests/union-type-tests/tests/test_union_types_with_headers.bal @@ -0,0 +1,254 @@ +import ballerina/data.csv; +import ballerina/test; + +final csv:TransformOptions op1 = {headerOrder: ["a", "b", "c", "d"], outputWithHeaders: false}; +final csv:TransformOptions op2 = {headerOrder: ["a", "b", "c", "d"], outputWithHeaders: true}; +final csv:TransformOptions op3 = {outputWithHeaders: true}; +final csv:TransformOptions op4 = {headerOrder: ["a", "b"], outputWithHeaders: true}; + +final csv:ParseOptions op5 = {outputWithHeaders: true}; + +final csv:ParseListOptions op6 = {outputWithHeaders: true}; +final csv:ParseListOptions op7 = {outputWithHeaders: true, headerRows: 1}; +final csv:ParseListOptions op8 = {outputWithHeaders: true, headerRows: 2}; +final csv:ParseListOptions op9 = {outputWithHeaders: true, headerRows: 2, customHeaders: ["a", "b", "c", "d"]}; +final csv:ParseListOptions op10 = {outputWithHeaders: true, headerRows: 1, customHeaders: ["a", "b", "c", "d"]}; +final csv:ParseListOptions op11 = {outputWithHeaders: true, customHeaders: ["a", "b", "c", "d"]}; +final csv:ParseListOptions op12 = {headerRows: 1, customHeaders: ["a", "b", "c", "d"]}; +final csv:ParseListOptions op13 = {customHeaders: ["a", "b", "c", "d"]}; + +type UnionType1 boolean[][]|string[][]; + +type UnionType2 ([boolean...]|[string, string...])[]; + +type UnionType3 record {int d1;}[]|record {}[]; + +type UnionType4 (map|map)[]; + +final string[][] csv1 = [["1", "2", "3", "4"], ["5", "6", "7", "8"], ["9", "10", "11", "12"], ["13", "14", "15", "16"]]; +final var csv2 = [{a: 1, b: 2, c: 3, d: 4}, {a: 5, b: 6, c: 7, d: 8}, {a: 9, b: 10, c: 11, d: 12}, {a: 13, b: 14, c: 15, d: 16}]; +final var csv3 = string `a,b,c,d + 1,2,3,4 + 5,6,7,8 + 9,10,11,12 + 13,14,15,16`; +final var csv4 = [{a: "1", b: "2", c: "3", d: "4"}, {a: "5", b: "6", c: "7", d: "8"}, {a: "9", b: "10", c: "11", d: "12"}, {a: "13", b: "14", c: "15", d: "16"}]; + +final var result1 = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]]; +final var result2 = [{a: 1, b: 2, c: 3, d: 4}, {a: 5, b: 6, c: 7, d: 8}, {a: 9, b: 10, c: 11, d: 12}, {a: 13, b: 14, c: 15, d: 16}]; +final var result3 = [{a: "1", b: "2", c: "3", d: "4"}, {a: "5", b: "6", c: "7", d: "8"}, {a: "9", b: "10", c: "11", d: "12"}, {a: "13", b: "14", c: "15", d: "16"}]; +final var result4 = [["1", "2", "3", "4"], ["5", "6", "7", "8"], ["9", "10", "11", "12"], ["13", "14", "15", "16"]]; +final var result5 = [["a", "b", "c", "d"], [1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]]; +final var result6 = [["a", "b", "c", "d"], ["1", "2", "3", "4"], ["5", "6", "7", "8"], ["9", "10", "11", "12"], ["13", "14", "15", "16"]]; +final var result7 = [{'1: 1, '2: 2, '3: 3, '4: 4}, {'1: 5, '2: 6, '3: 7, '4: 8}, {'1: 9, '2: 10, '3: 11, '4: 12}, {'1: 13, '2: 14, '3: 15, '4: 16}]; +final var result8 = [{'1: "1", '2: "2", '3: "3", '4: "4"}, {'1: "5", '2: "6", '3: "7", '4: "8"}, {'1: "9", '2: "10", '3: "11", '4: "12"}, {'1: "13", '2: "14", '3: "15", '4: "16"}]; + +@test:Config +function testParseStringWithMapWithOutputHeaders() { + UnionType3|csv:Error psu3 = csv:parseString(csv3); + test:assertEquals(psu3, result2); + + psu3 = csv:parseString(csv3, op5); + test:assertEquals(psu3, result2); + + UnionType4|csv:Error psu4 = csv:parseString(csv3); + test:assertEquals(psu4, result3); + + psu4 = csv:parseString(csv3, op5); + test:assertEquals(psu4, result3); +} + +@test:Config +function testParseStringWithListWithOutputHeaders() { + UnionType1|csv:Error psu1 = csv:parseString(csv3); + test:assertEquals(psu1, result4); + + psu1 = csv:parseString(csv3, op5); + test:assertEquals(psu1, result6); + + UnionType2|csv:Error psu2 = csv:parseString(csv3); + test:assertEquals(psu2, result4); + + psu2 = csv:parseString(csv3, op5); + test:assertEquals(psu2, result6); +} + +@test:Config +function testTransformWithMapWithOutputHeaders() { + UnionType3|csv:Error psu3 = csv:transform(csv2); + test:assertEquals(psu3, result2); + + psu3 = csv:transform(csv2, op1); + test:assertEquals(psu3, result2); + + psu3 = csv:transform(csv2, op2); + test:assertEquals(psu3, result2); + + psu3 = csv:transform(csv2, op3); + test:assertEquals(psu3, result2); + + psu3 = csv:transform(csv2, op4); + test:assertEquals(psu3, result2); + + UnionType4|csv:Error psu4 = csv:transform(csv4); + test:assertEquals(psu4, result3); + + psu4 = csv:transform(csv4, op1); + test:assertEquals(psu4, result3); + + psu4 = csv:transform(csv4, op2); + test:assertEquals(psu4, result3); + + psu4 = csv:transform(csv4, op3); + test:assertEquals(psu4, result3); + + psu4 = csv:transform(csv4, op4); + test:assertEquals(psu4, result3); +} + +@test:Config +function testTransformWithListWithOutputHeaders() { + UnionType1|csv:Error psu1 = csv:transform(csv4); + test:assertEquals(psu1, result4); + + psu1 = csv:transform(csv4, op1); + test:assertEquals(psu1, result4); + + psu1 = csv:transform(csv4, op2); + test:assertEquals(psu1, result6); + + psu1 = csv:transform(csv4, op3); + test:assertEquals(psu1, result6); + + UnionType2|csv:Error psu2 = csv:transform(csv4); + test:assertEquals(psu2, result4); + + psu2 = csv:transform(csv4, op1); + test:assertEquals(psu2, result4); + + psu2 = csv:transform(csv4, op2); + test:assertEquals(psu2, result6); + + psu2 = csv:transform(csv4, op3); + test:assertEquals(psu2, result6); +} + +@test:Config +function testParseListWithMapWithOutputHeaders() { + UnionType3|csv:Error psu3 = csv:parseList(csv1); + test:assertEquals(psu3, result7); + + psu3 = csv:parseList(csv1, op6); + test:assertEquals(psu3, result7); + + psu3 = csv:parseList(csv1, op7); + test:assertEquals(psu3, [{"1": 5, "2": 6, "3": 7, "4": 8}, {"1": 9, "2": 10, "3": 11, "4": 12}, {"1": 13, "2": 14, "3": 15, "4": 16}]); + + psu3 = csv:parseList(csv1, op9); + test:assertEquals(psu3, [{"a": 9, "b": 10, "c": 11, "d": 12}, {"a": 13, "b": 14, "c": 15, "d": 16}]); + + UnionType4|csv:Error psu4 = csv:parseList(csv1); + test:assertEquals(psu4, result8); + + psu4 = csv:parseList(csv1, op6); + test:assertEquals(psu4, result8); + + psu4 = csv:parseList(csv1, op7); + test:assertEquals(psu4, [{"1": "5", "2": "6", "3": "7", "4": "8"}, {"1": "9", "2": "10", "3": "11", "4": "12"}, {"1": "13", "2": "14", "3": "15", "4": "16"}]); + + psu4 = csv:parseList(csv1, op9); + test:assertEquals(psu4, [{"a": "9", "b": "10", "c": "11", "d": "12"}, {"a": "13", "b": "14", "c": "15", "d": "16"}]); +} + +@test:Config +function testParseListWithListWithOutputHeaders() { + UnionType1|csv:Error psu1 = csv:parseList(csv1); + test:assertEquals(psu1, result4); + + psu1 = csv:parseList(csv1, op6); + test:assertEquals(psu1, result4); + + psu1 = csv:parseList(csv1, op7); + test:assertEquals(psu1, result4); + + psu1 = csv:parseList(csv1, op9); + test:assertEquals(psu1, [["a","b","c","d"],["9","10","11","12"],["13","14","15","16"]]); + + UnionType2|csv:Error psu2 = csv:parseList(csv1); + test:assertEquals(psu2, result4); + + psu2 = csv:parseList(csv1, op6); + test:assertEquals(psu2, result4); + + psu2 = csv:parseList(csv1, op7); + test:assertEquals(psu2, result4); + + psu2 = csv:parseList(csv1, op9); + test:assertEquals(psu2, [["a","b","c","d"],["9","10","11","12"],["13","14","15","16"]]); +} + +@test:Config +function testParseListWithMapWithOutputHeaders2() { + UnionType3|csv:Error psu3 = csv:parseList(csv1); + test:assertEquals(psu3, result7); + + psu3 = csv:parseList(csv1, op10); + test:assertEquals(psu3, [{"a":5,"b":6,"c":7,"d":8},{"a":9,"b":10,"c":11,"d":12},{"a":13,"b":14,"c":15,"d":16}]); + + psu3 = csv:parseList(csv1, op11); + test:assertEquals(psu3, result2); + + psu3 = csv:parseList(csv1, op12); + test:assertEquals(psu3, [{"a":5,"b":6,"c":7,"d":8},{"a":9,"b":10,"c":11,"d":12},{"a":13,"b":14,"c":15,"d":16}]); + + psu3 = csv:parseList(csv1, op13); + test:assertEquals(psu3, result2); + + UnionType4|csv:Error psu4 = csv:parseList(csv1); + test:assertEquals(psu4, result8); + + psu4 = csv:parseList(csv1, op10); + test:assertEquals(psu4, [{a: "5", b: "6", c: "7", d: "8"}, {a: "9", b: "10", c: "11", d: "12"}, {a: "13", b: "14", c: "15", d: "16"}]); + + psu4 = csv:parseList(csv1, op11); + test:assertEquals(psu4, result3); + + psu4 = csv:parseList(csv1, op12); + test:assertEquals(psu4, [{a: "5", b: "6", c: "7", d: "8"}, {a: "9", b: "10", c: "11", d: "12"}, {a: "13", b: "14", c: "15", d: "16"}]); + + psu4 = csv:parseList(csv1, op13); + test:assertEquals(psu4, result3); +} + +@test:Config +function testParseListWithListWithOutputHeaders2() { + UnionType1|csv:Error psu1 = csv:parseList(csv1); + test:assertEquals(psu1, result4); + + psu1 = csv:parseList(csv1, op10); + test:assertEquals(psu1, [["a","b","c","d"],["5","6","7","8"],["9","10","11","12"],["13","14","15","16"]]); + + psu1 = csv:parseList(csv1, op11); + test:assertEquals(psu1, result6); + + psu1 = csv:parseList(csv1, op12); + test:assertEquals(psu1, [["5","6","7","8"],["9","10","11","12"],["13","14","15","16"]]); + + psu1 = csv:parseList(csv1, op13); + test:assertEquals(psu1, result4); + + UnionType2|csv:Error psu2 = csv:parseList(csv1); + test:assertEquals(psu2, result4); + + psu2 = csv:parseList(csv1, op10); + test:assertEquals(psu2, [["a","b","c","d"],["5","6","7","8"],["9","10","11","12"],["13","14","15","16"]]); + + psu2 = csv:parseList(csv1, op11); + test:assertEquals(psu2, [["a","b","c","d"], ["1", "2", "3", "4"],["5","6","7","8"],["9","10","11","12"],["13","14","15","16"]]); + + psu2 = csv:parseList(csv1, op12); + test:assertEquals(psu2, [["5","6","7","8"],["9","10","11","12"],["13","14","15","16"]]); + + psu2 = csv:parseList(csv1, op13); + test:assertEquals(psu2, [["1", "2", "3", "4"],["5","6","7","8"],["9","10","11","12"],["13","14","15","16"]]); +} diff --git a/ballerina-tests/union-type-tests/tests/test_with_intersection_types.bal b/ballerina-tests/union-type-tests/tests/test_with_intersection_types.bal new file mode 100644 index 0000000..7661513 --- /dev/null +++ b/ballerina-tests/union-type-tests/tests/test_with_intersection_types.bal @@ -0,0 +1,183 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +import ballerina/data.csv; +import ballerina/test; + +@test:Config +function testIntersectionExpectedTypes() returns error? { + (int[] & readonly)[]|csv:Error a = csv:parseString(string `a,b + 1,2 + 4,5`); + test:assertTrue(a is (int[] & readonly)[]); + test:assertEquals(a, [[1, 2], [4, 5]]); + + ([string, string])[] & readonly|csv:Error a2 = csv:parseString(string `a,b + a,a + c,c`); + test:assertTrue(a2 is [string, string][] & readonly); + test:assertEquals(a2, [["a", "a"], ["c", "c"]]); + + (record {int a; string b;} & readonly)[]|csv:Error a3 = csv:parseString(string `a,b + 1,2 + 4,5`); + test:assertTrue(a3 is (record {int a; string b;} & readonly)[]); + test:assertEquals(a3, [{a: 1, b: "2"}, {a: 4, b: "5"}]); + + record {|string...;|}[] & readonly|csv:Error a4 = csv:parseString(string `a,b + a,a + c,c`); + test:assertTrue(a4 is record {|string...;|}[] & readonly); + test:assertEquals(a4, [{a: "a", b: "a"}, {a: "c", b: "c"}]); + + ([int] & readonly)[]|csv:Error a5 = csv:parseString(string `a,b + 1,2 + 4,5`); + test:assertTrue(a5 is ([int] & readonly)[]); + test:assertEquals(a5, [[1], [4]]); + + ([string, string])[] & readonly|csv:Error a6 = csv:parseString(string `a,b + a,a + c,c`); + test:assertTrue(a6 is [string, string][] & readonly); + test:assertEquals(a6, [["a", "a"], ["c", "c"]]); + + (record {int a; string b;} & readonly)[]|csv:Error a7 = csv:parseString(string `a,b + 1,2 + 4,5`); + test:assertTrue(a7 is record {int a; string b;}[] & readonly); + test:assertEquals(a7, [{a: 1, b: "2"}, {a: 4, b: "5"}]); + + map[] & readonly|csv:Error a8 = csv:parseString(string `a,b + a,a + c,c`); + test:assertTrue(a8 is map[] & readonly); + test:assertEquals(a8, [{a: "a", b: "a"}, {a: "c", b: "c"}]); + + (((int[] & readonly)|([string, string] & readonly)) & readonly)[]|csv:Error a9 = csv:parseString(string `a,b + 1,2 + a,a`); + test:assertTrue(a9 is (((int[] & readonly)|([string, string] & readonly)) & readonly)[]); + test:assertEquals(a9, [["1", "2"], ["a", "a"]]); + + ((record {string a; string b;} & readonly)|(record {int a; int b;} & readonly))[] + & readonly|csv:Error a10 = csv:parseString(string `a,b + a,a + 1,2`); + test:assertTrue(a10 is ((record {string a; string b;} & readonly)|(record {int a; int b;} & readonly))[] & readonly); + test:assertEquals(a10, [{a: "a", b: "a"}, {a: "1", b: "2"}]); +} + +@test:Config +function testIntersectionExpectedTypes2() returns error? { + (int[] & readonly)[]|csv:Error a = csv:transform([{"a": 1, "b": 2}, {"a": 4, "b": 5}], {headerOrder: ["a", "b"]}); + test:assertTrue(a is (int[] & readonly)[]); + test:assertEquals(a, [[1, 2], [4, 5]]); + + ([string, string])[] & readonly|csv:Error a2 = csv:transform([{"a": "a", "b": "a"}, {"a": "c", "b": "c"}], {headerOrder: ["a", "b"]}); + test:assertTrue(a2 is [string, string][] & readonly); + test:assertEquals(a2, [["a", "a"], ["c", "c"]]); + + (record {int a; string b;} & readonly)[]|csv:Error a3 = csv:transform([{"a": 1, "b": "2"}, {"a": 4, "b": "5"}], {}); + test:assertTrue(a3 is (record {int a; string b;} & readonly)[]); + test:assertEquals(a3, [{a: 1, b: "2"}, {a: 4, b: "5"}]); + + record {|string...;|}[] & readonly|csv:Error a4 = csv:transform([{"a": "a", "b": "a"}, {"a": "c", "b": "c"}], {}); + test:assertTrue(a4 is record {|string...;|}[] & readonly); + test:assertEquals(a4, [{a: "a", b: "a"}, {a: "c", b: "c"}]); + + ([int] & readonly)[]|csv:Error a5 = csv:transform([{"a": 1, "b": 2}, {"a": 4, "b": 5}], {headerOrder: ["a", "b"]}); + test:assertTrue(a5 is ([int] & readonly)[]); + test:assertEquals(a5, [[1], [4]]); + + ([string, string])[] & readonly|csv:Error a6 = csv:transform([{"a": "a", "b": "a"}, {"a": "c", "b": "c"}], {headerOrder: ["a", "b"]}); + test:assertTrue(a6 is [string, string][] & readonly); + test:assertEquals(a6, [["a", "a"], ["c", "c"]]); + + (record {int a; string b;} & readonly)[]|csv:Error a7 = csv:transform([{"a": 1, "b": "2"}, {"a": 4, "b": "5"}], {}); + test:assertTrue(a7 is record {int a; string b;}[] & readonly); + test:assertEquals(a7, [{a: 1, b: "2"}, {a: 4, b: "5"}]); + + map[] & readonly|csv:Error a8 = csv:transform([{"a": "a", "b": "a"}, {"a": "c", "b": "c"}], {}); + test:assertTrue(a8 is map[] & readonly); + test:assertEquals(a8, [{a: "a", b: "a"}, {a: "c", b: "c"}]); + + (((int[] & readonly)|([string, string] & readonly)) & readonly)[]|csv:Error a9 = csv:transform([{"a": 1, "b": 2}, {"a": "a", "b": "b"}], {headerOrder: ["a", "b"]}); + test:assertTrue(a9 is error); + test:assertEquals(( a9).message(), "source value cannot converted in to the '((int[] & readonly)|([string,string] & readonly))[]'"); + + (((int[] & readonly)|([string, string] & readonly)) & readonly)[]|csv:Error a9_2 = csv:transform([{"a": "1", "b": "2"}, {"a": "a", "b": "b"}], {headerOrder: ["a", "b"]}); + test:assertTrue(a9_2 is (((int[] & readonly)|([string, string] & readonly)) & readonly)[]); + test:assertEquals(a9_2, [["1", "2"], ["a", "b"]]); + + ((record {string a; string b;} & readonly)|(record {int a; int b;} & readonly))[] + & readonly|csv:Error a10 = csv:transform([{"a": "a", "b": "a"}, {"a": "1", "b": "2"}], {}); + test:assertTrue(a10 is ((record {string a; string b;} & readonly)|(record {int a; int b;} & readonly))[] & readonly); + test:assertEquals(a10, [{a: "a", b: "a"}, {a: "1", b: "2"}]); + + ((record {string a; string b;} & readonly)|(record {int a; int b;} & readonly))[] + & readonly|csv:Error a10_2 = csv:transform([{"a": "a", "b": "a"}, {"a": 1, "b": 2}], {}); + test:assertTrue(a10_2 is error); + test:assertEquals(( a10_2).message(), "source value cannot converted in to the '((union_type_tests:record {| string a; string b; anydata...; |} & readonly)|(union_type_tests:record {| int a; int b; anydata...; |} & readonly))[]'"); +} + +@test:Config +function testIntersectionExpectedTypes3() returns error? { + (int[] & readonly)[]|csv:Error a = csv:parseList([["1", "2"], ["4", "5"]], {}); + test:assertTrue(a is (int[] & readonly)[]); + test:assertEquals(a, [[1, 2], [4, 5]]); + + ([string, string])[] & readonly|csv:Error a2 = csv:parseList([["a", "a"], ["c", "c"]], {}); + test:assertTrue(a2 is [string, string][] & readonly); + test:assertEquals(a2, [["a", "a"], ["c", "c"]]); + + (record {int a; string b;} & readonly)[]|csv:Error a3 = csv:parseList([["1", "2"], ["4", "5"]], {customHeaders: ["a", "b"]}); + test:assertTrue(a3 is (record {int a; string b;} & readonly)[]); + test:assertEquals(a3, [{a: 1, b: "2"}, {a: 4, b: "5"}]); + + record {|string...;|}[] & readonly|csv:Error a4 = csv:parseList([["a", "a"], ["c", "c"]], {customHeaders: ["a", "b"]}); + test:assertTrue(a4 is record {|string...;|}[] & readonly); + test:assertEquals(a4, [{a: "a", b: "a"}, {a: "c", b: "c"}]); + + ([int] & readonly)[]|csv:Error a5 = csv:parseList([["1", "2"], ["4", "5"]], {}); + test:assertTrue(a5 is ([int] & readonly)[]); + test:assertEquals(a5, [[1], [4]]); + + ([string, string])[] & readonly|csv:Error a6 = csv:parseList([["a", "a"], ["c", "c"]], {}); + test:assertTrue(a6 is [string, string][] & readonly); + test:assertEquals(a6, [["a", "a"], ["c", "c"]]); + + (record {int a; string b;} & readonly)[]|csv:Error a7 = csv:parseList([["1", "2"], ["4", "5"]], {customHeaders: ["a", "b"]}); + test:assertTrue(a7 is record {int a; string b;}[] & readonly); + test:assertEquals(a7, [{a: 1, b: "2"}, {a: 4, b: "5"}]); + + map[] & readonly|csv:Error a8 = csv:parseList([["a", "a"], ["c", "c"]], {customHeaders: ["a", "b"]}); + test:assertTrue(a8 is map[] & readonly); + test:assertEquals(a8, [{a: "a", b: "a"}, {a: "c", b: "c"}]); + + (((int[] & readonly)|([string, string] & readonly)) & readonly)[]|csv:Error a9 = csv:parseList([["1", "2"], ["a", "b"]], {}); + test:assertTrue(a9 is (((int[] & readonly)|([string, string] & readonly)) & readonly)[]); + test:assertEquals(a9, [["1", "2"], ["a", "b"]]); + + ((record {string a; string b;} & readonly)|(record {int a; int b;} & readonly))[] + & readonly|csv:Error a10 = csv:parseList([["a", "a"], ["1", "2"]], {customHeaders: ["a", "b"]}); + test:assertTrue(a10 is ((record {string a; string b;} & readonly)|(record {int a; int b;} & readonly))[] & readonly); + test:assertEquals(a10, [{a: "a", b: "a"}, {a: "1", b: "2"}]); + + ((record {int a; int b;} & readonly)|(record {string a; string b;} & readonly))[] + & readonly|csv:Error a11 = csv:parseList([["a", "a"], ["1", "2"]], {customHeaders: ["a", "b"]}); + test:assertTrue(a11 is ((record {string a; string b;} & readonly)|(record {int a; int b;} & readonly))[] & readonly); + test:assertEquals(a11, [{a: "a", b: "a"}, {a: "1", b: "2"}]); +} diff --git a/ballerina-tests/union-type-tests/tests/test_with_singleton_test.bal b/ballerina-tests/union-type-tests/tests/test_with_singleton_test.bal new file mode 100644 index 0000000..57bfa05 --- /dev/null +++ b/ballerina-tests/union-type-tests/tests/test_with_singleton_test.bal @@ -0,0 +1,231 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/data.csv; +import ballerina/test; +import ballerina/csv_commons as common; + +type Singleton 1; + +@test:Config +function testSingletonExpectedTypes() returns error? { + 1[][]|csv:Error a = csv:parseString(string `a, b, c + 1, 1, 1 + 1, 1,1 `); + test:assertEquals(a, [[1, 1, 1], [1, 1, 1]]); + + record {1|2 a; 1 b;}[]|csv:Error a2 = csv:parseString(string `a, b, c + 1, 1, 1 + 1, 1,1 `); + test:assertEquals(a2, [{a: 1, b: 1, c: 1}, {a: 1, b: 1, c: 1}]); + + record {|1 a; 1|2...;|}[]|csv:Error a3 = csv:parseString(string `a, b, c + 1, 1, 1 + 1, 1,1 `); + test:assertEquals(a3, [{a: 1, b: 1, c: 1}, {a: 1, b: 1, c: 1}]); + + [Singleton, Singleton...][]|csv:Error a4 = csv:parseString(string `a, b, c + 1, 1, 1 + 1, 1,1 `); + test:assertEquals(a4, [[1, 1, 1], [1, 1, 1]]); + + record {|1|"a" a; 1 b;|}[]|csv:Error a5 = csv:parseString(string `a, b, c + 1, 1, 1 + 1, 1,1 `); + test:assertEquals(a5, [{a: 1, b: 1}, {a: 1, b: 1}]); + + [Singleton, Singleton][]|csv:Error a6 = csv:parseString(string `a, b, c + 1, 1, 1 + 1, 1,1 `); + test:assertEquals(a6, [[1, 1], [1, 1]]); + + record {|"a"|"c" a; "b" b;|}[]|csv:Error a7 = csv:parseString(string `a, b, c + a, c, 1 + 1, 1,1 `); + test:assertTrue(a7 is csv:Error); + test:assertEquals((a7).message(), common:generateErrorMessageForInvalidCast("c", "\"b\"")); + + ["a"|"d", "b"][]|csv:Error a8 = csv:parseString(string `a, b, c + a, b, 1 + c, b,1 `); + test:assertTrue(a8 is csv:Error); + test:assertEquals((a8).message(), common:generateErrorMessageForInvalidCast("c", "(\"a\"|\"d\")")); +} + +@test:Config +function testSingletonExpectedTypes2() returns error? { + 1[][]|csv:Error a = csv:parseString(string `a, b, c + 1, 1, 1 + 1, 1,1 `); + test:assertEquals(a, [[1, 1, 1], [1, 1, 1]]); + + record {1|2 a; 1 b;}[]|csv:Error a2 = csv:parseString(string `a, b, c + 1, 1, 1 + 1, 1,1 `); + test:assertEquals(a2, [{a: 1, b: 1, c: 1}, {a: 1, b: 1, c: 1}]); + + record {|1 a; 1|2...;|}[]|csv:Error a3 = csv:parseString(string `a, b, c + 1, 1, 1 + 1, 1,1 `); + test:assertEquals(a3, [{a: 1, b: 1, c: 1}, {a: 1, b: 1, c: 1}]); + + [Singleton, Singleton...][]|csv:Error a4 = csv:parseString(string `a, b, c + 1, 1, 1 + 1, 1,1 `); + test:assertEquals(a4, [[1, 1, 1], [1, 1, 1]]); + + record {|1|"a" a; 1 b;|}[]|csv:Error a5 = csv:parseString(string `a, b, c + 1, 1, 1 + 1, 1,1 `); + test:assertEquals(a5, [{a: 1, b: 1}, {a: 1, b: 1}]); + + [Singleton, Singleton][]|csv:Error a6 = csv:parseString(string `a, b, c + 1, 1, 1 + 1, 1,1 `); + test:assertEquals(a6, [[1, 1], [1, 1]]); + + record {|"a"|"c" a; "b" b;|}[]|csv:Error a7 = csv:parseString(string `a, b, c + a, c, 1 + 1, 1,1 `); + test:assertTrue(a7 is csv:Error); + test:assertEquals((a7).message(), common:generateErrorMessageForInvalidCast("c", "\"b\"")); + + ["a"|"d", "b"][]|csv:Error a8 = csv:parseString(string `a, b, c + a, b, 1 + c, b,1 `); + test:assertTrue(a8 is csv:Error); + test:assertEquals((a8).message(), common:generateErrorMessageForInvalidCast("c", "(\"a\"|\"d\")")); +} + +type SubType byte|int:Signed8|int:Signed16|int:Signed32|int:Unsigned8|int:Unsigned16|int:Unsigned32|string:Char; + +type SubtypeRecord record { + byte a; int:Signed8 c; int:Signed16 d; int:Signed32 e; + string:Char f; int:Unsigned8 g; int:Unsigned16 h; int:Unsigned32 i; +}; + +type SubtypeRecord2 record {| + byte a; int:Signed8 c; +|}; + +type SubtypeRecord3 record {| + SubType...; +|}; + +type SubtypeTuple [ + byte, int:Signed8, int:Signed16, int:Signed32, + string:Char, int:Unsigned8, int:Unsigned16, int:Unsigned32 +]; + +type SubtypeTuple2 [SubType, SubType]; + +type SubtypeTuple3 [SubType...]; + +@test:Config +function testSubtypeExpectedTypes() returns error? { + var value1 = [{a: 1, c: 1, d: 1, e: 1, f: "a", g: 1, h: 1, i: 1}, + {a: 1, c: 1, d: 1, e: 1, f: "a", g: 1, h: 1, i: 1}]; + var value2 = [["1", "1", "1", "1", "a", "1", "1", "1"], + ["1", "1", "1", "1", "a", "1", "1", "1"]]; + var value3 = [[1, 1, 1, 1, "a", 1, 1, 1], + [1, 1, 1, 1, "a", 1, 1, 1]]; + var value4 = [[1, 1, 1, 1, "a", 1, 1, 1], + [1, 1, 1, 1, "a", 1, 1, 1]]; + var value5 = [{a: 1, c: 1, d: 1, e: 1, f: "a", g: 1, h: 1, i: 1}, + {a: 1, c: 1, d: 1, e: 1, f: "a", g: 1, h: 1, i: 1}]; + var value6 = [[1, 1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 1]]; + var value7 = [{a: 1, c: 1, d: 1, e: 1, g: 1, h: 1, i: 1}, + {a: 1, c: 1, d: 1, e: 1, g: 1, h: 1, i: 1}]; + + SubtypeRecord[]|csv:Error a = csv:parseString(string `a, c, d, e, f, g, h, i + 1, 1, 1, 1, a, 1, 1, 1 + 1, 1, 1, 1, a, 1, 1, 1 `); + test:assertEquals(a, value1); + + SubtypeRecord2[]|csv:Error a2 = csv:parseString(string `a, c, d, e, f, g, h, i + 1, 1, 1, 1, a, 1, 1, 1 + 1, 1, 1, 1, a, 1, 1, 1 `); + + test:assertEquals(a2, [{a: 1, c: 1}, {a: 1, c: 1}]); + + SubtypeRecord3[]|csv:Error a3 = csv:parseString(string `a, c, d, e, g, h, i + 1, 1, 1, 1, 1, 1, 1 + 1, 1, 1, 1, 1, 1, 1 `); + test:assertEquals(a3, value7); + + SubtypeRecord3[]|csv:Error a3_2 = csv:parseString(string `a, c, d, e, f, g, h, i + 1, 1, 1, 1, a, 1, 1, 1 + 1, 1, 1, 1, a, 1, 1, 1 `); + test:assertEquals(a3_2, value5); + + SubtypeTuple[]|csv:Error a4 = csv:parseString(string `a, c, d, e, f, g, h, i + 1, 1, 1, 1, a, 1, 1, 1 + 1, 1, 1, 1, a, 1, 1, 1 `); + test:assertEquals(a4, value3); + + SubtypeTuple2[]|csv:Error a5 = csv:parseString(string `a, c, d, e, f, g, h, i + 1, 1, 1, 1, a, 1, 1, 1 + 1, 1, 1, 1, a, 1, 1, 1 `); + test:assertEquals(a5, [[1, 1], [1, 1]]); + + SubtypeTuple3[]|csv:Error a6 = csv:parseString(string `a, c, d, e, f, g, h, i + 1, 1, 1, 1, a, 1, 1, 1 + 1, 1, 1, 1, a, 1, 1, 1 `); + test:assertEquals(a6, value4); + + SubtypeTuple3[]|csv:Error a6_2 = csv:parseString(string `a, c, d, e, g, h, i + 1, 1, 1, 1, 1, 1, 1 + 1, 1, 1, 1, 1, 1, 1 `); + test:assertEquals(a6_2, value6); + + SubtypeRecord[]|csv:Error a7 = csv:transform(value1, {}); + test:assertEquals(a7, value1); + + SubtypeRecord2[]|csv:Error a8 = csv:transform(value1, {}); + + test:assertEquals(a8, [{a: 1, c: 1}, {a: 1, c: 1}]); + + SubtypeRecord3[]|csv:Error a9 = csv:transform(value1, {}); + test:assertEquals(a9, value1); + + SubtypeTuple[]|csv:Error a10 = csv:transform(value1, {headerOrder: ["a", "c", "d", "e", "f", "g", "h", "i"]}); + test:assertEquals(a10, value3); + + SubtypeTuple2[]|csv:Error a11 = csv:transform(value1, {headerOrder: ["a", "c", "d", "e", "f", "g", "h", "i"]}); + test:assertEquals(a11, [[1, 1], [1, 1]]); + + SubtypeTuple3[]|csv:Error a12 = csv:transform(value1, {headerOrder: ["a", "c", "d", "e", "f", "g", "h", "i"]}); + test:assertEquals(a12, value3); + + SubtypeRecord[]|csv:Error a13 = csv:parseList(value2, {customHeaders: ["a", "c", "d", "e", "f", "g", "h", "i"]}); + test:assertEquals(a13, value1); + + SubtypeRecord2[]|csv:Error a14 = csv:parseList(value2, {customHeaders: ["a", "c", "d", "e", "f", "g", "h", "i"]}); + test:assertEquals(a14, [{a: 1, c: 1}, {a: 1, c: 1}]); + + SubtypeRecord3[]|csv:Error a15 = csv:parseList(value2, {customHeaders: ["a", "c", "d", "e", "f", "g", "h", "i"]}); + test:assertEquals(a15, value1); + + SubtypeTuple[]|csv:Error a16 = csv:parseList(value2, {}); + test:assertEquals(a16, value3); + + SubtypeTuple2[]|csv:Error a17 = csv:parseList(value2, {}); + test:assertEquals(a17, [[1, 1], [1, 1]]); + + SubtypeTuple3[]|csv:Error a18 = csv:parseList(value2, {}); + test:assertEquals(a18, value3); +} diff --git a/ballerina-tests/union-type-tests/tests/test_with_union_types.bal b/ballerina-tests/union-type-tests/tests/test_with_union_types.bal new file mode 100644 index 0000000..594401b --- /dev/null +++ b/ballerina-tests/union-type-tests/tests/test_with_union_types.bal @@ -0,0 +1,1294 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/data.csv; +import ballerina/test; + +type RecA record {int a; string b; boolean c; decimal d; float e; () f;}; +type RecB record {|string...;|}; +type RecC record {int a; int b; int c;}; +type TupA [int, string, boolean, decimal, float, ()]; +type TupB [int...]; +type TupC [int, int, int]; + +@test:Config +function testParseToStringWithUnionExpectedTypes() returns error? { + (RecA|RecC)[]|csv:Error csv1op1 = csv:parseString(csvStringData1, {header: 1}); + test:assertEquals(csv1op1, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 2, b: "string2", c: false, d: 0, e: 0, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 4, b: "string4", c: true, d: -6.51, e: -6.51, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + (RecA|RecC)[]|csv:Error csv1op2 = csv:parseString(csvStringData1, {header: 1, skipLines: [2, 4]}); + test:assertTrue(csv1op2 is (RecA|RecC)[]); + test:assertEquals(csv1op2, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + (RecC|RecA)[]|csv:Error csv1op3 = csv:parseString(csvStringData1, {header: 1, skipLines: [2, 4]}); + test:assertEquals(csv1op3, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + (RecB|RecA)[]|csv:Error csv1op4 = csv:parseString(csvStringData1, {header: 1, skipLines: [2, 4]}); + test:assertEquals(csv1op4, [ + {a: "1", b: "string1", c: "true", d: "2.234", e: "2.234", f: "()"}, + {a: "3", b: "string3", c: "false", d: "1.23", e: "1.23", f: "()"}, + {a: "5", b: "string5", c: "true", d: "3", e: "3", f: "()"} + ]); + + (RecA|RecB)[]|csv:Error csv1op5 = csv:parseString(csvStringData1, {header: 1, skipLines: [2, 4]}); + test:assertEquals(csv1op5, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + (record {|int a;|}|record {|string b;|})[]|csv:Error csv1op6 = csv:parseString(csvStringData1, {header: 1, skipLines: [2, 4]}); + test:assertEquals(csv1op6, [ + {a: 1}, + {a: 3}, + {a: 5} + ]); + + (record {|string b;|}|record {|int a;|})[]|csv:Error csv1op7 = csv:parseString(csvStringData1, {header: 1, skipLines: [2, 4]}); + test:assertEquals(csv1op7, [ + {b: "string1"}, + {b: "string3"}, + {b: "string5"} + ]); + + (record {|string...;|}|record {|int...;|})[]|csv:Error csv1op8 = csv:parseString(csvStringData1, {header: 1}); + test:assertEquals(csv1op8, [ + {a: "1", b: "string1", c: "true", d: "2.234", e: "2.234", f: "()"}, + {a: "2", b: "string2", c: "false", d: "0", e: "0", f: "null"}, + {a: "3", b: "string3", c: "false", d: "1.23", e: "1.23", f: "()"}, + {a: "4", b: "string4", c: "true", d: "-6.51", e: "-6.51", f: "()"}, + {a: "5", b: "string5", c: "true", d: "3", e: "3", f: "()"} + ]); + + (record {|int...;|}|record {|string...;|})[]|csv:Error csv1op9 = csv:parseString(csvStringData1, {header: 1}); + test:assertEquals(csv1op9, [ + {a: 1}, + {a: 2, d: 0, e: 0}, + {a: 3}, + {a: 4}, + {a: 5, d: 3, e: 3} + ]); + + (record {|int a; string...;|}|record {|string a; int...;|})[]|csv:Error csv1op10 = csv:parseString(csvStringData1, {header: 1}); + test:assertEquals(csv1op10, [ + {a: 1, b: "string1", c: "true", d: "2.234", e: "2.234", f: "()"}, + {a: 2, b: "string2", c: "false", d: "0", e: "0", f: "null"}, + {a: 3, b: "string3", c: "false", d: "1.23", e: "1.23", f: "()"}, + {a: 4, b: "string4", c: "true", d: "-6.51", e: "-6.51", f: "()"}, + {a: 5, b: "string5", c: "true", d: "3", e: "3", f: "()"} + ]); + + (record {|string a; int...;|}|record {|int a; string...;|})[]|csv:Error csv1op11 = csv:parseString(csvStringData1, {header: 1}); + test:assertEquals(csv1op11, [ + {a: "1"}, + {a: "2", d: 0, e: 0}, + {a: "3"}, + {a: "4"}, + {a: "5", d: 3, e: 3} + ]); + + (record {|int a; int b;|}|record {|string a; string...;|})[]|csv:Error csv1op12 = csv:parseString(string ` + a,b + 1, 2 + a, b`, {header: 1}); + test:assertEquals(csv1op12, [ + {a: "1", b: "2"}, + {a: "a", b: "b"} + ]); + + ([int, int]|[string|int, string|int])[]|csv:Error csv1op13 = csv:parseString(string ` + a,b + 1, 2 + a, b`, {header: 1}); + test:assertEquals(csv1op13, [ + ["1", "2"], + ["a", "b"] + ]); +} + +record {int a; string b; boolean c; decimal d; float e; () f;}[] value = [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 2, b: "string2", c: false, d: 0, e: 0, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 4, b: "string4", c: true, d: -6.51, e: -6.51, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} +]; + +@test:Config +function testParseToStringWithUnionExpectedTypes2() returns error? { + (RecA|RecC)[]|csv:Error csv1op1 = csv:transform(value, {}); + test:assertEquals(csv1op1, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 2, b: "string2", c: false, d: 0, e: 0, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 4, b: "string4", c: true, d: -6.51, e: -6.51, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + (RecA|RecC)[]|csv:Error csv1op2 = csv:transform(value, {skipLines: [2, 4]}); + test:assertTrue(csv1op2 is (RecA|RecC)[]); + test:assertEquals(csv1op2, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + (RecC|RecA)[]|csv:Error csv1op3 = csv:transform(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op3, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + (RecB|RecA)[]|csv:Error csv1op4 = csv:transform(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op4, [ + {b: "string1"}, + {b: "string3"}, + {b: "string5"} + ]); + + (RecA|RecB)[]|csv:Error csv1op5 = csv:transform(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op5, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + (record {|int a;|}|record {|string b;|})[]|csv:Error csv1op6 = csv:transform(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op6, [ + {a: 1}, + {a: 3}, + {a: 5} + ]); + + (record {|string b;|}|record {|int a;|})[]|csv:Error csv1op7 = csv:transform(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op7, [ + {b: "string1"}, + {b: "string3"}, + {b: "string5"} + ]); + + (record {|string...;|}|record {|int...;|})[]|csv:Error csv1op8 = csv:transform(value, {}); + test:assertEquals(csv1op8, [ + {b: "string1"}, + {b: "string2"}, + {b: "string3"}, + {b: "string4"}, + {b: "string5"} + ]); + + (record {|int...;|}|record {|string...;|})[]|csv:Error csv1op9 = csv:transform(value, {}); + test:assertEquals(csv1op9, [ + {a: 1, d: 2, e: 2}, + {a: 2, d: 0, e: 0}, + {a: 3, d: 1, e: 1}, + {a: 4, d: -7, e: -7}, + {a: 5, d: 3, e: 3} + ]); + + (record {|int a; string...;|}|record {|string a; int...;|})[]|csv:Error csv1op10 = csv:transform(value, {}); + test:assertEquals(csv1op10, [ + {a: 1, b: "string1"}, + {a: 2, b: "string2"}, + {a: 3, b: "string3"}, + {a: 4, b: "string4"}, + {a: 5, b: "string5"} + ]); + + (record {|string a; int...;|}|record {|int a; string...;|})[]|csv:Error csv1op11 = csv:transform(value, {}); + test:assertEquals(csv1op11, [ + {a: 1, b: "string1"}, + {a: 2, b: "string2"}, + {a: 3, b: "string3"}, + {a: 4, b: "string4"}, + {a: 5, b: "string5"} + ]); + + (record {|string a; int...;|}|record {|string a; string...;|})[]|csv:Error csv1op12 = csv:transform(value, {}); + test:assertTrue(csv1op12 is csv:Error); + test:assertEquals((csv1op12).message(), "source value cannot converted in to the '(union_type_tests:record {| string a; int...; |}|union_type_tests:record {| string a; string...; |})[]'"); + + (record {|int a; int...;|}|record {|int|string a; int|string...;|})[]|csv:Error csv1op13 = csv:transform([{"a": 1, "b": 2}, {"a": "a", "b": "b"}], {}); + test:assertEquals(csv1op13, [ + {a: 1, b: 2}, + {a: "a", b: "b"} + ]); + + (record {|int a; int...;|}|record {|string a; string...;|})[]|csv:Error csv1op14 = csv:transform([{"a": 1, "b": 2}, {"a": "a", "b": "b"}], {}); + test:assertTrue(csv1op14 is csv:Error); + test:assertEquals((csv1op14).message(), "source value cannot converted in to the '(union_type_tests:record {| int a; int...; |}|union_type_tests:record {| string a; string...; |})[]'"); +} + +@test:Config +function testParseToStringWithUnionExpectedTypes3() returns error? { + string[][] value = [ + ["1", "string1", "true", "2.234", "2.234", "()"], + ["2", "string2", "false", "0", "0", "()"], + ["3", "string3", "false", "1.23", "1.23", "()"], + ["4", "string4", "true", "-6.51", "-6.51", "()"], + ["5", "string5", "true", "3", "3.0", "()"] + ]; + + (RecA|RecC)[]|csv:Error csv1op1 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"]}); + test:assertEquals(csv1op1, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 2, b: "string2", c: false, d: 0, e: 0, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 4, b: "string4", c: true, d: -6.51, e: -6.51, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + (RecA|RecC)[]|csv:Error csv1op2 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertTrue(csv1op2 is (RecA|RecC)[]); + test:assertEquals(csv1op2, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + (RecC|RecA)[]|csv:Error csv1op3 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op3, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + (RecB|RecA)[]|csv:Error csv1op4 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op4, [ + {a: "1", b: "string1", c: "true", d: "2.234", e: "2.234", f: "()"}, + {a: "3", b: "string3", c: "false", d: "1.23", e: "1.23", f: "()"}, + {a: "5", b: "string5", c: "true", d: "3", e: "3.0", f: "()"} + ]); + + (RecA|RecB)[]|csv:Error csv1op5 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op5, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + (record {|int a;|}|record {|string b;|})[]|csv:Error csv1op6 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op6, [ + {a: 1}, + {a: 3}, + {a: 5} + ]); + + (record {|string b;|}|record {|int a;|})[]|csv:Error csv1op7 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op7, [ + {b: "string1"}, + {b: "string3"}, + {b: "string5"} + ]); + + (record {|string...;|}|record {|int...;|})[]|csv:Error csv1op8 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"], skipLines: [4, 2]}); + test:assertEquals(csv1op8, [ + {a: "1", b: "string1", c: "true", d: "2.234", e: "2.234", f: "()"}, + {a: "3", b: "string3", c: "false", d: "1.23", e: "1.23", f: "()"}, + {a: "5", b: "string5", c: "true", d: "3", e: "3.0", f: "()"} + ]); + + (record {|int...;|}|record {|string...;|})[]|csv:Error csv1op9 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"]}); + test:assertEquals(csv1op9, [ + {a: 1}, + {a: 2, d: 0, e: 0}, + {a: 3}, + {a: 4}, + {a: 5, d: 3} + ]); + + (record {|int a; string...;|}|record {|string a; int...;|})[]|csv:Error csv1op10 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"], skipLines: [2, -1, 4]}); + test:assertEquals(csv1op10, [ + {a: 1, b: "string1", c: "true", d: "2.234", e: "2.234", f: "()"}, + {a: 3, b: "string3", c: "false", d: "1.23", e: "1.23", f: "()"}, + {a: 5, b: "string5", c: "true", d: "3", e: "3.0", f: "()"} + ]); + + (record {|string a; int...;|}|record {|int a; string...;|})[]|csv:Error csv1op11 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"]}); + test:assertEquals(csv1op11, [ + {a: "1"}, + {a: "2", d: 0, e: 0}, + {a: "3"}, + {a: "4"}, + {a: "5", d: 3} + ]); + + (record {|int a; int...;|}|record {|string a; string...;|})[]|csv:Error csv1op13 = csv:parseList([["1", "2"], ["a", "b"]], {customHeaders: ["a", "b"]}); + test:assertEquals(csv1op13, [ + {a: "1", b: "2"}, + {a: "a", b: "b"} + ]); +} + + +@test:Config +function testParseToStringWithUnionExpectedTypes4() returns error? { + record {int a; string b; boolean c; decimal d; float e; () f;}[] value = [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 2, b: "string2", c: false, d: 0, e: 0, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 4, b: "string4", c: true, d: -6.51, e: -6.51, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]; + + (TupA|TupC)[]|csv:Error csv1op1 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"]}); + test:assertEquals(csv1op1, [ + [1, "string1", true, 2.234, 2.234, ()], + [2, "string2", false, 0, 0, ()], + [3, "string3", false, 1.23, 1.23, ()], + [4, "string4", true, -6.51, -6.51, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + (TupA|TupC)[]|csv:Error csv1op2 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertTrue(csv1op2 is (TupA|TupC)[]); + test:assertEquals(csv1op2, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + (TupC|TupA)[]|csv:Error csv1op3 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op3, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + (TupB|TupA)[]|csv:Error csv1op4 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op4, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + (TupB|[boolean])[]|csv:Error csv1op4_2 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertTrue(csv1op4_2 is csv:Error); + test:assertEquals((csv1op4_2).message(), "source value cannot converted in to the '(union_type_tests:TupB|[boolean])[]'"); + + (TupA|TupB)[]|csv:Error csv1op5 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op5, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + ([int]|[string])[]|csv:Error csv1op6 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op6, [ + [1], + [3], + [5] + ]); + + ([string]|[int])[]|csv:Error csv1op7 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op7, [ + [1], + [3], + [5] + ]); + + ([string...]|[int...])[]|csv:Error csv1op8 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"]}); + test:assertTrue(csv1op8 is csv:Error); + test:assertEquals((csv1op8).message(), "source value cannot converted in to the '([string...]|[int...])[]'"); + + ([int...]|[string...])[]|csv:Error csv1op9 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"]}); + test:assertTrue(csv1op9 is csv:Error); + test:assertEquals((csv1op9).message(), "source value cannot converted in to the '([int...]|[string...])[]'"); + + ([int, string...]|[string, int...])[]|csv:Error csv1op10 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"]}); + test:assertTrue(csv1op10 is csv:Error); + test:assertEquals((csv1op10).message(), "source value cannot converted in to the '([int,string...]|[string,int...])[]'"); + + ([string, int...]|[int, string...])[]|csv:Error csv1op11 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"]}); + test:assertTrue(csv1op11 is csv:Error); + test:assertEquals((csv1op11).message(), "source value cannot converted in to the '([string,int...]|[int,string...])[]'"); + + ([string, int...]|[string, string...])[]|csv:Error csv1op12 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"]}); + test:assertTrue(csv1op12 is csv:Error); + test:assertEquals((csv1op12).message(), "source value cannot converted in to the '([string,int...]|[string,string...])[]'"); + + ([string, string...]|[int|string, int|string...])[]|csv:Error csv1op13 = csv:transform([{"a": 1, "b": 2}, {"a": "a", "b": "b"}], {headerOrder: ["a", "b"]}); + test:assertEquals(csv1op13, [ + [1, 2], + ["a", "b"] + ]); + + ([int, int...]|[string, string...])[]|csv:Error csv1op14 = csv:transform([{"a": 1, "b": 2}, {"a": "a", "b": "b"}], {headerOrder: ["a", "b"]}); + test:assertTrue(csv1op14 is csv:Error); + test:assertEquals((csv1op14).message(), "source value cannot converted in to the '([int,int...]|[string,string...])[]'"); +} + +@test:Config +function testParseToStringWithUnionExpectedTypes5() returns error? { + string[][] value = [ + ["1", "string1", "true", "2.234", "2.234", "()"], + ["2", "string2", "false", "0", "0", "()"], + ["3", "string3", "false", "1.23", "1.23", "()"], + ["4", "string4", "true", "-6.51", "-6.51", "()"], + ["5", "string5", "true", "3", "3.0", "()"] + ]; + + (TupA|TupC)[]|csv:Error csv1op1 = csv:parseList(value, {}); + test:assertEquals(csv1op1, [ + [1, "string1", true, 2.234, 2.234, ()], + [2, "string2", false, 0, 0, ()], + [3, "string3", false, 1.23, 1.23, ()], + [4, "string4", true, -6.51, -6.51, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + (TupA|TupC)[]|csv:Error csv1op2 = csv:parseList(value, {skipLines: [2, 4]}); + test:assertTrue(csv1op2 is (TupA|TupC)[]); + test:assertEquals(csv1op2, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + (TupC|TupA)[]|csv:Error csv1op3 = csv:parseList(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op3, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + (TupB|TupA)[]|csv:Error csv1op4 = csv:parseList(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op4, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + (TupB|[boolean])[]|csv:Error csv1op4_2 = csv:parseList(value, {skipLines: [2, 4]}); + test:assertTrue(csv1op4_2 is csv:Error); + test:assertEquals((csv1op4_2).message(), "source value cannot converted in to the '(union_type_tests:TupB|[boolean])[]'"); + + (TupA|TupB)[]|csv:Error csv1op5 = csv:parseList(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op5, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + ([int]|[string])[]|csv:Error csv1op6 = csv:parseList(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op6, [ + [1], + [3], + [5] + ]); + + ([string]|[int])[]|csv:Error csv1op7 = csv:parseList(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op7, [ + ["1"], + ["3"], + ["5"] + ]); + + ([string...]|[int...])[]|csv:Error csv1op8_2 = csv:parseList(value, {}); + test:assertEquals(csv1op8_2, value); + + ([int...]|[string...])[]|csv:Error csv1op9 = csv:parseList(value, {}); + test:assertEquals(csv1op9, value); + + ([int, string...]|[string, int...])[]|csv:Error csv1op10 = csv:parseList(value, {}); + test:assertEquals(csv1op10, [ + [1, "string1", "true", "2.234", "2.234", "()"], + [2, "string2", "false", "0", "0", "()"], + [3, "string3", "false", "1.23", "1.23", "()"], + [4, "string4", "true", "-6.51", "-6.51", "()"], + [5, "string5", "true", "3", "3.0", "()"] + ]); + + ([string, int...]|[int, string...])[]|csv:Error csv1op11 = csv:parseList(value, {}); + test:assertEquals(csv1op11, [ + [1, "string1", "true", "2.234", "2.234", "()"], + [2, "string2", "false", "0", "0", "()"], + [3, "string3", "false", "1.23", "1.23", "()"], + [4, "string4", "true", "-6.51", "-6.51", "()"], + [5, "string5", "true", "3", "3.0", "()"] + ]); + + ([string, int...]|[string, string...])[]|csv:Error csv1op12 = csv:parseList(value, {}); + test:assertEquals(csv1op12, value); + + ([int, int...]|[string, string...])[]|csv:Error csv1op13 = csv:parseList([["1", "2"], ["a", "b"]], {}); + test:assertEquals(csv1op13, [ + ["1", "2"], + ["a", "b"] + ]); +} + +@test:Config +function testParseToStringWithUnionExpectedTypes6() returns error? { + RecA[]|RecC[]|csv:Error csv1op1 = csv:parseString(csvStringData1, {header: 1}); + test:assertEquals(csv1op1, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 2, b: "string2", c: false, d: 0, e: 0, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 4, b: "string4", c: true, d: -6.51, e: -6.51, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + RecA[]|RecC[]|csv:Error csv1op2 = csv:parseString(csvStringData1, {header: 1, skipLines: [2, 4]}); + test:assertTrue(csv1op2 is RecA[]|RecC[]); + test:assertEquals(csv1op2, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + RecC[]|RecA[]|csv:Error csv1op3 = csv:parseString(csvStringData1, {header: 1, skipLines: [2, 4]}); + test:assertEquals(csv1op3, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + RecB[]|RecA[]|csv:Error csv1op4 = csv:parseString(csvStringData1, {header: 1, skipLines: [2, 4]}); + test:assertEquals(csv1op4, [ + {a: "1", b: "string1", c: "true", d: "2.234", e: "2.234", f: "()"}, + {a: "3", b: "string3", c: "false", d: "1.23", e: "1.23", f: "()"}, + {a: "5", b: "string5", c: "true", d: "3", e: "3", f: "()"} + ]); + + RecA[]|RecB[]|csv:Error csv1op5 = csv:parseString(csvStringData1, {header: 1, skipLines: [2, 4]}); + test:assertEquals(csv1op5, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + record {|int a;|}[]|record {|string b;|}[]|csv:Error csv1op6 = csv:parseString(csvStringData1, {header: 1, skipLines: [2, 4]}); + test:assertEquals(csv1op6, [ + {a: 1}, + {a: 3}, + {a: 5} + ]); + + record {|string b;|}[]|record {|int a;|}[]|csv:Error csv1op7 = csv:parseString(csvStringData1, {header: 1, skipLines: [2, 4]}); + test:assertEquals(csv1op7, [ + {b: "string1"}, + {b: "string3"}, + {b: "string5"} + ]); + + record {|string...;|}[]|record {|int...;|}[]|csv:Error csv1op8 = csv:parseString(csvStringData1, {header: 1}); + test:assertEquals(csv1op8, [ + {a: "1", b: "string1", c: "true", d: "2.234", e: "2.234", f: "()"}, + {a: "2", b: "string2", c: "false", d: "0", e: "0", f: "null"}, + {a: "3", b: "string3", c: "false", d: "1.23", e: "1.23", f: "()"}, + {a: "4", b: "string4", c: "true", d: "-6.51", e: "-6.51", f: "()"}, + {a: "5", b: "string5", c: "true", d: "3", e: "3", f: "()"} + ]); + + record {|int...;|}[]|record {|string...;|}[]|csv:Error csv1op9 = csv:parseString(csvStringData1, {header: 1}); + test:assertEquals(csv1op9, [ + {a: 1}, + {a: 2, d: 0, e: 0}, + {a: 3}, + {a: 4}, + {a: 5, d: 3, e: 3} + ]); + + record {|int a; string...;|}[]|record {|string a; int...;|}[]|csv:Error csv1op10 = csv:parseString(csvStringData1, {header: 1}); + test:assertEquals(csv1op10, [ + {a: 1, b: "string1", c: "true", d: "2.234", e: "2.234", f: "()"}, + {a: 2, b: "string2", c: "false", d: "0", e: "0", f: "null"}, + {a: 3, b: "string3", c: "false", d: "1.23", e: "1.23", f: "()"}, + {a: 4, b: "string4", c: "true", d: "-6.51", e: "-6.51", f: "()"}, + {a: 5, b: "string5", c: "true", d: "3", e: "3", f: "()"} + ]); + + record {|string a; int...;|}[]|record {|int a; string...;|}[]|csv:Error csv1op11 = csv:parseString(csvStringData1, {header: 1}); + test:assertEquals(csv1op11, [ + {a: "1"}, + {a: "2", d: 0, e: 0}, + {a: "3"}, + {a: "4"}, + {a: "5", d: 3, e: 3} + ]); + + record {|int a; int b;|}[]|record {|string a; string...;|}[]|csv:Error csv1op12 = csv:parseString(string ` + a,b + 1, 2 + a, b`, {header: 1}); + test:assertEquals(csv1op12, [ + {a: "1", b: "2"}, + {a: "a", b: "b"} + ]); +} + +@test:Config +function testParseToStringWithUnionExpectedTypes7() returns error? { + record {int a; string b; boolean c; decimal d; float e; () f;}[] value = [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 2, b: "string2", c: false, d: 0, e: 0, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 4, b: "string4", c: true, d: -6.51, e: -6.51, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]; + + RecA[]|RecC[]|csv:Error csv1op1 = csv:transform(value, {}); + test:assertEquals(csv1op1, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 2, b: "string2", c: false, d: 0, e: 0, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 4, b: "string4", c: true, d: -6.51, e: -6.51, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + RecA[]|RecC[]|csv:Error csv1op2 = csv:transform(value, {skipLines: [2, 4]}); + test:assertTrue(csv1op2 is RecA[]|RecC[]); + test:assertEquals(csv1op2, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + RecC[]|RecA[]|csv:Error csv1op3 = csv:transform(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op3, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + RecB[]|RecA[]|csv:Error csv1op4 = csv:transform(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op4, [ + {b: "string1"}, + {b: "string3"}, + {b: "string5"} + ]); + + RecA[]|RecB[]|csv:Error csv1op5 = csv:transform(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op5, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + record {|int a;|}[]|record {|string b;|}[]|csv:Error csv1op6 = csv:transform(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op6, [ + {a: 1}, + {a: 3}, + {a: 5} + ]); + + record {|string b;|}[]|record {|int a;|}[]|csv:Error csv1op7 = csv:transform(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op7, [ + {b: "string1"}, + {b: "string3"}, + {b: "string5"} + ]); + + record {|string...;|}[]|record {|int...;|}[]|csv:Error csv1op8 = csv:transform(value, {}); + test:assertEquals(csv1op8, [ + {b: "string1"}, + {b: "string2"}, + {b: "string3"}, + {b: "string4"}, + {b: "string5"} + ]); + + record {|int...;|}[]|record {|string...;|}[]|csv:Error csv1op9 = csv:transform(value, {}); + test:assertEquals(csv1op9, [ + {a: 1, d: 2, e: 2}, + {a: 2, d: 0, e: 0}, + {a: 3, d: 1, e: 1}, + {a: 4, d: -7, e: -7}, + {a: 5, d: 3, e: 3} + ]); + + record {|int a; string...;|}[]|record {|string a; int...;|}[]|csv:Error csv1op10 = csv:transform(value, {}); + test:assertEquals(csv1op10, [ + {a: 1, b: "string1"}, + {a: 2, b: "string2"}, + {a: 3, b: "string3"}, + {a: 4, b: "string4"}, + {a: 5, b: "string5"} + ]); + + record {|string a; int...;|}[]|record {|int a; string...;|}[]|csv:Error csv1op11 = csv:transform(value, {}); + test:assertEquals(csv1op11, [ + {a: 1, b: "string1"}, + {a: 2, b: "string2"}, + {a: 3, b: "string3"}, + {a: 4, b: "string4"}, + {a: 5, b: "string5"} + ]); + + record {|string a; int...;|}[]|record {|string a; string...;|}[]|csv:Error csv1op12 = csv:transform(value, {}); + test:assertTrue(csv1op12 is csv:Error); + test:assertEquals((csv1op12).message(), "source value cannot converted in to the '(union_type_tests:record {| string a; int...; |}[]|union_type_tests:record {| string a; string...; |}[])'"); +} + +@test:Config +function testParseToStringWithUnionExpectedTypes8() returns error? { + string[][] value = [ + ["1", "string1", "true", "2.234", "2.234", "()"], + ["2", "string2", "false", "0", "0", "()"], + ["3", "string3", "false", "1.23", "1.23", "()"], + ["4", "string4", "true", "-6.51", "-6.51", "()"], + ["5", "string5", "true", "3", "3.0", "()"] + ]; + + RecA[]|RecC[]|csv:Error csv1op1 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"]}); + test:assertEquals(csv1op1, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 2, b: "string2", c: false, d: 0, e: 0, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 4, b: "string4", c: true, d: -6.51, e: -6.51, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + RecA[]|RecC[]|csv:Error csv1op2 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertTrue(csv1op2 is RecA[]|RecC[]); + test:assertEquals(csv1op2, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + RecC[]|RecA[]|csv:Error csv1op3 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op3, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + RecB[]|RecA[]|csv:Error csv1op4 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op4, [ + {a: "1", b: "string1", c: "true", d: "2.234", e: "2.234", f: "()"}, + {a: "3", b: "string3", c: "false", d: "1.23", e: "1.23", f: "()"}, + {a: "5", b: "string5", c: "true", d: "3", e: "3.0", f: "()"} + ]); + + RecA[]|RecB[]|csv:Error csv1op5 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op5, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]); + + record {|int a;|}[]|record {|string b;|}[]|csv:Error csv1op6 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op6, [ + {a: 1}, + {a: 3}, + {a: 5} + ]); + + record {|string b;|}[]|record {|int a;|}[]|csv:Error csv1op7 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op7, [ + {b: "string1"}, + {b: "string3"}, + {b: "string5"} + ]); + + record {|string...;|}[]|record {|int...;|}[]|csv:Error csv1op8 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"], skipLines: [4, 2]}); + test:assertEquals(csv1op8, [ + {a: "1", b: "string1", c: "true", d: "2.234", e: "2.234", f: "()"}, + {a: "3", b: "string3", c: "false", d: "1.23", e: "1.23", f: "()"}, + {a: "5", b: "string5", c: "true", d: "3", e: "3.0", f: "()"} + ]); + + record {|int...;|}[]|record {|string...;|}[]|csv:Error csv1op9 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"]}); + test:assertEquals(csv1op9, [ + {a: 1}, + {a: 2, d: 0, e: 0}, + {a: 3}, + {a: 4}, + {a: 5, d: 3} + ]); + + record {|int a; string...;|}[]|record {|string a; int...;|}[]|csv:Error csv1op10 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"], skipLines: [2, -1, 4]}); + test:assertEquals(csv1op10, [ + {a: 1, b: "string1", c: "true", d: "2.234", e: "2.234", f: "()"}, + {a: 3, b: "string3", c: "false", d: "1.23", e: "1.23", f: "()"}, + {a: 5, b: "string5", c: "true", d: "3", e: "3.0", f: "()"} + ]); + + record {|string a; int...;|}[]|record {|int a; string...;|}[]|csv:Error csv1op11 = csv:parseList(value, {customHeaders: ["a", "b", "c", "d", "e", "f"]}); + test:assertEquals(csv1op11, [ + {a: "1"}, + {a: "2", d: 0, e: 0}, + {a: "3"}, + {a: "4"}, + {a: "5", d: 3} + ]); + + record {|int a; int...;|}[]|record {|string a; string...;|}[]|csv:Error csv1op13 = csv:parseList([["1", "2"], ["a", "b"]], {customHeaders: ["a", "b"]}); + test:assertEquals(csv1op13, [ + {a: "1", b: "2"}, + {a: "a", b: "b"} + ]); +} + + +@test:Config +function testParseToStringWithUnionExpectedTypes9() returns error? { + record {int a; string b; boolean c; decimal d; float e; () f;}[] value = [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 2, b: "string2", c: false, d: 0, e: 0, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 4, b: "string4", c: true, d: -6.51, e: -6.51, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: ()} + ]; + + TupA[]|TupC[]|csv:Error csv1op1 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"]}); + test:assertEquals(csv1op1, [ + [1, "string1", true, 2.234, 2.234, ()], + [2, "string2", false, 0, 0, ()], + [3, "string3", false, 1.23, 1.23, ()], + [4, "string4", true, -6.51, -6.51, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + TupA[]|TupC[]|csv:Error csv1op2 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertTrue(csv1op2 is TupA[]|TupC[]); + test:assertEquals(csv1op2, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + TupC[]|TupA[]|csv:Error csv1op3 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op3, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + TupB[]|TupA[]|csv:Error csv1op4 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op4, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + TupB[]|[boolean][]|csv:Error csv1op4_2 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertTrue(csv1op4_2 is csv:Error); + test:assertEquals((csv1op4_2).message(), "source value cannot converted in to the '(union_type_tests:TupB[]|[boolean][])'"); + + TupA[]|TupB[]|csv:Error csv1op5 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op5, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + [int][]|[string][]|csv:Error csv1op6 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op6, [ + [1], + [3], + [5] + ]); + + [string][]|[int][]|csv:Error csv1op7 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"], skipLines: [2, 4]}); + test:assertEquals(csv1op7, [ + [1], + [3], + [5] + ]); + + [string...][]|[int...][]|csv:Error csv1op8 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"]}); + test:assertTrue(csv1op8 is csv:Error); + test:assertEquals((csv1op8).message(), "source value cannot converted in to the '([string...][]|[int...][])'"); + + [int...][]|[string...][]|csv:Error csv1op9 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"]}); + test:assertTrue(csv1op9 is csv:Error); + test:assertEquals((csv1op9).message(), "source value cannot converted in to the '([int...][]|[string...][])'"); + + [int, string...][]|[string, int...][]|csv:Error csv1op10 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"]}); + test:assertTrue(csv1op10 is csv:Error); + test:assertEquals((csv1op10).message(), "source value cannot converted in to the '([int,string...][]|[string,int...][])'"); + + [string, int...][]|[int, string...][]|csv:Error csv1op11 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"]}); + test:assertTrue(csv1op11 is csv:Error); + test:assertEquals((csv1op11).message(), "source value cannot converted in to the '([string,int...][]|[int,string...][])'"); + + [string, int...][]|[string, string...][]|csv:Error csv1op12 = csv:transform(value, {headerOrder: ["a", "b", "c", "d", "e", "f"]}); + test:assertTrue(csv1op12 is csv:Error); + test:assertEquals((csv1op12).message(), "source value cannot converted in to the '([string,int...][]|[string,string...][])'"); +} + +@test:Config +function testParseToStringWithUnionExpectedTypes10() returns error? { + string[][] value = [ + ["1", "string1", "true", "2.234", "2.234", "()"], + ["2", "string2", "false", "0", "0", "()"], + ["3", "string3", "false", "1.23", "1.23", "()"], + ["4", "string4", "true", "-6.51", "-6.51", "()"], + ["5", "string5", "true", "3", "3.0", "()"] + ]; + + TupA[]|TupC[]|csv:Error csv1op1 = csv:parseList(value, {}); + test:assertEquals(csv1op1, [ + [1, "string1", true, 2.234, 2.234, ()], + [2, "string2", false, 0, 0, ()], + [3, "string3", false, 1.23, 1.23, ()], + [4, "string4", true, -6.51, -6.51, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + TupA[]|TupC[]|csv:Error csv1op2 = csv:parseList(value, {skipLines: [2, 4]}); + test:assertTrue(csv1op2 is TupA[]|TupC[]); + test:assertEquals(csv1op2, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + TupC[]|TupA[]|csv:Error csv1op3 = csv:parseList(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op3, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + TupB[]|TupA[]|csv:Error csv1op4 = csv:parseList(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op4, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + TupB[]|[boolean][]|csv:Error csv1op4_2 = csv:parseList(value, {skipLines: [2, 4]}); + test:assertTrue(csv1op4_2 is csv:Error); + test:assertEquals((csv1op4_2).message(), "source value cannot converted in to the '(union_type_tests:TupB[]|[boolean][])'"); + + TupA[]|TupB[]|csv:Error csv1op5 = csv:parseList(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op5, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + [int][]|[string][]|csv:Error csv1op6 = csv:parseList(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op6, [ + [1], + [3], + [5] + ]); + + [string][]|[int][]|csv:Error csv1op7 = csv:parseList(value, {skipLines: [2, 4]}); + test:assertEquals(csv1op7, [ + ["1"], + ["3"], + ["5"] + ]); + + [string...][]|[int...][]|csv:Error csv1op8_2 = csv:parseList(value, {}); + test:assertEquals(csv1op8_2, value); + + [int...][]|[string...][]|csv:Error csv1op9 = csv:parseList(value, {}); + test:assertEquals(csv1op9, value); + + [int, string...][]|[string, int...][]|csv:Error csv1op10 = csv:parseList(value, {}); + test:assertEquals(csv1op10, [ + [1, "string1", "true", "2.234", "2.234", "()"], + [2, "string2", "false", "0", "0", "()"], + [3, "string3", "false", "1.23", "1.23", "()"], + [4, "string4", "true", "-6.51", "-6.51", "()"], + [5, "string5", "true", "3", "3.0", "()"] + ]); + + [string, int...][]|[int, string...][]|csv:Error csv1op11 = csv:parseList(value, {}); + test:assertEquals(csv1op11, [ + [1, "string1", "true", "2.234", "2.234", "()"], + [2, "string2", "false", "0", "0", "()"], + [3, "string3", "false", "1.23", "1.23", "()"], + [4, "string4", "true", "-6.51", "-6.51", "()"], + [5, "string5", "true", "3", "3.0", "()"] + ]); + + [string, int...][]|[string, string...][]|csv:Error csv1op12 = csv:parseList(value, {}); + test:assertEquals(csv1op12, value); +} + +@test:Config +function testUnionTypeWithOrdering() returns error? { + string[][] value1 = [["1", "1.0", "true", "a"], ["2", "2.0", "false", "b"]]; + string value2 = string `a,b, c, d + 1, 1.0, true, a + 2, 2.0, false, b`; + record {}[] value3 = [{"a": 1, "b": 1.0, "c": true, "d": "a"}, {"a": 2, "b": 2.0, "c": false, "d":"b"}]; + + string[][]|anydata[][]|error csv1op1 = csv:parseList(value1); + test:assertEquals(csv1op1, [ + ["1", "1.0", "true", "a"], + ["2", "2.0", "false", "b"] + ]); + + anydata[][]|string[][]|error csv1op2 = csv:parseList(value1); + test:assertEquals(csv1op2, [ + [1, 1.0, true, "a"], + [2, 2.0, false, "b"] + ]); + + string[][2]|float[][2]|error csv1op1_2 = csv:parseList(value1); + test:assertEquals(csv1op1_2, [ + ["1", "1.0"], + ["2", "2.0"] + ]); + + float[][2]|string[][2]|error csv1op2_2 = csv:parseList(value1); + test:assertEquals(csv1op2_2, [ + [1f, 1.0], + [2f, 2.0] + ]); + + (string|anydata)[][]|error csv1op1_3 = csv:parseList(value1); + test:assertEquals(csv1op1_3, [ + ["1", "1.0", "true", "a"], + ["2", "2.0", "false", "b"] + ]); + + (anydata|string)[][]|error csv1op2_3 = csv:parseList(value1); + test:assertEquals(csv1op2_3, [ + [1, 1.0, true, "a"], + [2, 2.0, false, "b"] + ]); + + (string|float)[][2]|error csv1op1_4 = csv:parseList(value1); + test:assertEquals(csv1op1_4, [ + ["1", "1.0"], + ["2", "2.0"] + ]); + + (float|string)[][2]|error csv1op2_4 = csv:parseList(value1); + test:assertEquals(csv1op2_4, [ + [1f, 1.0], + [2f, 2.0] + ]); + + string[][]|anydata[][]|error recCsv1op1 = csv:transform(value3); + test:assertEquals(recCsv1op1, [ + [1, 1.0, true, "a"], + [2, 2.0, false, "b"] + ]); + + anydata[][]|string[][]|error recCsv1op2 = csv:transform(value3); + test:assertEquals(recCsv1op2, [ + [1, 1.0, true, "a"], + [2, 2.0, false, "b"] + ]); + + string[][2]|float[][2]|error recCsv1op1_2 = csv:transform(value3); + test:assertEquals(recCsv1op1_2, [ + [1f, 1.0], + [2f, 2.0] + ]); + + float[][2]|string[][2]|error recCsv1op2_2 = csv:transform(value3); + test:assertEquals(recCsv1op2_2, [ + [1f, 1.0], + [2f, 2.0] + ]); + + (string|anydata)[][]|error recCsv1op1_3 = csv:transform(value3); + test:assertEquals(recCsv1op1_3, [ + [1, 1.0, true, "a"], + [2, 2.0, false, "b"] + ]); + + (anydata|string)[][]|error recCsv1op2_3 = csv:transform(value3); + test:assertEquals(recCsv1op2_3, [ + [1, 1.0, true, "a"], + [2, 2.0, false, "b"] + ]); + + (string|float)[][2]|error recCsv1op1_4 = csv:transform(value3); + test:assertEquals(recCsv1op1_4, [ + [1f, 1.0], + [2f, 2.0] + ]); + + (float|string)[][2]|error recCsv1op2_4 = csv:transform(value3); + test:assertEquals(recCsv1op2_4, [ + [1f, 1.0], + [2f, 2.0] + ]); + + string[][]|anydata[][]|error parseStrCsv1op1 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op1, [ + ["1", "1.0", "true", "a"], + ["2", "2.0", "false", "b"] + ]); + + anydata[][]|string[][]|error parseStrCsv1op2 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op2, [ + [1, 1.0, true, "a"], + [2, 2.0, false, "b"] + ]); + + string[][2]|float[][2]|error parseStrCsv1op1_2 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op1_2, [ + ["1", "1.0"], + ["2", "2.0"] + ]); + + float[][2]|string[][2]|error parseStrCsv1op2_2 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op2_2, [ + [1f, 1.0], + [2f, 2.0] + ]); + + (string|anydata)[][]|error parseStrCsv1op1_3 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op1_3, [ + ["1", "1.0", "true", "a"], + ["2", "2.0", "false", "b"] + ]); + + (anydata|string)[][]|error parseStrCsv1op2_3 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op2_3, [ + [1, 1.0, true, "a"], + [2, 2.0, false, "b"] + ]); + + (string|float)[][2]|error parseStrCsv1op1_4 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op1_4, [ + ["1", "1.0"], + ["2", "2.0"] + ]); + + (float|string)[][2]|error parseStrCsv1op2_4 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op2_4, [ + [1f, 1.0], + [2f, 2.0] + ]); +} + +@test:Config +function testUnionTypeWithNull() returns error? { + string[][] value1 = [["1", "()", "true", "a"], ["2", "()", "false", "b"]]; + string value2 = string `a,b, c, d + 1, (), true, a + 2, (), false, b`; + record {}[] value3 = [{"a": 1, "b": (), "c": true, "d": "a"}, {"a": 2, "b": (), "c": false, "d":"b"}]; + string value4 = string `a,b, c, d + 1, , true, a + 2, , false, b`; + string value5 = string `a,b, c, d + 1, null, true, a + 2, null, false, b`; + string value6 = string `a,b, c, d + 1, N/A, true, a + 2, N/A, false, b`; + string[][] value7 = [["1", "null", "true", "a"], ["2", "null", "false", "b"]]; + + string?[][2]|float[][2]|error csv1op1_2 = csv:parseList(value1); + test:assertEquals(csv1op1_2, [ + ["1", ()], + ["2", ()] + ]); + + csv1op1_2 = csv:parseList(value7); + test:assertEquals(csv1op1_2, [ + ["1", ()], + ["2", ()] + ]); + + float[][2]|string?[][2]|error csv1op2_2 = csv:parseList(value1); + test:assertEquals(csv1op2_2, [ + ["1", ()], + ["2", ()] + ]); + + csv1op2_2 = csv:parseList(value7); + test:assertEquals(csv1op2_2, [ + ["1", ()], + ["2", ()] + ]); + + (string|float?)[][2]|error csv1op1_4 = csv:parseList(value1); + test:assertEquals(csv1op1_4, [ + ["1", ()], + ["2", ()] + ]); + + csv1op1_4 = csv:parseList(value7); + test:assertEquals(csv1op1_4, [ + ["1", ()], + ["2", ()] + ]); + + (float|string?)[][2]|error csv1op2_4 = csv:parseList(value1); + test:assertEquals(csv1op2_4, [ + [1f, ()], + [2f, ()] + ]); + + csv1op2_4 = csv:parseList(value7); + test:assertEquals(csv1op2_4, [ + [1f, ()], + [2f, ()] + ]); + + int?[][2]|float[][2]|error recCsv1op1_2 = csv:transform(value3); + test:assertEquals(recCsv1op1_2, [ + [1, ()], + [2, ()] + ]); + + float?[][2]|int?[][2]|error recCsv1op2_2 = csv:transform(value3); + test:assertEquals(recCsv1op2_2, [ + [1f, ()], + [2f, ()] + ]); + + (float|string?)[][2]|error recCsv1op2_4 = csv:transform(value3); + test:assertEquals(recCsv1op2_4, [ + [1f, ()], + [2f, ()] + ]); + + string?[][2]|float[][2]|error parseStrCsv1op1_2 = csv:parseString(value2); + test:assertEquals(parseStrCsv1op1_2, [ + ["1", ()], + ["2", ()] + ]); + + float[][2]|string?[][2]|error parseStrCsv1op2_2 = csv:parseString(value4, {nilValue: ""}); + test:assertEquals(parseStrCsv1op2_2, [ + ["1", ()], + ["2", ()] + ]); + + (string|float?)[][2]|error parseStrCsv1op1_4 = csv:parseString(value5, {nilValue: "null"}); + test:assertEquals(parseStrCsv1op1_4, [ + ["1", ()], + ["2", ()] + ]); + + (float|string)?[][2]|error parseStrCsv1op2_4 = csv:parseString(value6, {nilValue: "N/A"}); + test:assertEquals(parseStrCsv1op2_4, [ + [1f, ()], + [2f, ()] + ]); +} diff --git a/ballerina-tests/user-config-tests/Ballerina.toml b/ballerina-tests/user-config-tests/Ballerina.toml new file mode 100644 index 0000000..7810c0a --- /dev/null +++ b/ballerina-tests/user-config-tests/Ballerina.toml @@ -0,0 +1,16 @@ +[package] +org = "ballerina" +name = "user_config_tests" +version = "0.1.0" + +[[dependency]] +org = "ballerina" +name = "csv_commons" +repository = "local" +version = "0.1.0" + +[platform.java17] +graalvmCompatible = true + +[build-options] +graalvmBuildOptions = "-H:+IncludeAllLocales" diff --git a/ballerina-tests/user-config-tests/Dependencies.toml b/ballerina-tests/user-config-tests/Dependencies.toml new file mode 100644 index 0000000..beb3621 --- /dev/null +++ b/ballerina-tests/user-config-tests/Dependencies.toml @@ -0,0 +1,98 @@ +# AUTO-GENERATED FILE. DO NOT MODIFY. + +# This file is auto-generated by Ballerina for managing dependency versions. +# It should not be modified by hand. + +[ballerina] +dependencies-toml-version = "2" +distribution-version = "2201.9.0" + +[[package]] +org = "ballerina" +name = "csv_commons" +version = "0.1.0" +scope = "testOnly" +modules = [ + {org = "ballerina", packageName = "csv_commons", moduleName = "csv_commons"} +] + +[[package]] +org = "ballerina" +name = "data.csv" +version = "0.1.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] +modules = [ + {org = "ballerina", packageName = "data.csv", moduleName = "data.csv"} +] + +[[package]] +org = "ballerina" +name = "jballerina.java" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "lang.__internal" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.object"} +] + +[[package]] +org = "ballerina" +name = "lang.array" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.__internal"} +] + +[[package]] +org = "ballerina" +name = "lang.error" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "lang.object" +version = "0.0.0" +scope = "testOnly" + +[[package]] +org = "ballerina" +name = "test" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.array"}, + {org = "ballerina", name = "lang.error"} +] +modules = [ + {org = "ballerina", packageName = "test", moduleName = "test"} +] + +[[package]] +org = "ballerina" +name = "user_config_tests" +version = "0.1.0" +dependencies = [ + {org = "ballerina", name = "csv_commons"}, + {org = "ballerina", name = "data.csv"}, + {org = "ballerina", name = "test"} +] +modules = [ + {org = "ballerina", packageName = "user_config_tests", moduleName = "user_config_tests"} +] + diff --git a/ballerina-tests/user-config-tests/tests/test_data_values.bal b/ballerina-tests/user-config-tests/tests/test_data_values.bal new file mode 100644 index 0000000..666d717 --- /dev/null +++ b/ballerina-tests/user-config-tests/tests/test_data_values.bal @@ -0,0 +1,148 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +final boolean b1 = true; +final false b2 = false; +final boolean? b3 = (); +final boolean|int b4 = false; + +final () n1 = (); +final int? n2 = (); +final () n3 = null; + +final int i1 = 1; + +final map bm1 = {b1, b2}; +final map bm2 = {b1, b2, b3, n1, n3}; +final map bm3 = {b1, b2, b3, b4, i1}; +final map<()> bm4 = {n1, n3}; +final map bm5 = {b1, b2, b3, b4:true}; + +final string csvStringData1 = string ` + a, b, c, d, e, f + 1, string1, true, 2.234, 2.234, () + 2, string2, false, 0, 0, null + 3, string3, false, 1.23, 1.23, () + 4, string4, true, -6.51, -6.51, () + 5, string5, true, 3, 3, ()`; + +final string csvStringData2 = string ` + hello, hello, (), 12, true, 12.34 + // comment + + a, b, c, d, e, f + + + 1, string1, true, 2.234, 2.234, () + 2, string2, false, 0, 0, null + 3, string3, false, 1.23, 1.23, () + 4, string4, true, -6.51, -6.51, () + 5, string5, true, 3, 3, ()`; + +final string csvStringData3 = string ` + a, b, c, d, e, f + + + + 1, string1, true, 2.234, 2.234, () + 2, string2, false, 0, 0, null + 3, string3, false, 1.23, 1.23, () + + 4, string4, true, -6.51, -6.51, () + 5, string5, true, 3, 3, ()`; + +final string csvStringData4 = string ` + 1, string1, true, 2.234, 2.234, () + 2, string2, false, 0, 0, null + 3, string3, false, 1.23, 1.23, () + 4, string4, true, -6.51, -6.51, () + 5, string5, true, 3, 3, ()`; + +final string csvStringData5 = string ` + + + + 1, string1, true, 2.234, 2.234, () + 2, string2, false, 0, 0, null + + 3, string3, false, 1.23, 1.23, () + 4, string4, true, -6.51, -6.51, () + + 5, string5, true, 3, 3, ()`; + +final string csvStringData6 = string ` + + 1, string1, true, 2.234, 2.234, () + 2, string2, false, 0, 0, null + + 3, string3, false, 1.23, 1.23, () + 4, string4, true, -6.51, -6.51, () + + 5, string5, true, 3, 3, ()`; + +final string csvStringData7 = string ` + a@ b@ c@ d@ e@ f + 1@ string@ true@ 2.234@ -3.21@ () + 2@ s,tring@ true@ 2.234@ -3.21@ null + 3@ stri,ng@ true@ 2.234@ -3.21@ () + 4@ string@ true@ 2.234@ -3.21@ () + 5@ string@ true@ 2.234@ -3.21@ ()`; +final string csvStringData8 = string ` + a@ b@ c@ d@ e@ f + + + + 1@ stri,ng@ true@ 2.234@ -3.21@ () + 2@ string@ true@ 2.234@ ()@-3.21 + 3@ string@ true@ 2.234@ -3.21@ null + + 4@ s,tring@ true@ 2.234@ -3.21@ () + 5@ string@ true@ 2.234@ -3.21@ ()`; + +final string csvStringData9 = string ` + + 1@ string@ true@ 2.234@ -3.21@ () + 2@ string@ true@ 2.234@ -3.21@ null + + 3@ string@ true@ 2.234@ -3.21@ () + 4@ string@ true@ 2.234@ ()@-3.21 + + 5@ string@ true@ 2.234@ -3.21@ null`; + +final string csvStringData10 = string ` + hello, hello, (), 12, true, 12.34 + // comment + + a, b, c, d, e, f + + + 1, string1, true, 2.234, 2.234, () + 2, string2, false, 0, 0, null + 3, string3, false, 1.23, 1.23, () + 4, string4, true, -6.51, -6.52, () + 5, string5, true, 3, 31, ()`; + +final string csvStringData11 = string ` + a, b, c, d, e, f + + + + 1, string1, true, 2.234, 2.234, () + 2, string2, false, 0, 0, null + 3, string3, false, 1.23, 1.23, () + + 4, string4, true, -6.51, -6.52, () + 5, string5, true, 3, 3, ()`; diff --git a/ballerina-tests/user-config-tests/tests/types.bal b/ballerina-tests/user-config-tests/tests/types.bal new file mode 100644 index 0000000..f3b94e7 --- /dev/null +++ b/ballerina-tests/user-config-tests/tests/types.bal @@ -0,0 +1,89 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/data.csv; + +type RecordWithCustomAnnotation record { + @csv:Name { + value: "c" + } + int a; + int b; +}; + +type RecordWithCustomAnnotation2 record { + @csv:Name { + value: "c" + } + int a?; + @csv:Name { + value: "d" + } + int? b; +}; + +type RecordWithCustomAnnotation3 record {| + @csv:Name { + value: "c" + } + int a?; + @csv:Name { + value: "d" + } + int? b; +|}; + +type RecordWithCustomAnnotation4 record {| + @csv:Name { + value: "c" + } + int a; + @csv:Name { + value: "d" + } + int b; + boolean...; +|}; + +type RecordWithCustomAnnotation5 record { + @csv:Name { + value: "c" + } + int a; + @csv:Name { + value: "d" + } + int b; + @csv:Name { + value: "e" + } + int c; +}; + +type RecordWithCustomAnnotation6 record { + @csv:Name { + value: "c" + } + int a; + @csv:Name { + value: "d" + } + int b; + @csv:Name { + value: "a" + } + int c; +}; diff --git a/ballerina-tests/user-config-tests/tests/user_config_projection_tests.bal b/ballerina-tests/user-config-tests/tests/user_config_projection_tests.bal new file mode 100644 index 0000000..e6dd846 --- /dev/null +++ b/ballerina-tests/user-config-tests/tests/user_config_projection_tests.bal @@ -0,0 +1,675 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/csv_commons as common; +import ballerina/data.csv; +import ballerina/test; + +@test:Config +function testCustomNameAnnotation() returns error? { + RecordWithCustomAnnotation[]|csv:Error cn1 = csv:parseString(string `b,c + 1,3`, {}); + test:assertEquals(cn1, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation[]|csv:Error cn2 = csv:parseString(string `c,b + 3,1`, {}); + test:assertEquals(cn2, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation[]|csv:Error cn3 = csv:parseString(string `f,c,b,e + 3,3,1,"cde" + 3,3,1,"cde"`, {}); + test:assertEquals(cn3, [{b: 1, a: 3, f: 3, e: "cde"}, {b: 1, a: 3, f: 3, e: "cde"}]); + + RecordWithCustomAnnotation2[]|csv:Error cn4 = csv:parseString(string `d,c + 1,3`, {}); + test:assertEquals(cn4, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation2[]|csv:Error cn5 = csv:parseString(string `c,d + 3,1`, {}); + test:assertEquals(cn5, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation2[]|csv:Error cn6 = csv:parseString(string `c,f,d,e + 3,3,1,"cde" + 3,3,1,"cde"`, {}); + test:assertEquals(cn6, [{b: 1, a: 3, f: 3, e: "cde"}, {b: 1, a: 3, f: 3, e: "cde"}]); + + RecordWithCustomAnnotation2[]|csv:Error cn7 = csv:parseString(string `a,b + 3,1`, {}); + test:assertTrue(cn7 is csv:Error); + test:assertEquals((cn7).message(), "duplicate field found in record fields: 'a'"); + + RecordWithCustomAnnotation2[]|csv:Error cn8 = csv:parseString(string ` c,d,a,b + 3,1,4,5`, {}); + test:assertTrue(cn8 is csv:Error); + test:assertEquals((cn8).message(), "duplicate field found in record fields: 'a'"); + + RecordWithCustomAnnotation3[]|csv:Error cn9 = csv:parseString(string `d,c + 1,3`, {}); + test:assertEquals(cn9, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation3[]|csv:Error cn10 = csv:parseString(string `c,d + 3,1`, {}); + test:assertEquals(cn10, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation3[]|csv:Error cn11 = csv:parseString(string `c,f,d,e + 3,3,1,"cde" + 3,3,1,"cde"`, {}); + test:assertEquals(cn11, [{b: 1, a: 3}, {b: 1, a: 3}]); + + RecordWithCustomAnnotation3[]|csv:Error cn12 = csv:parseString(string `a,b + 3,1`, {}); + test:assertTrue(cn12 is csv:Error); + test:assertEquals((cn12).message(), "duplicate field found in record fields: 'a'"); + + RecordWithCustomAnnotation3[]|csv:Error cn13 = csv:parseString(string ` c,d,a,b + 3,1,4,5`, {}); + test:assertTrue(cn13 is csv:Error); + test:assertEquals((cn13).message(), "duplicate field found in record fields: 'a'"); + + RecordWithCustomAnnotation4[]|csv:Error cn14 = csv:parseString(string `d,c,z + 1,3,true`, {}); + test:assertEquals(cn14, [{b: 1, a: 3, z: true}]); + + RecordWithCustomAnnotation4[]|csv:Error cn15 = csv:parseString(string `c,d + 3,1`, {}); + test:assertEquals(cn15, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation4[]|csv:Error cn16 = csv:parseString(string `c,f,d,e + 3,3,1,"cde" + 3,3,1,"cde"`, {}); + test:assertEquals(cn16, [{b: 1, a: 3}, {b: 1, a: 3}]); + + RecordWithCustomAnnotation4[]|csv:Error cn17 = csv:parseString(string `a,b + 3,1`, {}); + test:assertTrue(cn17 is csv:Error); + test:assertEquals((cn17).message(), "duplicate field found in record fields: 'a'"); + + RecordWithCustomAnnotation4[]|csv:Error cn18 = csv:parseString(string ` c,d,a,b + 3,1,4,5`, {}); + test:assertTrue(cn18 is csv:Error); + test:assertEquals((cn18).message(), "duplicate field found in record fields: 'a'"); + + RecordWithCustomAnnotation5[]|csv:Error cn19 = csv:parseString(string ` c,d,a,b + 3,1,4,5`, {}); + test:assertTrue(cn19 is csv:Error); + test:assertEquals((cn19).message(), "duplicate field found in record fields: 'a'"); + + RecordWithCustomAnnotation5[]|csv:Error cn20 = csv:parseString(string ` c,d,e + 3,1,4 + 3,1,4`, {}); + test:assertEquals(cn20, [{a: 3, b: 1, c: 4}, {a: 3, b: 1, c: 4}]); + + RecordWithCustomAnnotation6[]|csv:Error cn21 = csv:parseString(string ` c,d,a + 3,1,4 + 3,1,4`, {}); + test:assertEquals(cn21, [{a: 3, b: 1, c: 4}, {a: 3, b: 1, c: 4}]); + + RecordWithCustomAnnotation[]|csv:Error cnrr1 = csv:transform([{"b": 1, "c": 3}], {}); + test:assertEquals(cnrr1, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation[]|csv:Error cnrr2 = csv:transform([{"c": 3, "b": 1}], {}); + test:assertEquals(cnrr2, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation[]|csv:Error cnrr3 = csv:transform( + [{"f": 3, "c": 3, "b": 1, "e": "cde"}, {"f": 3, "c": 3, "b": 1, "e": "cde"}], {}); + test:assertEquals(cnrr3, [{b: 1, a: 3, f: 3, e: "cde"}, {b: 1, a: 3, f: 3, e: "cde"}]); + + RecordWithCustomAnnotation2[]|csv:Error cnrr4 = csv:transform([{"d": 1, "c": 3}], {}); + test:assertEquals(cnrr4, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation2[]|csv:Error cnrr5 = csv:transform([{"c": 3, "d": 1}], {}); + test:assertEquals(cnrr5, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation2[]|csv:Error cnrr6 = csv:transform( + [{"c": 3, "f": 3, "d": 1, "e": "cde"}, {"c": 3, "f": 3, "d": 1, "e": "cde"}], {}); + test:assertEquals(cnrr6, [{b: 1, a: 3, f: 3, e: "cde"}, {b: 1, a: 3, f: 3, e: "cde"}]); + + RecordWithCustomAnnotation2[]|csv:Error cnrr7 = csv:transform([{"a":3, "b": 1}], {}); + test:assertTrue(cnrr7 is csv:Error); + test:assertEquals((cnrr7).message(), "duplicate field found in record fields: 'a'"); + + RecordWithCustomAnnotation2[]|csv:Error cnrr8 = csv:transform([{"c": 3, "d": 1, "a": 4, "b": 5}], {}); + test:assertTrue(cnrr8 is csv:Error); + test:assertEquals((cnrr8).message(), "duplicate field found in record fields: 'a'"); + + RecordWithCustomAnnotation3[]|csv:Error cnrr9 = csv:transform([{"d": 1, "c": 3}], {}); + test:assertEquals(cnrr9, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation3[]|csv:Error cnrr10 = csv:transform([{"c": 3, "d": 1}], {}); + test:assertEquals(cnrr10, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation3[]|csv:Error cnrr11 = csv:transform( + [{"c": 3, "f": 3, "d": 1, "e": "cde"}, {"c": 3, "f": 3, "d": 1, "e": "cde"}], {}); + test:assertEquals(cnrr11, [{b: 1, a: 3}, {b: 1, a: 3}]); + + RecordWithCustomAnnotation3[]|csv:Error cnrr12 = csv:transform([{"a": 3, "b": 1}], {}); + test:assertTrue(cnrr12 is csv:Error); + test:assertEquals((cnrr12).message(), "duplicate field found in record fields: 'a'"); + + RecordWithCustomAnnotation3[]|csv:Error cnrr13 = csv:transform([{"c": 3, "d": 1, "a": 4, "b": 5}], {}); + test:assertTrue(cnrr13 is csv:Error); + test:assertEquals((cnrr13).message(), "duplicate field found in record fields: 'a'"); + + RecordWithCustomAnnotation4[]|csv:Error cnrr14 = csv:transform([{"d": 1, "c": 3, "z": true}], {}); + test:assertEquals(cnrr14, [{b: 1, a: 3, z: true}]); + + RecordWithCustomAnnotation4[]|csv:Error cnrr15 = csv:transform([{"c": 3, "d": 1}], {}); + test:assertEquals(cnrr15, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation4[]|csv:Error cnrr16 = csv:transform( + [{"c": 3, "f": 3, "d": 1, "e": "cde"}, {"c": 3, "f": 3, "d": 1, "e": "cde"}], {}); + test:assertEquals(cnrr16, [{b: 1, a: 3}, {b: 1, a: 3}]); + + RecordWithCustomAnnotation5[]|csv:Error cnrr20 = csv:transform( + [{"c": 3, "d": 1, "e": 4}, {"c": 3, "d": 1, "e": 4}], {}); + test:assertEquals(cnrr20, [{a: 3, b: 1, c: 4}, {a: 3, b: 1, c: 4}]); + + RecordWithCustomAnnotation6[]|csv:Error cnrr21 = csv:transform( + [{"c": 3, "d": 1, "a": 4}, {"c": 3, "d": 1, "a": 4}], {}); + test:assertEquals(cnrr21, [{a: 3, b: 1, c: 4}, {a: 3, b: 1, c: 4}]); +} + +@test:Config +function testCustomNameAnnotation2() returns error? { + RecordWithCustomAnnotation[]|csv:Error cntr1 = csv:parseList([["1", "3"]], {customHeaders: ["b", "c"]}); + test:assertEquals(cntr1, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation[]|csv:Error cntr2 = csv:parseList([["3", "1"]], {customHeaders: ["c", "b"]}); + test:assertEquals(cntr2, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation[]|csv:Error cntr3 = csv:parseList( + [["3", "3", "1", "cde"], ["3", "3", "1", "cde"]], {customHeaders: ["f", "c", "b", "e"]}); + test:assertEquals(cntr3, [{b: 1, a: 3, f: 3, e: "cde"}, {b: 1, a: 3, f: 3, e: "cde"}]); + + RecordWithCustomAnnotation2[]|csv:Error cntr4 = csv:parseList([["1", "3"]], {customHeaders: ["d", "c"]}); + test:assertEquals(cntr4, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation2[]|csv:Error cntr5 = csv:parseList([["3","1"]], {customHeaders: ["c", "d"]}); + test:assertEquals(cntr5, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation2[]|csv:Error cntr6 = csv:parseList( + [["3", "3", "1", "cde"], ["3", "3", "1", "cde"]], {customHeaders: ["c", "f", "d", "e"]}); + test:assertEquals(cntr6, [{b: 1, a: 3, f: 3, e: "cde"}, {b: 1, a: 3, f: 3, e: "cde"}]); + + RecordWithCustomAnnotation2[]|csv:Error cntr7 = csv:parseList([["3", "1"]], {customHeaders: ["a", "b"]}); + test:assertTrue(cntr7 is csv:Error); + test:assertEquals((cntr7).message(), "duplicate field found in record fields: 'a'"); + + RecordWithCustomAnnotation2[]|csv:Error cntr8 = csv:parseList([["3", "1", "4", "5"]], {customHeaders: ["c", "d", "a", "b"]}); + test:assertTrue(cntr8 is csv:Error); + test:assertEquals((cntr8).message(), "duplicate field found in record fields: 'a'"); + + RecordWithCustomAnnotation3[]|csv:Error cntr9 = csv:parseList([["1", "3"]], {customHeaders: ["d", "c"]}); + test:assertEquals(cntr9, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation3[]|csv:Error cntr10 = csv:parseList([["3", "1"]], {customHeaders: ["c", "d"]}); + test:assertEquals(cntr10, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation3[]|csv:Error cntr11 = csv:parseList( + [["3", "3", "1", "cde"], ["3", "3", "1", "cde"]], {customHeaders: ["c", "f", "d", "e"]}); + test:assertEquals(cntr11, [{b: 1, a: 3}, {b: 1, a: 3}]); + + RecordWithCustomAnnotation3[]|csv:Error cntr12 = csv:parseList([["3", "1"]], {customHeaders: ["a", "b"]}); + test:assertTrue(cntr12 is csv:Error); + test:assertEquals((cntr12).message(), common:generateErrorMessageForInvalidHeaders(string `["3","1"]`, "user_config_tests:RecordWithCustomAnnotation3")); + + RecordWithCustomAnnotation3[]|csv:Error cntr13 = csv:parseList([["3", "1", "4", "5"]], {customHeaders: ["c", "d", "a", "b"]}); + test:assertTrue(cntr13 is csv:Error); + test:assertEquals((cntr13).message(), "duplicate field found in record fields: 'a'"); + + RecordWithCustomAnnotation4[]|csv:Error cntr14 = csv:parseList([["1", "3", "true"]], {customHeaders: ["d", "c", "z"]}); + test:assertEquals(cntr14, [{b: 1, a: 3, z: true}]); + + RecordWithCustomAnnotation4[]|csv:Error cntr15 = csv:parseList([["3", "1"]], {customHeaders: ["c", "d"]}); + test:assertEquals(cntr15, [{b: 1, a: 3}]); + + RecordWithCustomAnnotation4[]|csv:Error cntr16 = csv:parseList( + [["3", "3", "1", "cde"], ["3", "3", "1", "cde"]], {customHeaders: ["c", "f", "d", "e"]}); + test:assertEquals(cntr16, [{b: 1, a: 3}, {b: 1, a: 3}]); + + RecordWithCustomAnnotation4[]|csv:Error cntr17 = csv:parseList([["3", "1"]], {customHeaders: ["a", "b"]}); + test:assertTrue(cntr17 is csv:Error); + test:assertEquals((cntr17).message(), "duplicate field found in record fields: 'a'"); + + RecordWithCustomAnnotation4[]|csv:Error cntr18 = csv:parseList([["3", "1", "4", "5"]], {customHeaders: ["c", "d", "a", "b"]}); + test:assertTrue(cntr18 is csv:Error); + test:assertEquals((cntr18).message(), "duplicate field found in record fields: 'a'"); + + RecordWithCustomAnnotation5[]|csv:Error cntr19 = csv:parseList([["3", "1", "4", "5"]], {customHeaders: ["c", "d", "a", "b"]}); + test:assertTrue(cntr19 is csv:Error); + test:assertEquals((cntr19).message(), "duplicate field found in record fields: 'a'"); + + RecordWithCustomAnnotation5[]|csv:Error cntr20 = csv:parseList( + [["3", "1", "4"], ["3", "1", "4"]], {customHeaders: ["c", "d", "e"]}); + test:assertEquals(cntr20, [{a: 3, b: 1, c: 4}, {a: 3, b: 1, c: 4}]); + + RecordWithCustomAnnotation6[]|csv:Error cntr21 = csv:parseList( + [["3", "1", "4"], ["3", "1", "4"]], {customHeaders: ["c", "d", "a"]}); + test:assertEquals(cntr21, [{a: 3, b: 1, c: 4}, {a: 3, b: 1, c: 4}]); +} + +@test:Config +function testAbsentAsNilableConfig() returns error? { + record {|int a; int? g; int? h;|}[]|csv:Error cn = csv:parseString(csvStringData1, { + allowDataProjection: {absentAsNilableType: true}, + header: 1 + }); + test:assertEquals(cn, [{a: 1, g: (), h: ()}, {a: 2, g: (), h: ()}, {a: 3, g: (), h: ()}, {a: 4, g: (), h: ()}, {a: 5, g: (), h: ()}]); + + record {|int a; int? g?;|}[]|csv:Error cn2 = csv:parseString(csvStringData1, { + allowDataProjection: {absentAsNilableType: true}, + header: 1, + skipLines: "2-10" + }); + test:assertEquals(cn2, [{a: 1}]); + + record {|int a; int g?;|}[]|csv:Error cn3 = csv:parseString(csvStringData1, { + allowDataProjection: {absentAsNilableType: true}, + header: 1, + skipLines: "2-10" + }); + test:assertEquals(cn3, [{a: 1}]); + + record {|int a; int g;|}[]|csv:Error cn4 = csv:parseString(csvStringData1, { + allowDataProjection: {absentAsNilableType: true}, + header: 1, + skipLines: "2-10" + }); + test:assertTrue(cn4 is csv:Error); + test:assertEquals((cn4).message(), common:generateErrorMessageForMissingRequiredField("g")); + + int?[][]|csv:Error cn5 = csv:parseString(string ` + a,b,c + 1, 1,1`, { + allowDataProjection: {absentAsNilableType: true}, + header: 1 + }); + test:assertEquals(cn5, [[1, 1, 1]]); + + map[]|csv:Error cn6 = csv:parseString(string ` + a,b,c + 1, 1,1`, { + allowDataProjection: {absentAsNilableType: true}, + header: 1 + }); + test:assertEquals(cn6, [{a: 1, b: 1, c: 1}]); + + [int, string?, boolean?, decimal?, float?, (), string?][]|csv:Error cn7 = csv:parseString(csvStringData1, { + allowDataProjection: {absentAsNilableType: true}, + header: 1, + skipLines: "2-10" + }); + test:assertEquals(cn7, [[1, "string1", true, 2.234, 2.234, null, null]]); + + [int, string?, boolean?, decimal?, float?, (), string?][]|csv:Error cn8 = csv:parseString(csvStringData1, { + allowDataProjection: {absentAsNilableType: true}, + header: 1, + skipLines: "2-10" + }); + test:assertEquals(cn8, [[1, "string1", true, 2.234, 2.234, null, null]]); + + [int, string?, boolean?, decimal?, float?, (), string?][]|csv:Error cn9 = csv:parseString(csvStringData1, { + allowDataProjection: {absentAsNilableType: true}, + header: 1, + skipLines: "2-10" + }); + test:assertEquals(cn9, [[1, "string1", true, 2.234, 2.234, null, null]]); + + [int, string?, boolean?, decimal?, float?, (), string?][]|csv:Error cn10 = csv:parseString(csvStringData1, { + allowDataProjection: {absentAsNilableType: true}, + header: 1, + skipLines: "2-10" + }); + test:assertEquals(cn10, [[1, "string1", true, 2.234, 2.234, null, null]]); +} + +@test:Config +function testAbsentAsNilableConfig2() returns error? { + record {|int a; int? g; int? h;|}[]|csv:Error cn = csv:transform([{"a": 1}, {"a": 2}, {"a": 3}], { + allowDataProjection: {absentAsNilableType: true}, skipLines: [3] + }); + test:assertEquals(cn, [{a: 1, g: (), h: ()}, {a: 2, g: (), h: ()}]); + + record {|int a; int? g?;|}[]|csv:Error cn2 = csv:transform([{"a": 1}, {"a": 2}, {"a": 3}], { + allowDataProjection: {absentAsNilableType: true}, skipLines: [3] + }); + test:assertEquals(cn2, [{a: 1}, {a: 2}]); + + record {|int a; int g?;|}[]|csv:Error cn3 = csv:transform([{"a": 1}, {"a": 2}, {"a": 3}], { + allowDataProjection: {absentAsNilableType: true}, skipLines: [3] + }); + test:assertEquals(cn3, [{a: 1}, {a: 2}]); + + record {|int a; int g;|}[]|csv:Error cn4 = csv:transform([{"a": 1}, {"a": 2}, {"a": 3}], { + allowDataProjection: {absentAsNilableType: true}, skipLines: [3] + }); + test:assertTrue(cn4 is csv:Error); + test:assertEquals((cn4).message(), common:generateErrorMessageForMissingRequiredField("g")); + + record {|string a; int? g; int? h;|}[]|csv:Error cn5 = csv:parseList([["a"], ["a"], ["a"]], {customHeaders: ["a"], + allowDataProjection: {absentAsNilableType: true}, skipLines: [3] + }); + test:assertEquals(cn5, [{a: "a", g: (), h: ()}, {a: "a", g: (), h: ()}]); + + record {|string a; int? g?;|}[]|csv:Error cn6 = csv:parseList([["a"], ["a"], ["a"]], {customHeaders: ["a"], + allowDataProjection: {absentAsNilableType: true}, skipLines: [3] + }); + test:assertEquals(cn6, [{a: "a"}, {a: "a"}]); + + record {|string a; int g?;|}[]|csv:Error cn7 = csv:parseList([["a"], ["a"], ["b"]], {customHeaders: ["a"], + allowDataProjection: {absentAsNilableType: true}, skipLines: [2] + }); + test:assertEquals(cn7, [{a: "a"}, {a: "b"}]); + + record {|string a; int g;|}[]|csv:Error cn8 = csv:parseList([["a"], ["a"], ["a"]], {customHeaders: ["a"], + allowDataProjection: {absentAsNilableType: true}, skipLines: [3] + }); + test:assertTrue(cn8 is csv:Error); + test:assertEquals((cn8).message(), common:generateErrorMessageForMissingRequiredField("g")); +} + +@test:Config +function testNilAsOptionalConfig() returns error? { + record {|int a; int f?;|}[]|csv:Error cn = csv:parseString(csvStringData1, { + allowDataProjection: {nilAsOptionalField: true}, + header: 1 + }); + test:assertEquals(cn, [{a: 1}, {a: 2}, {a: 3}, {a: 4}, {a: 5}]); + + record {|int a; int? f?;|}[]|csv:Error cn2 = csv:parseString(csvStringData1, { + allowDataProjection: {nilAsOptionalField: true}, + header: 1, + skipLines: "2-10" + }); + test:assertEquals(cn2, [{a: 1, f: ()}]); + + record {|int a; int f?;|}[]|csv:Error cn3 = csv:parseString(csvStringData1, { + allowDataProjection: {nilAsOptionalField: true}, + header: 1, + skipLines: "2-10" + }); + test:assertEquals(cn3, [{a: 1}]); + + record {|int a; int f;|}[]|csv:Error cn4 = csv:parseString(csvStringData1, { + allowDataProjection: {nilAsOptionalField: true}, + header: 1, + skipLines: "2-10" + }); + test:assertTrue(cn4 is csv:Error); + test:assertEquals((cn4).message(), common:generateErrorMessageForInvalidCast("()", "int")); + + int?[][]|csv:Error cn5 = csv:parseString(string ` + a,b,c + 1, 1,1`, { + allowDataProjection: {nilAsOptionalField: true}, + header: 1 + }); + test:assertEquals(cn5, [[1, 1, 1]]); + + map[]|csv:Error cn6 = csv:parseString(string ` + a,b,c + 1, 1,1`, { + allowDataProjection: {nilAsOptionalField: true}, + header: 1 + }); + test:assertEquals(cn6, [{a: 1, b: 1, c: 1}]); + + [int, string?, boolean?, decimal?, float?, (), string?][]|csv:Error cn7 = csv:parseString(csvStringData1, { + allowDataProjection: {nilAsOptionalField: true}, + header: 1, + skipLines: "2-10" + }); + test:assertEquals(cn7, [[1, "string1", true, 2.234, 2.234, null, null]]); + + [int, string?, boolean?, decimal?, float?, (), string?][]|csv:Error cn8 = csv:parseString(csvStringData1, { + allowDataProjection: {nilAsOptionalField: true}, + header: 1, + skipLines: "2-10" + }); + test:assertEquals(cn8, [[1, "string1", true, 2.234, 2.234, null, null]]); + + [int, string?, boolean?, decimal?, float?, (), string?][]|csv:Error cn9 = csv:parseString(csvStringData1, { + allowDataProjection: {nilAsOptionalField: true}, + header: 1, + skipLines: "2-10" + }); + test:assertEquals(cn9, [[1, "string1", true, 2.234, 2.234, null, null]]); + + [int, string?, boolean?, decimal?, float?, (), string?][]|csv:Error cn10 = csv:parseString(csvStringData1, { + allowDataProjection: {nilAsOptionalField: true}, + header: 1, + skipLines: "2-10" + }); + test:assertEquals(cn10, [[1, "string1", true, 2.234, 2.234, null, null]]); +} + +@test:Config +function testNilAsOptionalConfig2() returns error? { + record {|int a; int? f;|}[]|csv:Error cn = csv:transform([{"a": 1, "f": ()}, {"a": 2, "f": ()}, {"a": 3, "f": ()}], { + allowDataProjection: {nilAsOptionalField: true}, skipLines: [3] + }); + test:assertEquals(cn, [{a: 1, f: ()}, {a: 2, f: ()}]); + + record {|int a; int? f?;|}[]|csv:Error cn2 = csv:transform([{"a": 1, "f": ()}, {"a": 2, "f": ()}, {"a": 3, "f": ()}], { + allowDataProjection: {nilAsOptionalField: true}, skipLines: [3] + }); + test:assertEquals(cn2, [{a: 1, f: ()}, {a: 2, f: ()}]); + + record {|int a; int f?;|}[]|csv:Error cn3 = csv:transform([{"a": 1, "f": ()}, {"a": 2, "f": ()}, {"a": 3, "f": ()}], { + allowDataProjection: {nilAsOptionalField: true}, skipLines: [3] + }); + test:assertEquals(cn3, [{a: 1}, {a: 2}]); + + record {|int a; int f;|}[]|csv:Error cn4 = csv:transform([{"a": 1, "f": ()}, {"a": 2, "f": ()}, {"a": 3, "f": ()}], { + allowDataProjection: {nilAsOptionalField: true}, skipLines: [3] + }); + test:assertTrue(cn4 is csv:Error); + test:assertEquals((cn4).message(), common:generateErrorMessageForInvalidFieldType("null", "f")); +} + +@test:Config +function testDataProjectionConfig() returns error? { + string csvValue1 = string `a,b + "a",2 + b,4`; + record {}[] csvValue2 = [{"a": "a", "b": 2}, {"a": "b", "b": 4}]; + + record {}[]|csv:Error cn = csv:parseString(csvValue1, { + allowDataProjection: false + }); + test:assertEquals(cn, [{"a": "a", "b": 2}, {"a": "b", "b": 4}]); + + record {|string a; int b;|}[]|csv:Error cn_2 = csv:parseString(csvValue1, { + allowDataProjection: false + }); + test:assertEquals(cn_2, [{"a": "a", "b": 2}, {"a": "b", "b": 4}]); + + record {|string a;|}[]|csv:Error cn2 = csv:parseString(csvValue1, { + allowDataProjection: false + }); + test:assertTrue(cn2 is csv:Error); + test:assertEquals((cn2).message(), "no mapping field in the expected type for header 'b'"); + + record {|string a; int...;|}[]|csv:Error cn3 = csv:parseString(csvValue1, { + allowDataProjection: false + }); + test:assertEquals(cn3, [{"a": "a", "b": 2}, {"a": "b", "b": 4}]); + + record {|string...;|}[]|csv:Error cn4 = csv:parseString(csvValue1, { + allowDataProjection: false + }); + test:assertEquals(cn4, [{"a": "a", "b": "2"}, {"a": "b", "b": "4"}]); + + record {|string a?;|}[]|csv:Error cn5 = csv:parseString(csvValue1, { + allowDataProjection: false + }); + test:assertTrue(cn5 is csv:Error); + test:assertEquals((cn5).message(), "no mapping field in the expected type for header 'b'"); + + record {|string? a;|}[]|csv:Error cn6 = csv:parseString(csvValue1, { + allowDataProjection: false + }); + test:assertTrue(cn6 is csv:Error); + test:assertEquals((cn6).message(), "no mapping field in the expected type for header 'b'"); + + anydata[][]|csv:Error c7 = csv:parseString(csvValue1, { + allowDataProjection: false + }); + test:assertEquals(c7, [["a", 2], ["b", 4]]); + + [string, int][]|csv:Error cn7_2 = csv:parseString(csvValue1, { + allowDataProjection: false + }); + test:assertEquals(cn7_2, [["a", 2], ["b", 4]]); + + [string][]|csv:Error cn8 = csv:parseString(csvValue1, { + allowDataProjection: false + }); + test:assertTrue(cn8 is csv:Error); + test:assertEquals((cn8).message(), "invalid array size for expected tuple type, cannot be greater than '1'"); + + [string][]|csv:Error cn8_2 = csv:parseString(csvValue1, { + allowDataProjection: {} + }); + test:assertEquals(cn8_2, [["a"], ["b"]]); + + [int][]|csv:Error cn8_3 = csv:parseString(csvValue1, { + allowDataProjection: {} + }); + test:assertTrue(cn8_3 is csv:Error); + test:assertEquals((cn8_3).message(), common:generateErrorMessageForInvalidCast("a", "int")); + + [string, int...][]|csv:Error cn9 = csv:parseString(csvValue1, { + allowDataProjection: false + }); + test:assertEquals(cn9, [["a", 2], ["b", 4]]); + + [string...][]|csv:Error cn10 = csv:parseString(csvValue1, { + allowDataProjection: false + }); + test:assertEquals(cn10, [["a", "2"], ["b", "4"]]); + + [string, ()][]|csv:Error cn11 = csv:parseString(csvValue1, { + allowDataProjection: false + }); + test:assertTrue(cn11 is csv:Error); + test:assertEquals((cn11).message(), common:generateErrorMessageForInvalidCast("2", "()")); + + string[][]|csv:Error cn12 = csv:parseString(csvValue1, { + allowDataProjection: false + }); + test:assertEquals(cn12, [["a", "2"], ["b", "4"]]); + + string[][1]|csv:Error cn13 = csv:parseString(csvValue1, { + allowDataProjection: false + }); + test:assertTrue(cn13 is csv:Error); + test:assertEquals((cn13).message(), "invalid array size for expected array type, cannot be greater than '1'"); + + record {}[]|csv:Error cn14 = csv:transform(csvValue2, { + allowDataProjection: false + }); + test:assertEquals(cn14, [{"a": "a", "b": 2}, {"a": "b", "b": 4}]); + + record {|string a; int b;|}[]|csv:Error cn14_2 = csv:transform(csvValue2, { + allowDataProjection: false + }); + test:assertEquals(cn14_2, [{"a": "a", "b": 2}, {"a": "b", "b": 4}]); + + record {|string a;|}[]|csv:Error cn15 = csv:transform(csvValue2, { + allowDataProjection: false + }); + test:assertTrue(cn15 is csv:Error); + test:assertEquals((cn15).message(), "no mapping field in the expected type for header 'b'"); + + record {|string a; int...;|}[]|csv:Error cn16 = csv:transform(csvValue2, { + allowDataProjection: false + }); + test:assertEquals(cn16, [{"a": "a", "b": 2}, {"a": "b", "b": 4}]); + + record {|string...;|}[]|csv:Error cn17 = csv:transform(csvValue2, { + allowDataProjection: false + }); + test:assertEquals(cn17, [{"a": "a"}, {"a": "b"}]); + + record {|string a?;|}[]|csv:Error cn18 = csv:transform(csvValue2, { + allowDataProjection: false + }); + test:assertTrue(cn18 is csv:Error); + test:assertEquals((cn18).message(), "no mapping field in the expected type for header 'b'"); + + record {|string? a;|}[]|csv:Error cn19 = csv:transform(csvValue2, { + allowDataProjection: false + }); + test:assertTrue(cn19 is csv:Error); + test:assertEquals((cn19).message(), "no mapping field in the expected type for header 'b'"); + + anydata[][]|csv:Error c20 = csv:transform(csvValue2, {headerOrder: ["a", "b"], + allowDataProjection: false + }); + test:assertEquals(c20, [["a", 2], ["b", 4]]); + + [string, int][]|csv:Error cn20_2 = csv:transform(csvValue2, {headerOrder: ["a", "b"], + allowDataProjection: false + }); + test:assertEquals(cn20_2, [["a", 2], ["b", 4]]); + + [string][]|csv:Error cn21 = csv:transform(csvValue2, {headerOrder: ["a", "b"], + allowDataProjection: false + }); + test:assertTrue(cn21 is csv:Error); + test:assertEquals((cn21).message(), "invalid array size for expected tuple type, cannot be greater than '1'"); + + [string][]|csv:Error cn21_2 = csv:transform(csvValue2, {headerOrder: ["a", "b"], + allowDataProjection: {} + }); + test:assertEquals(cn21_2, [["a"], ["b"]]); + + [int][]|csv:Error cn21_3 = csv:transform(csvValue2, {headerOrder: ["a", "b"], + allowDataProjection: {} + }); + test:assertTrue(cn21_3 is csv:Error); + test:assertEquals((cn21_3).message(), common:generateErrorMessageForInvalidValueForArrayType("a", "0", "int")); + + [string, int...][]|csv:Error cn22 = csv:transform(csvValue2, {headerOrder: ["a", "b"], + allowDataProjection: false + }); + test:assertEquals(cn22, [["a", 2], ["b", 4]]); + + [string...][]|csv:Error cn23 = csv:transform(csvValue2, {headerOrder: ["a", "b"], + allowDataProjection: false + }); + test:assertTrue(cn23 is csv:Error); + test:assertEquals(( cn23).message(), common:generateErrorMessageForInvalidValueForArrayType("2", "1", "string")); + + [string, ()][]|csv:Error cn24 = csv:transform(csvValue2, {headerOrder: ["a", "b"], + allowDataProjection: false + }); + test:assertTrue(cn24 is csv:Error); + test:assertEquals((cn24).message(), common:generateErrorMessageForInvalidValueForArrayType("2", "1", "()")); + + string[][]|csv:Error cn25 = csv:transform(csvValue2, {headerOrder: ["a", "b"], + allowDataProjection: false + }); + test:assertTrue(cn25 is csv:Error); + test:assertEquals(( cn25).message(), common:generateErrorMessageForInvalidValueForArrayType("2", "1", "string")); + + string[][1]|csv:Error cn26 = csv:transform(csvValue2, {headerOrder: ["a", "b"], + allowDataProjection: false + }); + test:assertTrue(cn26 is csv:Error); + test:assertEquals((cn26).message(), "invalid array size for expected array type, cannot be greater than '1'"); +} diff --git a/ballerina-tests/user-config-tests/tests/user_config_with_parser_options_test.bal b/ballerina-tests/user-config-tests/tests/user_config_with_parser_options_test.bal new file mode 100644 index 0000000..503a922 --- /dev/null +++ b/ballerina-tests/user-config-tests/tests/user_config_with_parser_options_test.bal @@ -0,0 +1,1030 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/csv_commons as common; +import ballerina/data.csv; +import ballerina/test; + +@test:Config +function testCSVEncoding() returns error? { + record {}[]|csv:Error rec; + + string csvStr = string `value + Alice + 2πr + €123`; + byte[] csvBytes = string:toBytes(csvStr); + + rec = csv:parseBytes(csvBytes, {locale: "fr_FR", encoding: "ISO-8859-1"}); + test:assertEquals((check rec)[0], {value: "Alice"}); +} + +@test:Config +function testCSVLocale() { + record {|string name; decimal completed\ tasks; string city;|}[]|csv:Error rec; + + rec = csv:parseString(string ` + name, completed tasks, city + Alice, "1234", New York + Bob, "1,234", London + €123, "12,34", Berlin`, {header: 1, locale: "fr_FR"}); + test:assertEquals(rec, [ + {name: "Alice", "completed tasks": 1234, city: "New York"}, + {name: "Bob", "completed tasks": 1.234, city: "London"}, + {name: "€123", "completed tasks": 12.34, city: "Berlin"} + ]); + + rec = csv:parseString(string ` + name, completed tasks, city + Alice, "1234", New York + Bob, "1.234", London + €123, "12.34", Berlin`, {header: 1, locale: "en"}); + test:assertEquals(rec, [ + {name: "Alice", "completed tasks": 1234, city: "New York"}, + {name: "Bob", "completed tasks": 1.234, city: "London"}, + {name: "€123", "completed tasks": 12.34, city: "Berlin"} + ]); + + rec = csv:parseString(string ` + name, completed tasks, city + Alice, "1234", New York + Bob, "1.234", London + €123, "12.34", Berlin`, {header: 1, locale: "en_US_WIN"}); + test:assertEquals(rec, [ + {name: "Alice", "completed tasks": 1234, city: "New York"}, + {name: "Bob", "completed tasks": 1.234, city: "London"}, + {name: "€123", "completed tasks": 12.34, city: "Berlin"} + ]); +} + +@test:Config {dependsOn: [testCSVLocale]} +function testFromCsvStringWithParserOptions() { + [int, string, boolean, decimal, float, string][]|csv:Error csv1op3 = csv:parseString(csvStringData1, option3); + test:assertEquals(csv1op3, [ + [3, "string3", false, 1.23, 1.23, "()"], + [4, "string4", true, -6.51, -6.51, "()"], + [5, "string5", true, 3, 3.0, "()"] + ]); + + record {int a; string b; boolean c; decimal d; float e; string f;}[]|csv:Error csv1op3_2 = csv:parseString(csvStringData1, ptOption1); + test:assertEquals(csv1op3_2, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: "()"}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: "()"}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: "()"} + ]); + + record {int a; string b; boolean c; decimal d; float e; string f;}[]|csv:Error csv1op3_3 = csv:parseString(csvStringData1, ptOption2); + test:assertEquals(csv1op3_3, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: "()"}, + {a: 5, b: "string5", c: true, d: 3, e: 3.0, f: "()"} + ]); + + [int, string, boolean, decimal, float, string][]|csv:Error csv2op4 = csv:parseString(csvStringData2, option4); + test:assertEquals(csv2op4, []); + + record {}[]|csv:Error csv2op4_2 = csv:parseString(csvStringData2, ptOption3); + test:assertEquals(csv2op4_2, []); + + record {}[]|csv:Error csv2op4_3 = csv:parseString(csvStringData2, ptOption4); + test:assertEquals(csv2op4_3, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: "()"}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: "()"}, + {a: 5, b: "string5", c: true, d: 3, e: 3, f: "()"} + ]); + + [int, string, boolean, decimal, float, string][]|csv:Error csv3op3 = csv:parseString(csvStringData3, option3); + test:assertEquals(csv3op3, [ + [3, "string3", false, 1.23, 1.23, "()"], + [4, "string4", true, -6.51, -6.51, "()"], + [5, "string5", true, 3, 3.0, "()"] + ]); + + record {}[]|csv:Error csv3op3_2 = csv:parseString(csvStringData3, ptOption1); + test:assertEquals(csv3op3_2, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: "()"}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: "()"}, + {a: 5, b: "string5", c: true, d: 3, e: 3, f: "()"} + ]); + + record {}[]|csv:Error csv3op3_3 = csv:parseString(csvStringData3, ptOption2); + test:assertEquals(csv3op3_3, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: "()"}, + {a: 5, b: "string5", c: true, d: 3, e: 3, f: "()"} + ]); + + record {}[]|csv:Error csv3op3_4 = csv:parseString(csvStringData11, {header: 9, skipLines: "2-10"}); + test:assertEquals(csv3op3_4, [ + {'4: 5, string4: "string5", "true": true, "-6.51": 3, "-6.52": 3, "()": null} + ]); +} + +@test:Config {dependsOn: [testCSVLocale]} +function testFromCsvStringWithHeaderLessParserOptions() { + [int, string, boolean, decimal, float, ()][]|csv:Error csv1op6 = csv:parseString(csvStringData1, option6); + test:assertTrue(csv1op6 is csv:Error); + test:assertEquals((csv1op6).message(), common:generateErrorMessageForInvalidCast("null", "()")); + + record {}[]|csv:Error csv1op5_2 = csv:parseString(csvStringData1, ptOption5); + test:assertEquals(csv1op5_2, [ + {'1: "a", '2: "b", '3: "c", '4: "d", '5: "e", '6: "f"}, + {'1: 1, '2: "string1", '3: true, '4: 2.234, '5: 2.234, '6: ()}, + {'1: 3, '2: "string3", '3: false, '4: 1.23, '5: 1.23, '6: ()}, + {'1: 5, '2: "string5", '3: true, '4: 3, '5: 3, '6: ()} + ]); + + record {}[]|csv:Error csv1op6_2 = csv:parseString(csvStringData1, {header: null, skipLines: [3, 5]}); + test:assertEquals(csv1op6_2, [ + {'1: "a", '2: "b", '3: "c", '4: "d", '5: "e", '6: "f"}, + {'1: 1, '2: "string1", '3: true, '4: 2.234, '5: 2.234, '6: ()}, + {'1: 3, '2: "string3", '3: false, '4: 1.23, '5: 1.23, '6: ()}, + {'1: 5, '2: "string5", '3: true, '4: 3, '5: 3, '6: ()} + ]); + + record {}[]|csv:Error csv3op6_2 = csv:parseString(csvStringData3, {header: (), skipLines: [1, 3, 5, -1, 100, 100]}); + test:assertEquals(csv3op6_2, [ + {'1: 1, '2: "string1", '3: true, '4: 2.234, '5: 2.234, '6: ()}, + {'1: 3, '2: "string3", '3: false, '4: 1.23, '5: 1.23, '6: ()}, + {'1: 5, '2: "string5", '3: true, '4: 3, '5: 3, '6: ()} + ]); + + record {}[]|csv:Error csv4op6_2 = csv:parseString(csvStringData4, {header: (), skipLines: [2, 4, -1, 100, 100]}); + test:assertEquals(csv4op6_2, [ + {'1: 1, '2: "string1", '3: true, '4: 2.234, '5: 2.234, '6: ()}, + {'1: 3, '2: "string3", '3: false, '4: 1.23, '5: 1.23, '6: ()}, + {'1: 5, '2: "string5", '3: true, '4: 3, '5: 3, '6: ()} + ]); + + record {}[]|csv:Error csv5op6_2 = csv:parseString(csvStringData5, {header: null, skipLines: [2, 4, -1, 100, 100]}); + test:assertEquals(csv5op6_2, [ + {'1: 1, '2: "string1", '3: true, '4: 2.234, '5: 2.234, '6: ()}, + {'1: 3, '2: "string3", '3: false, '4: 1.23, '5: 1.23, '6: ()}, + {'1: 5, '2: "string5", '3: true, '4: 3, '5: 3, '6: ()} + ]); + + record {}[]|csv:Error csv6op6_2 = csv:parseString(csvStringData6, {header: (), skipLines: [2, 4, -1, 100, 100]}); + test:assertEquals(csv6op6_2, [ + {'1: 1, '2: "string1", '3: true, '4: 2.234, '5: 2.234, '6: ()}, + {'1: 3, '2: "string3", '3: false, '4: 1.23, '5: 1.23, '6: ()}, + {'1: 5, '2: "string5", '3: true, '4: 3, '5: 3, '6: ()} + ]); + + record {}[]|csv:Error csv2op6_2 = csv:parseString(csvStringData2, {header: (), skipLines: [5, 7]}); + test:assertEquals(csv2op6_2, [ + {'1: "hello", '2: "hello", '3: (), '4: 12, '5: true, '6: 12.34}, + {'1: "// comment"}, + {'1: "a", '2: "b", '3: "c", '4: "d", '5: "e", '6: "f"}, + {'1: 1, '2: "string1", '3: true, '4: 2.234, '5: 2.234, '6: ()}, + {'1: 3, '2: "string3", '3: false, '4: 1.23, '5: 1.23, '6: ()}, + {'1: 5, '2: "string5", '3: true, '4: 3, '5: 3, '6: ()} + ]); +} + +@test:Config {dependsOn: [testCSVLocale]} +function testHeaderOption() { + record {}[]|csv:Error csv2cop1 = csv:parseString(csvStringData2, {header: 4}); + test:assertEquals(csv2cop1, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 2, b: "string2", c: false, d: 0, e: 0, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 4, b: "string4", c: true, d: -6.51, e: -6.51, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3, f: ()} + ]); + + record {}[]|csv:Error csv2cop2 = csv:parseString(csvStringData10, {header: 100}); + test:assertTrue(csv2cop2 is csv:Error); + test:assertEquals((csv2cop2).message(), "provided header row is empty"); + + record {}[]|csv:Error csv2cop3 = csv:parseString(csvStringData10, {header: 11}); + test:assertEquals(csv2cop3, []); + + record {}[]|csv:Error csv2cop3_2 = csv:parseString(csvStringData10, {header: 9}); + test:assertTrue(csv2cop3_2 is csv:Error); + test:assertEquals((csv2cop3_2).message(), "duplicate header found: '1.23'"); + + record {}[]|csv:Error csv2cop4 = csv:parseString(csvStringData10, {header: 10}); + test:assertEquals(csv2cop4, [{'4: 5, string4: "string5", "true": true, "-6.51": 3, "-6.52": 31, "()": ()}]); + + record {}[]|csv:Error csv1cop5 = csv:parseString(csvStringData1, {}); + test:assertTrue(csv1cop5 is csv:Error); + test:assertEquals((csv1cop5).message(), "provided header row is empty"); +} + +@test:Config {dependsOn: [testCSVLocale]} +function testNullConfigOption() { + string csvValue1 = string `a + ()`; + string csvValue2 = string `a + null`; + string csvValue3 = string `c, a + true, e`; + string csvValue4 = string `a + Null`; + string csvValue5 = string `b, a + bN/Aa,N/A`; + + record {() a;}[]|csv:Error cn = csv:parseString(csvValue1, {nilValue: ()}); + test:assertEquals(cn, [{a: ()}]); + + cn = csv:parseString(csvValue2, {nilValue: ()}); + test:assertEquals(cn, [{a: ()}]); + + cn = csv:parseString(csvValue3, {nilValue: ()}); + test:assertTrue(cn is csv:Error); + test:assertEquals((cn).message(), common:generateErrorMessageForInvalidCast("e", "()")); + + cn = csv:parseString(csvValue4, {nilValue: ()}); + test:assertEquals(cn, [{a: ()}]); + + cn = csv:parseString(csvValue1, {nilValue: null}); + test:assertEquals(cn, [{a: ()}]); + + cn = csv:parseString(csvValue2, {nilValue: null}); + test:assertEquals(cn, [{a: ()}]); + + cn = csv:parseString(csvValue3, {nilValue: null}); + test:assertTrue(cn is csv:Error); + test:assertEquals((cn).message(), common:generateErrorMessageForInvalidCast("e", "()")); + + cn = csv:parseString(csvValue4, {nilValue: null}); + test:assertEquals(cn, [{a: ()}]); + + cn = csv:parseString(csvValue1, {nilValue: "()"}); + test:assertEquals(cn, [{a: ()}]); + + cn = csv:parseString(csvValue2, {nilValue: "()"}); + test:assertTrue(cn is csv:Error); + test:assertEquals((cn).message(), common:generateErrorMessageForInvalidCast("null", "()")); + + cn = csv:parseString(csvValue3, {nilValue: "()"}); + test:assertTrue(cn is csv:Error); + test:assertEquals((cn).message(), common:generateErrorMessageForInvalidCast("e", "()")); + + cn = csv:parseString(csvValue4, {nilValue: "()"}); + test:assertTrue(cn is csv:Error); + test:assertEquals((cn).message(), common:generateErrorMessageForInvalidCast("Null", "()")); + + cn = csv:parseString(csvValue5, {nilValue: "N/A"}); + test:assertEquals(cn, [{b: "bN/Aa", a: ()}]); + + cn = csv:parseString(csvValue2, {nilValue: "null"}); + test:assertEquals(cn, [{a: ()}]); + + cn = csv:parseString(csvValue4, {nilValue: "null"}); + test:assertTrue(cn is csv:Error); + test:assertEquals((cn).message(), common:generateErrorMessageForInvalidCast("Null", "()")); + + cn = csv:parseString(csvValue1, {nilValue: "null"}); + test:assertTrue(cn is csv:Error); + test:assertEquals((cn).message(), common:generateErrorMessageForInvalidCast("()", "()")); +} + +@test:Config {dependsOn: [testCSVLocale]} +function testCommentConfigOption() { + string csvValue1 = string `a + 1`; + string csvValue2 = string `a # comment + 1`; + string csvValue3 = string `a #, c + 1#, e`; + string csvValue4 = string `a + # comment + 1`; + string csvValue5 = string `a, b + 1,2 + # comment`; + string csvValue6 = string `a, b + 1,2 # comment + # comment`; + string csvValue7 = string `a, b + 1#,2 comment + # comment`; + string csvValue8 = string `a#, b + 1, 2 # comment + # comment`; + string csvValue9 = string `a,# b + 1 ,#2 # comment + # comment`; + string csvValue10 = string `a# b + 1 ,#2 # comment + # comment`; + + record {int a;}[]|csv:Error cn; + + cn = csv:parseString(csvValue1); + test:assertEquals(cn, [{a: 1}]); + + cn = csv:parseString(csvValue2); + test:assertEquals(cn, [{a: 1}]); + + cn = csv:parseString(csvValue3); + test:assertEquals(cn, [{a: 1}]); + + cn = csv:parseString(csvValue4); + test:assertEquals(cn, [{a: 1}]); + + cn = csv:parseString(csvValue5); + test:assertEquals(cn, [{a: 1, b: 2}]); + + cn = csv:parseString(csvValue6); + test:assertEquals(cn, [{a: 1, b: 2}]); + + cn = csv:parseString(csvValue7); + test:assertEquals(cn, [{a: 1}]); + + record {|int a; int b;|}[]|csv:Error cn2 = csv:parseString(csvValue7, {header: 0}); + test:assertEquals(cn2, [{a: 1, b: 0}]); + + cn = csv:parseString(csvValue8); + test:assertTrue(cn is csv:Error); + // TODO:Fix the error message + // test:assertEquals(( cn).message(), common:generateErrorMessageForInvalidCast("1, 2", "int")); + + cn = csv:parseString(csvValue10); + test:assertTrue(cn is csv:Error); + test:assertEquals(( cn).message(), "invalid number of headers"); + + cn = csv:parseString(csvValue9); + test:assertTrue(cn is csv:Error); + test:assertEquals(( cn).message(), "provided header row is empty"); +} + +@test:Config {dependsOn: [testCSVLocale]} +function testCommentConfigOption2() { + string csvValue1 = string `a + 1`; + string csvValue2 = string `a & comment + 1`; + string csvValue3 = string `a &, c + 1&, e`; + string csvValue4 = string `a + + + + & comment + 1`; + string csvValue5 = string `a&, b + 1, 2 & comment + + + & comment`; + string csvValue6 = string ` + + a,& b + 1 ,&2 & comment + + & comment`; + + string csvValue7 = string ` + + a& b + 1 ,&2 & comment + + & comment`; + + string csvValue8 = string ` + + a,e& b + 1 ,&2 & comment + + & comment`; + + record {int a; int b;}[]|csv:Error cn; + record {int c;}[]|csv:Error cn2; + + cn = csv:parseString(csvValue1, {comment: "&"}); + test:assertTrue(cn is csv:Error); + test:assertEquals((cn).message(), common:generateErrorMessageForMissingRequiredField("b")); + + cn = csv:parseString(csvValue2, {comment: "&"}); + test:assertTrue(cn is csv:Error); + test:assertEquals((cn).message(), common:generateErrorMessageForMissingRequiredField("b")); + + cn = csv:parseString(csvValue3, {comment: "&"}); + test:assertTrue(cn is csv:Error); + test:assertEquals((cn).message(), common:generateErrorMessageForMissingRequiredField("b")); + + cn = csv:parseString(csvValue4, {comment: "&"}); + test:assertTrue(cn is csv:Error); + test:assertEquals((cn).message(), common:generateErrorMessageForMissingRequiredField("b")); + + cn = csv:parseString(csvValue5, {comment: "&"}); + test:assertTrue(cn is csv:Error); + // TODO: Fix the error message + // test:assertEquals(( cn).message(), common:generateErrorMessageForMissingRequiredField("b")); + + cn = csv:parseString(csvValue6, {comment: "&", header: 2}); + test:assertTrue(cn is csv:Error); + test:assertEquals((cn).message(), "provided header row is empty"); + + cn = csv:parseString(csvValue8, {comment: "&", header: 2}); + test:assertTrue(cn is csv:Error); + test:assertEquals((cn).message(), common:generateErrorMessageForMissingRequiredField("b")); + + cn = csv:parseString(csvValue7, {comment: "&", header: 2}); + test:assertTrue(cn is csv:Error); + test:assertEquals((cn).message(), "invalid number of headers"); + + cn2 = csv:parseString(csvValue1, {comment: "&"}); + test:assertTrue(cn2 is csv:Error); + test:assertEquals((cn2).message(), common:generateErrorMessageForMissingRequiredField("c")); + + cn2 = csv:parseString(csvValue2, {comment: "&"}); + test:assertTrue(cn2 is csv:Error); + test:assertEquals((cn2).message(), common:generateErrorMessageForMissingRequiredField("c")); + + cn2 = csv:parseString(csvValue3, {comment: "&"}); + test:assertTrue(cn2 is csv:Error); + test:assertEquals((cn2).message(), common:generateErrorMessageForMissingRequiredField("c")); + + cn2 = csv:parseString(csvValue4, {comment: "&"}); + test:assertTrue(cn2 is csv:Error); + test:assertEquals((cn2).message(), common:generateErrorMessageForMissingRequiredField("c")); + + cn2 = csv:parseString(csvValue5, {comment: "&"}); + test:assertTrue(cn2 is csv:Error); + // TODO: Fix the error message + // test:assertEquals(( cn2).message(), common:generateErrorMessageForMissingRequiredField("c")); + + cn2 = csv:parseString(csvValue6, {header: 2, comment: "&"}); + test:assertTrue(cn2 is csv:Error); + test:assertEquals((cn2).message(), "provided header row is empty"); +} + +@test:Config {dependsOn: [testCSVLocale]} +function testSkipLineParserOption() { + [int, string, boolean, decimal, float, ()][]|csv:Error csv1cp = csv:parseString(csvStringData1, {skipLines: [], header: 1}); + test:assertEquals(csv1cp, [ + [1, "string1", true, 2.234, 2.234, ()], + [2, "string2", false, 0, 0, ()], + [3, "string3", false, 1.23, 1.23, ()], + [4, "string4", true, -6.51, -6.51, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + csv1cp = csv:parseString(csvStringData1, {skipLines: [0], header: 1}); + test:assertEquals(csv1cp, [ + [1, "string1", true, 2.234, 2.234, ()], + [2, "string2", false, 0, 0, ()], + [3, "string3", false, 1.23, 1.23, ()], + [4, "string4", true, -6.51, -6.51, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + csv1cp = csv:parseString(csvStringData1, {skipLines: [0, 4, 10], header: 1}); + test:assertEquals(csv1cp, [ + [1, "string1", true, 2.234, 2.234, ()], + [2, "string2", false, 0, 0, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + csv1cp = csv:parseString(csvStringData1, {skipLines: [1, 2, 3, 4, 5], header: 1}); + test:assertEquals(csv1cp, [ + ]); + + csv1cp = csv:parseString(csvStringData1, {skipLines: "1-5", header: 1}); + test:assertEquals(csv1cp, [ + ]); + + csv1cp = csv:parseString(csvStringData1, {skipLines: [4, 2], header: 1}); + test:assertEquals(csv1cp, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + csv1cp = csv:parseString(csvStringData1, {skipLines: "2-4", header: 1}); + test:assertEquals(csv1cp, [ + [1, "string1", true, 2.234, 2.234, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + csv1cp = csv:parseString(csvStringData1, {skipLines: [4, 2, -1], header: 1}); + test:assertEquals(csv1cp, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + csv1cp = csv:parseString(csvStringData1, {skipLines: [4, -1, 2], header: 1}); + test:assertEquals(csv1cp, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + csv1cp = csv:parseString(csvStringData1, {skipLines: [5, 4, 3, 2, 1], header: 1}); + test:assertEquals(csv1cp, [ + ]); + + csv1cp = csv:parseString(csvStringData1, {skipLines: [10], header: 1}); + test:assertEquals(csv1cp, [ + [1, "string1", true, 2.234, 2.234, ()], + [2, "string2", false, 0, 0, ()], + [3, "string3", false, 1.23, 1.23, ()], + [4, "string4", true, -6.51, -6.51, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + csv1cp = csv:parseString(csvStringData1, {skipLines: [-2], header: 1}); + test:assertEquals(csv1cp, [ + [1, "string1", true, 2.234, 2.234, ()], + [2, "string2", false, 0, 0, ()], + [3, "string3", false, 1.23, 1.23, ()], + [4, "string4", true, -6.51, -6.51, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + csv1cp = csv:parseString(csvStringData1, {skipLines: [-2, 0, 2], header: 1}); + test:assertEquals(csv1cp, [ + [1, "string1", true, 2.234, 2.234, ()], + [3, "string3", false, 1.23, 1.23, ()], + [4, "string4", true, -6.51, -6.51, ()], + [5, "string5", true, 3, 3.0, ()] + ]); + + csv1cp = csv:parseString(csvStringData1, {skipLines: "0-2", header: 1}); + test:assertEquals(csv1cp, [ + [3, "string3", false, 1.23, 1.23, ()], + [4, "string4", true, -6.51, -6.51, ()], + [5, "string5", true, 3, 3.0, ()] + ]); +} + +@test:Config {dependsOn: [testCSVLocale]} +function testCustomHeaderOption() { + anydata[][]|csv:Error bm1ba = csv:transform([bm1, bm1], {headerOrder: ["b1", "b2"]}); + test:assertEquals(bm1ba, [ + [true, false], + [true, false] + ]); + + anydata[][]|csv:Error bm1ba2 = csv:transform([bm1, bm1], {headerOrder: ["b2", "b1"]}); + test:assertEquals(bm1ba2, [ + [false, true], + [false, true] + ]); + + anydata[][]|csv:Error bm2ba = csv:transform([bm2, bm2], {headerOrder: ["b1", "n1", "b2", "n2", "b3"]}); + test:assertTrue(bm2ba is csv:Error); + test:assertEquals((bm2ba).message(), "CSV data rows with varying headers are not yet supported"); + + anydata[][]|csv:Error bm3ba = csv:transform([bm3, bm3], {headerOrder: ["b1", "b4", "b2", "n2", "i1"]}); + test:assertTrue(bm3ba is csv:Error); + test:assertEquals((bm3ba).message(), "CSV data rows with varying headers are not yet supported"); + + anydata[][]|csv:Error bm3ba2 = csv:transform([bm3, bm3], {headerOrder: ["b1", "b3", "b4", "b2", "i2"]}); + test:assertTrue(bm3ba2 is csv:Error); + test:assertEquals((bm3ba2).message(), "CSV data rows with varying headers are not yet supported"); + + [boolean...][]|csv:Error bm3ba4 = csv:transform([bm3, bm3], {headerOrder: ["n2"]}); + test:assertTrue(bm3ba4 is csv:Error); + test:assertEquals((bm3ba4).message(), "invalid number of headers"); + + [boolean...][]|csv:Error bm3ba5 = csv:transform([bm3, bm3], {headerOrder: []}); + test:assertTrue(bm3ba5 is csv:Error); + test:assertEquals((bm3ba5).message(), "invalid number of headers"); + + record {}[]|csv:Error ct1br = csv:parseList([["a", "1", "true"], ["a", "1", "true"]], {customHeaders: ["a", "b"]}); + test:assertTrue(ct1br is csv:Error); + test:assertEquals((ct1br).message(), "invalid number of headers"); + + record {}[]|csv:Error ct1br2 = csv:parseList([["a", "1", "true"], ["a", "1", "true"]], {customHeaders: ["a", "b", "c", "d"]}); + test:assertTrue(ct1br2 is csv:Error); + test:assertEquals((ct1br2).message(), "invalid number of headers"); + + record {}[]|csv:Error ct1br2_2 = csv:parseList([["a", "1", "true"], ["a", "1", "true"]], {customHeaders: ["a", "c", "b", "d"]}); + test:assertTrue(ct1br2_2 is csv:Error); + test:assertEquals((ct1br2_2).message(), "invalid number of headers"); + + record {}[]|csv:Error ct1br3 = csv:parseList([["a", "1", "true"], ["a", "1", "true"]], {customHeaders: []}); + test:assertTrue(ct1br3 is csv:Error); + test:assertEquals((ct1br3).message(), "invalid number of headers"); + + record {|string a; string b; string c;|}[]|csv:Error ct1br5 = csv:parseList([["a", "1", "true"], ["a", "1", "true"]], {customHeaders: ["a", "e", "b"]}); + test:assertTrue(ct1br5 is csv:Error); + // TODO: Fix the error message + test:assertEquals((ct1br5).message(), common:generateErrorMessageForMissingRequiredField("c")); + + record {string a; string b; string c;}[]|csv:Error ct1br6 = csv:parseList([["a", "1", "true"], ["a", "1", "true"]], {customHeaders: ["a", "e", "b"]}); + test:assertTrue(ct1br6 is csv:Error); + test:assertEquals((ct1br6).message(), common:generateErrorMessageForMissingRequiredField("c")); + + record {string a; string b;}[]|csv:Error ct1br7 = csv:parseList([["a", "1", "true"], ["a", "1", "true"]], { + customHeaders: ["a", "e", "b"] + }); + test:assertEquals(ct1br7, [ + {a: "a", e: 1, b: "true"}, + {a: "a", e: 1, b: "true"} + ]); + + record {|string a; string b;|}[]|csv:Error ct1br8 = csv:parseList([["a", "1", "true"], ["a", "1", "true"]], { + customHeaders: ["a", "e", "b"] + }); + test:assertEquals(ct1br8, [ + {a: "a", b: "true"}, + {a: "a", b: "true"} + ]); + + record {|string...;|}[]|csv:Error ct1br9 = csv:parseList([["a", "1", "true"], ["a", "1", "true"]], { + customHeaders: ["a", "e", "b"] + }); + test:assertEquals(ct1br9, [ + {a: "a", b: "true", e: "1"}, + {a: "a", b: "true", e: "1"} + ]); + + record {|string...;|}[]|csv:Error ct1br10 = csv:parseList([["a", "1", "true"], ["a", "1", "true"]], { + }); + test:assertEquals(ct1br10, [ + {'1: "a", '3: "true", '2: "1"}, + {'1: "a", '3: "true", '2: "1"} + ]); +} + +@test:Config +function testCustomHeaderParserOption2() { + record {}[]|csv:Error ct1br = csv:parseString(csvStringData1, {header: 1}); + test:assertEquals(ct1br, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 2, b: "string2", c: false, d: 0, e: 0, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 4, b: "string4", c: true, d: -6.51, e: -6.51, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3, f: ()} + ]); + + ct1br = csv:parseString(csvStringData1, {header: 1}); + test:assertEquals(ct1br, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 2, b: "string2", c: false, d: 0, e: 0, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 4, b: "string4", c: true, d: -6.51, e: -6.51, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3, f: ()} + ]); + + ct1br = csv:parseString(csvStringData1, {header: 1}); + test:assertEquals(ct1br, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 2, b: "string2", c: false, d: 0, e: 0, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 4, b: "string4", c: true, d: -6.51, e: -6.51, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3, f: ()} + ]); + + record {}[]|csv:Error ct1br2 = csv:parseString(csvStringData4, {header: (), customHeadersIfHeadersAbsent: []}); + test:assertTrue(ct1br2 is csv:Error); + test:assertEquals((ct1br2).message(), "invalid number of headers"); + + record {int a; string b; boolean c; decimal d; float e; () f;}[]|csv:Error ct1br3 = csv:parseString(csvStringData4, {header: null, customHeadersIfHeadersAbsent: ["a", "b"]}); + test:assertTrue(ct1br3 is csv:Error); + test:assertEquals((ct1br3).message(), "invalid number of headers"); + + record {int a; string b; boolean c; decimal d; float e; () f;}[]|csv:Error ct1br4 = csv:parseString(csvStringData1, {header: 1}); + test:assertEquals(ct1br4, [ + {a: 1, b: "string1", c: true, d: 2.234, e: 2.234, f: ()}, + {a: 2, b: "string2", c: false, d: 0, e: 0, f: ()}, + {a: 3, b: "string3", c: false, d: 1.23, e: 1.23, f: ()}, + {a: 4, b: "string4", c: true, d: -6.51, e: -6.51, f: ()}, + {a: 5, b: "string5", c: true, d: 3, e: 3, f: ()} + ]); + + record {() a; float b; decimal c; boolean d; string e; int f;}[]|csv:Error ct1br5 = csv:parseString(csvStringData4, {header: 1}); + test:assertTrue(ct1br5 is csv:Error); + + ct1br5 = csv:parseString(csvStringData4, {header: (), customHeadersIfHeadersAbsent: ["f", "e", "d", "c", "b", "a"]}); + test:assertEquals(ct1br5, [ + {f: 1, e: "string1", d: true, c: 2.234, b: 2.234, a: ()}, + {f: 2, e: "string2", d: false, c: 0, b: 0, a: ()}, + {f: 3, e: "string3", d: false, c: 1.23, b: 1.23, a: ()}, + {f: 4, e: "string4", d: true, c: -6.51, b: -6.51, a: ()}, + {f: 5, e: "string5", d: true, c: 3, b: 3, a: ()} + ]); + + record {() a; float b; decimal c; boolean d; string e; int f;}[]|csv:Error ct1br5_2 = csv:parseString(csvStringData4, {header: null, skipLines: [1], customHeadersIfHeadersAbsent: ["f", "e", "d", "c", "b", "a"]}); + test:assertEquals(ct1br5_2, [ + {f: 2, e: "string2", d: false, c: 0, b: 0, a: ()}, + {f: 3, e: "string3", d: false, c: 1.23, b: 1.23, a: ()}, + {f: 4, e: "string4", d: true, c: -6.51, b: -6.51, a: ()}, + {f: 5, e: "string5", d: true, c: 3, b: 3, a: ()} + ]); + + record {() a; float b; decimal c; boolean d; string e; int f;}[]|csv:Error ct1br5_3 = csv:parseString(csvStringData4, {skipLines: [1], header: (), customHeadersIfHeadersAbsent: ["f", "e", "d", "c", "b", "a"]}); + test:assertEquals(ct1br5_3, [ + {f: 2, e: "string2", d: false, c: 0, b: 0, a: ()}, + {f: 3, e: "string3", d: false, c: 1.23, b: 1.23, a: ()}, + {f: 4, e: "string4", d: true, c: -6.51, b: -6.51, a: ()}, + {f: 5, e: "string5", d: true, c: 3, b: 3, a: ()} + ]); + + record {|boolean d1; string e1;|}[]|csv:Error ct1br7 = csv:parseString(csvStringData4, {header: (), customHeadersIfHeadersAbsent: ["f1", "e1", "d1", "c1", "b1", "a1"]}); + test:assertEquals(ct1br7, [ + {e1: "string1", d1: true}, + {e1: "string2", d1: false}, + {e1: "string3", d1: false}, + {e1: "string4", d1: true}, + {e1: "string5", d1: true} + ]); + + record {|boolean d1; string e1;|}[]|csv:Error ct1br7_2 = csv:parseString(csvStringData4, {header: null, skipLines: [1], customHeadersIfHeadersAbsent: ["f1", "e1", "d1", "c1", "b1", "a1"]}); + test:assertEquals(ct1br7_2, [ + {e1: "string2", d1: false}, + {e1: "string3", d1: false}, + {e1: "string4", d1: true}, + {e1: "string5", d1: true} + ]); + + record {|boolean d1; string e1;|}[]|csv:Error ct1br8 = csv:parseString(csvStringData4, {header: (), customHeadersIfHeadersAbsent: ["e1", "d1"]}); + test:assertTrue(ct1br8 is csv:Error); + test:assertEquals((ct1br8).message(), common:generateErrorMessageForInvalidCast("string1", "boolean")); + + record {|boolean d1; string e1;|}[]|csv:Error ct1br9 = csv:parseString(csvStringData4, {header: (), customHeadersIfHeadersAbsent: ["f1", "e1", "d1", "c1", "b1", "a1"]}); + test:assertEquals(ct1br9, [ + {e1: "string1", d1: true}, + {e1: "string2", d1: false}, + {e1: "string3", d1: false}, + {e1: "string4", d1: true}, + {e1: "string5", d1: true} + ]); + + record {|boolean d1; string e1;|}[]|csv:Error ct1br11 = csv:parseString(csvStringData1, {header: (), customHeadersIfHeadersAbsent: ["f1", "e1"]}); + test:assertTrue(ct1br11 is csv:Error); + test:assertEquals((ct1br11).message(), "invalid number of headers"); + + record {|string d1; string e1;|}[]|csv:Error ct1br12 = csv:parseString(csvStringData4, {header: null, customHeadersIfHeadersAbsent: ["f1", "e1", "d1", "c1", "b1", "a1"]}); + test:assertEquals(ct1br12, [ + {e1: "string1", d1: "true"}, + {e1: "string2", d1: "false"}, + {e1: "string3", d1: "false"}, + {e1: "string4", d1: "true"}, + {e1: "string5", d1: "true"} + ]); + + record {|string d1; string e1;|}[]|csv:Error ct1br13 = csv:parseString(csvStringData1, {header: (), customHeadersIfHeadersAbsent: ["f1", "e1", "dd1", "c1", "b1", "a1"]}); + test:assertTrue(ct1br13 is csv:Error); + test:assertEquals((ct1br13).message(), common:generateErrorMessageForMissingRequiredField("d1")); +} + +@test:Config +function testTextQuotesWithParserOptions() { + string csvValue1 = string ` + a, b, c + 1, "2", "3" + "1", 2, 3 + 1, "2", 3 + + "1", "2", "3"`; + + string csvValue2 = string ` + a, b, c + 1, "2, 3", 3 + 1, "2, 3",3 + 4, 5, 6 + `; + + string csvValue3 = string `a, b, c + "1", ""2"", "3" + 4, "5, 6"b" " a "", ""6""`; + + string csvValue4 = string `a, b, c + 1, '2', 3 + 4, '5, '6'7', 8 + 4, "5", '4, '5"a", ,"," a '6'7'`; + + string csvValue5 = string `a, b, c + 1, "2", "3" + 1, 2, 3 + "1", "2", 3 + 1, "2", "3" + `; + + string csvValue6 = string `a, b, c + 1, "2 a ","", "3" + 1, 2, 3 + "1", "2", 3 + 1, "2", "3" + `; + + record {int a;}[]|csv:Error cn = csv:parseString(csvValue1, {header: 1}); + test:assertEquals(cn, [{"a": 1, "b": 2, "c": 3}, {"a": 1, "b": 2, "c": 3}, {"a": 1, "b": 2, "c": 3}, {"a": 1, "b": 2, "c": 3}]); + + record {|int c; string...;|}[]|csv:Error cn2 = csv:parseString(csvValue2, {header: 1}); + test:assertEquals(cn2, [{"a": "1", "b": "2, 3", "c": 3}, {"a": "1", "b": "2, 3", "c": 3}, {"a": "4", "b": "5", "c": 6}]); + + record {|string b; string c;|}[]|csv:Error cn3 = csv:parseString(csvValue3, {}); + test:assertEquals(cn3, [{"b": "\"2\"", "c": "3"}, {"b": "5, 6\"b\" \" a \"", c: "\"6\""}]); + + record {}[]|csv:Error cn4 = csv:parseString(csvValue4, {textEnclosure: "'"}); + test:assertEquals(cn4, [{"a": 1, "b": 2, "c": 3}, {"a": 4, b: "5, '6'7", c: 8}, {a: 4, b: "\"5\"", c: "4, '5\"a\", ,\",\" a '6'7"}]); + + anydata[][]|csv:Error cn4_2 = csv:parseString(csvValue4, {textEnclosure: "'"}); + test:assertEquals(cn4_2, [[1, 2, 3], [4, "5, '6'7", 8], [4, "\"5\"", "4, '5\"a\", ,\",\" a '6'7"]]); + + record {}[]|csv:Error cn5 = csv:parseString(csvValue5, {}); + test:assertEquals(cn5, [{a: 1, b: 2, c: 3}, {a: 1, b: 2, c: 3}, {a: 1, b: 2, c: 3}, {a: 1, b: 2, c: 3}]); + + record {}[]|csv:Error cn6 = csv:parseString(csvValue6, {}); + test:assertTrue(cn6 is csv:Error); + test:assertEquals((cn6).message(), "invalid number of headers"); + + cn6 = csv:parseString(string `a,b,c,d + 1,1,1,1,1`, {}); + test:assertTrue(cn6 is csv:Error); + test:assertEquals((cn6).message(), "invalid number of headers"); +} + +@test:Config +function testHeaderQuotesWithParserOptions() { + string csvValue1 = string ` + "a", b, c + 1, "2", "3" + "1", 2, 3 + 1, "2", 3 + + "1", "2", "3"`; + + string csvValue2 = string ` + "a, b, c", "b,c", "c,d" + 1, "2, 3", 3 + 1, "2, 3",3 + 4, 5, 6 + `; + + string csvValue3 = string `'a '1'a5,6', 'b", " ","""', c + 1, '2', 3 + 4, '5, '6'7', 8 + 4, "5", '4, '5"a", ,"," a '6'7'`; + + record {}[]|csv:Error cn = csv:parseString(csvValue1, {header: 1}); + test:assertEquals(cn, [{"a": 1, "b": 2, "c": 3}, {"a": 1, "b": 2, "c": 3}, {"a": 1, "b": 2, "c": 3}, {"a": 1, "b": 2, "c": 3}]); + + record {}[]|csv:Error cn2 = csv:parseString(csvValue2, {header: 1}); + test:assertEquals(cn2, [{"a, b, c": 1, "b,c": "2, 3", "c,d": 3}, {"a, b, c": 1, "b,c": "2, 3", "c,d": 3}, {"a, b, c": 4, "b,c": 5, "c,d": 6}]); + + anydata[][]|csv:Error cn2_2 = csv:parseString(csvValue2, {header: 1}); + test:assertEquals(cn2_2, [[1, "2, 3", 3], [1, "2, 3", 3], [4, 5, 6]]); + + record {}[]|csv:Error cn3 = csv:parseString(csvValue3, {textEnclosure: "'"}); + test:assertEquals(cn3, [{"a '1'a5,6": 1, "b\", \" \",\"\"\"": 2, "c": 3}, {"a '1'a5,6": 4, "b\", \" \",\"\"\"": "5, '6'7", c: 8}, {"a '1'a5,6": 4, "b\", \" \",\"\"\"": "\"5\"", c: "4, '5\"a\", ,\",\" a '6'7"}]); +} + +@test:Config +function testEscapeCharactersWithParserOptions() { + string csvValue1 = string ` + "a", b, c + 1, "2a\t", "3b\n" + "1c\n", 2, 3 + 1, "2a\"", 3 + + "1a\\", "2b\\"", "3"`; + + string csvValue2 = string ` + "a\"", "\tb\t\n", c + 1, "2a\t", "3b\n" + "1c\n", "/2/", 3 + 1, "2a\"", "3" + + "1a\\", "2b\\"", "3"`; + + string csvValue3 = string ` + "a", b, c + 1, "2\t", "3\n" + "1\n", 2, 3 + 1, "2\"", 3 + + "1\\", "2\\"", "3"`; + + record {}[]|csv:Error cn = csv:parseString(csvValue1, {header: 1}); + test:assertEquals(cn, [{"a": 1, "b": "2a\t", "c": "3b\n"}, {"a": "1c\n", "b": 2, "c": 3}, {"a": 1, "b": "2a\"", "c": 3}, {"a": "1a\\", "b": "2b\\\"", "c": 3}]); + + record {}[]|csv:Error cn2 = csv:parseString(csvValue2, {header: 1}); + test:assertEquals(cn2, [{"a\"": 1, "\tb\t\n": "2a\t", "c": "3b\n"}, {"a\"": "1c\n", "\tb\t\n": "/2/", "c": 3}, {"a\"": 1, "\tb\t\n": "2a\"", "c": 3}, {"a\"": "1a\\", "\tb\t\n": "2b\\\"", "c": 3}]); + + record {}[]|csv:Error cn3 = csv:parseString(csvValue3, {header: 1}); + test:assertEquals(cn3, [{"a": 1, "b": "2\t", "c": "3\n"}, {"a": "1\n", "b": 2, "c": 3}, {"a": 1, "b": "2\"", "c": 3}, {"a": "1\\", "b": "2\\\"", "c": 3}]); + + anydata[][]|csv:Error cn_2 = csv:parseString(csvValue1, {header: 1}); + test:assertEquals(cn_2, [[1, "2a\t", "3b\n"], ["1c\n", 2, 3], [1, "2a\"", 3], ["1a\\", "2b\\\"", 3]]); + + anydata[][]|csv:Error cn2_2 = csv:parseString(csvValue2, {header: 1}); + test:assertEquals(cn2_2, [[1, "2a\t", "3b\n"], ["1c\n", "/2/", 3], [1, "2a\"", 3], ["1a\\", "2b\\\"", 3]]); + + anydata[][]|csv:Error cn3_2 = csv:parseString(csvValue3, {header: 1}); + test:assertEquals(cn3_2, [[1, "2\t", "3\n"], ["1\n", 2, 3], [1, "2\"", 3], ["1\\", "2\\\"", 3]]); +} + +@test:Config +function testDelimiterWithParserOptions() { + record {}[]|csv:Error cn = csv:parseString(csvStringData7, {header: 1, delimiter: "@"}); + test:assertEquals(cn, [ + {a: 1, b: "string", c: true, d: 2.234, e: -3.21, f: ()}, + {a: 2, b: "s,tring", c: true, d: 2.234, e: -3.21, f: ()}, + {a: 3, b: "stri,ng", c: true, d: 2.234, e: -3.21, f: ()}, + {a: 4, b: "string", c: true, d: 2.234, e: -3.21, f: ()}, + {a: 5, b: "string", c: true, d: 2.234, e: -3.21, f: ()} + ]); + + anydata[][]|csv:Error cn2 = csv:parseString(csvStringData7, {header: 1, delimiter: "@"}); + test:assertEquals(cn2, [ + [1, "string", true, 2.234, -3.21, ()], + [2, "s,tring", true, 2.234, -3.21, ()], + [3, "stri,ng", true, 2.234, -3.21, ()], + [4, "string", true, 2.234, -3.21, ()], + [5, "string", true, 2.234, -3.21, ()] + ]); +} + +@test:Config +function testLineTerminatorWithParserOptions() { + string csvValue = string `a,b + 1,"2\n3"`; + + record {}[]|csv:Error cn = csv:parseString(csvValue, {header: 0, lineTerminator: [csv:CRLF, csv:LF]}); + test:assertEquals(cn, [{a: 1, b: "2\n3"}]); + + cn = csv:parseString(csvValue, {header: 0, lineTerminator: [csv:CRLF, csv:LF]}); + test:assertEquals(cn, [{a: 1, b: "2\n3"}]); + + cn = csv:parseString(csvValue, {header: 0, lineTerminator: [csv:CRLF, csv:LF]}); + test:assertEquals(cn, [{a: 1, b: "2\n3"}]); + + anydata[][]|csv:Error cn2 = csv:parseString(csvValue, {header: 0, lineTerminator: [csv:CRLF, csv:LF]}); + test:assertEquals(cn2, [[1, "2\n3"]]); + + cn2 = csv:parseString(csvValue, {header: 0, lineTerminator: [csv:CRLF, csv:LF]}); + test:assertEquals(cn2, [[1, "2\n3"]]); + + cn2 = csv:parseString(csvValue, {header: 0, lineTerminator: [csv:CRLF, csv:LF]}); + test:assertEquals(cn2, [[1, "2\n3"]]); + + cn2 = csv:parseString(csvValue, {header: 0, lineTerminator: csv:LF}); + test:assertEquals(cn2, [[1, "2\n3"]]); +} + +@test:Config +function testSkipLineParameterWithOutputHeaderConfig() { + var csv1 = [{a: 1, b: 2}, {a: 2, b: 3}, {a: 3, b: 4}, {a: 4, b: 5}]; + string[][] csv2 = [["1", "2"], ["2", "3"], ["3", "4"], ["4", "5"]]; + var csv3 = string `a,b + 1,2 + 2,3 + 3,4 + 4,5`; + + record {}[]|csv:Error result = csv:parseString(csv3, {skipLines: "2-3"}); + test:assertEquals(result, [{a: 1, b: 2}, {a: 4, b: 5}]); + + anydata[][]|csv:Error result2 = csv:parseString(csv3, {outputWithHeaders: true, skipLines: "2-3"}); + test:assertEquals(result2, [["a", "b"], [1, 2], [4, 5]]); + + result = csv:transform(csv1, {skipLines: "2-3"}); + test:assertEquals(result, [{a: 1, b: 2}, {a: 4, b: 5}]); + + result2 = csv:transform(csv1, {outputWithHeaders: true, skipLines: "2-3"}); + test:assertEquals(result2, [["a", "b"], [1, 2], [4, 5]]); + + result = csv:parseList(csv2, {skipLines: "2-3", customHeaders: ["a", "b"]}); + test:assertEquals(result, [{a: 1, b: 2}, {a: 4, b: 5}]); + + result2 = csv:parseList(csv2, {outputWithHeaders: true, skipLines: "2-3", customHeaders: ["a", "b"]}); + test:assertEquals(result2, [["a", "b"], [1, 2], [4, 5]]); + + result = csv:parseList(csv2, {headerRows: 1, skipLines: "2-3", customHeaders: ["a", "b"]}); + test:assertEquals(result, [{a: 2, b: 3}]); + + result2 = csv:parseList(csv2, {outputWithHeaders: true, headerRows: 1, skipLines: "2-3", customHeaders: ["a", "b"]}); + test:assertEquals(result2, [["a", "b"], [2, 3]]); + + result = csv:parseList(csv2, {headerRows: 2, skipLines: "2-3", customHeaders: ["a", "b"]}); + test:assertEquals(result, [{a: 3, b: 4}]); + + result2 = csv:parseList(csv2, {outputWithHeaders: true, headerRows: 2, skipLines: "2-3", customHeaders: ["a", "b"]}); + test:assertEquals(result2, [["a", "b"], [3, 4]]); + + result = csv:parseList(csv2, {headerRows: 1, skipLines: "2-3"}); + test:assertEquals(result, [{'1: 2, '2: 3}]); + + result2 = csv:parseList(csv2, {outputWithHeaders: true, headerRows: 1, skipLines: "2-3"}); + test:assertEquals(result2, [[1, 2], [2, 3]]); + + result2 = csv:parseList(csv2, {outputWithHeaders: false, headerRows: 1, skipLines: "2-3"}); + test:assertEquals(result2, [[2, 3]]); + + result = csv:parseList(csv2, {headerRows: 2, customHeaders: ["a", "b"], skipLines: "2-3"}); + test:assertEquals(result, [{a: 3, b: 4}]); + + result2 = csv:parseList(csv2, {outputWithHeaders: true, headerRows: 2, customHeaders: ["a", "b"], skipLines: "2-3"}); + test:assertEquals(result2, [["a", "b"], [3, 4]]); +} diff --git a/ballerina-tests/user-config-tests/tests/user_configs.bal b/ballerina-tests/user-config-tests/tests/user_configs.bal new file mode 100644 index 0000000..43312a5 --- /dev/null +++ b/ballerina-tests/user-config-tests/tests/user_configs.bal @@ -0,0 +1,45 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/data.csv; + +// Valid parser options +final csv:ParseOptions option1 = {delimiter: "@", nilValue: "null", lineTerminator: [csv:LF]}; +final csv:ParseOptions option2 = {nilValue: "N/A", lineTerminator: [csv:CRLF, csv:LF], comment: "/"}; +final csv:ParseOptions option3 = {nilValue: "()", header: 1, skipLines: [1, 2]}; +final csv:ParseOptions option4 = {nilValue: "", header: 4, skipLines: "1-5"}; +final csv:ParseOptions option5 = {nilValue: "", header: 4, skipLines: "1-1"}; +final csv:ParseOptions option6 = {nilValue: "()", header: (), skipLines: [1, 2]}; + +final csv:ParseOptions ptOption1 = {nilValue: "", header: 1, skipLines: [2, 4]}; +final csv:ParseOptions ptOption2 = {nilValue: "", header: 1, skipLines: "2-4"}; +final csv:ParseOptions ptOption3 = {nilValue: "", header: 4, skipLines: "1-5"}; +final csv:ParseOptions ptOption4 = {nilValue: "", header: 4, skipLines: [-1, -2, 4, 2]}; +final csv:ParseOptions ptOption5 = {header: (), skipLines: [-1, -2, 5, 3]}; + +// Invalid parser options +final csv:ParseOptions invalidParserOptions1 = {header: 4}; +final csv:ParseOptions invalidParserOptions2 = {comment: "$"}; +final csv:ParseOptions invalidParserOptions3 = {lineTerminator: csv:CRLF}; +final csv:ParseOptions invalidParserOptions4 = {skipLines: [1000, 1001]}; +final csv:ParseOptions invalidParserOptions5 = {skipLines: "a-b"}; +final csv:ParseOptions invalidParserOptions6 = {skipLines: "3-1"}; +final csv:ParseOptions invalidParserOptions7 = {skipLines: "a-5"}; +final csv:ParseOptions invalidParserOptions8 = {skipLines: "6-a"}; +final csv:ParseOptions invalidParserOptions9 = {skipLines: "a-5"}; +final csv:ParseOptions invalidParserOptions10 = {skipLines: "-1-6"}; +final csv:ParseOptions invalidParserOptions11 = {nilValue: "", header: 4, skipLines: "0-10"}; +final csv:ParseOptions invalidParserOptions12 = {skipLines: [1, 3, 4, -1]}; diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index ecae9dc..07bed5f 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -8,12 +8,19 @@ repository = "https://github.com/ballerina-platform/module-ballerina-data.csv" license = ["Apache-2.0"] distribution = "2201.9.0" export = ["data.csv"] +readme = "Package.md" [platform.java17] graalvmCompatible = true [[platform.java17.dependency]] -groupId = "io.ballerina.stdlib" +groupId = "io.ballerina.lib" artifactId = "data.csv-native" version = "0.1.0" path = "../native/build/libs/data.csv-native-0.1.0-SNAPSHOT.jar" + +[[platform.java17.dependency]] +groupId = "io.ballerina.stdlib" +artifactId = "constraint-native" +version = "1.5.0" +path = "./lib/constraint-native-1.5.0.jar" diff --git a/ballerina/CompilerPlugin.toml b/ballerina/CompilerPlugin.toml index e503c45..e7c869f 100644 --- a/ballerina/CompilerPlugin.toml +++ b/ballerina/CompilerPlugin.toml @@ -1,6 +1,6 @@ [plugin] -id = "constraint-compiler-plugin" -class = "io.ballerina.stdlib.data.csvdata.compiler.CsvDataCompilerPlugin" +id = "data.csv-compiler-plugin" +class = "io.ballerina.lib.data.csvdata.compiler.CsvDataCompilerPlugin" [[dependency]] path = "../compiler-plugin/build/libs/data.csv-compiler-plugin-0.1.0-SNAPSHOT.jar" diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index c07b540..0bfd723 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -5,7 +5,7 @@ [ballerina] dependencies-toml-version = "2" -distribution-version = "2201.9.0" +distribution-version = "2201.10.0-20240801-104200-87df251c" [[package]] org = "ballerina" diff --git a/ballerina/Package.md b/ballerina/Package.md index 7da4a1a..79dcd39 100644 --- a/ballerina/Package.md +++ b/ballerina/Package.md @@ -1 +1,157 @@ -# Ballerina CSV Data module \ No newline at end of file +# Ballerina CSV Data Library + +The Ballerina CSV Data Library is a comprehensive toolkit designed to facilitate the handling and manipulation of CSV data within Ballerina applications. It streamlines the process of converting CSV data to native Ballerina data types, enabling developers to work with CSV content seamlessly and efficiently. + +## Features + +- **Versatile CSV Data Input**: Accept CSV data as a string, byte array, or a stream and convert it into a subtype of ballerina records or lists. +- **CSV to anydata Value transformation**: Transform CSV data into expected type which is subtype of ballerina record arrays or anydata arrays. +- **Projection Support**: Perform selective conversion of CSV data subsets into ballerina record array or anydata array values through projection. + +## Usage + +### Converting CSV string to a record array + +To convert a CSV string into a record array value, you can use the `parseString` function from the library. The following example demonstrates how to transform a CSV document into an array of records. + +```ballerina +import ballerina/data.csv; +import ballerina/io; + +type Book record { + string name; + string author; + int year; +}; + +public function main() returns error? { + string csvString = string `name,author,year + Clean Code,Robert C. Martin,2008 + The Pragmatic Programmer,Andrew Hunt and David Thomas,1999`; + + Book[] books = check csv:parseString(csvString); + foreach var book in books { + io:println(book); + } +} +``` + +### Converting external CSV document to a record value + +For transforming CSV content from an external source into a record value, the `parseString`, `parseBytes` and `parseStream` functions can be used. This external source can be in the form of a string or a byte array/byte block stream that houses the CSV data. This is commonly extracted from files or network sockets. The example below demonstrates the conversion of an CSV value from an external source into a record value. + +```ballerina +import ballerina/data.csv; +import ballerina/io; + +type Book record { + string name; + string author; + int year; +}; + +public function main() returns error? { + // Read the CSV content as a string + string csvContent = check io:fileReadString("path/to/file.csv"); + Book[] book = check csv:parseString(csvContent); + io:println(book); + + // Read the CSV content as a stream + stream csvStream = check io:fileReadBlocksAsStream("path/to/file.csv"); + Book[] book2 = check csv:parseStream(csvStream); + io:println(book2); +} +``` + +Make sure to handle possible errors that may arise during the file reading or CSV to record/array conversion process. The `check` keyword is utilized to handle these errors, but more sophisticated error handling can be implemented as per your requirements. + +## CSV to record array/anydata array of array representation + +The CSV Object can be represented as a value of type `record/map array` or `string array of array` in Ballerina, which facilitates a structured and type-safe approach to handling CSV data. +The conversion of CSV data to subtype of `record array` or `anydata array of array` representation is a fundamental feature of the library. + +```ballerina +import ballerina/data.csv; +import ballerina/io; + +type Book record { + string name; + int year; +}; + +public function main() returns error? { + string[][] bookArray = [["Clean Code","2008"],["Clean Architecture","2017"]]; + Book[] bookRecords = [{name: "Clean Code", year: 2008}, {name: "Clean Architecture", year: 2017}]; + + // Parse and output a record array from a source of string array of arrays. + Book[] books = check csv:parseList(bookArray, {customHeaders: ["name", "year"]}); + io:println(books); + + // Parse and output a tuple array from a source of string array of arrays. + [string, int][] books2 = check csv:parseList(bookArray, {customHeaders: ["name", "year"]}); + io:println(books2); + + // Transform CSV records to a string array of arrays. + [string, int][] books3 = check csv:transform(bookRecords); + io:println(books3); +} +``` + +### Controlling the CSV value to record array conversion + +The library allows for selective conversion of CSV into closed record arrays. This is beneficial when the CSV data contains headers that are not necessary to be transformed into record fields. + +```ballerina +import ballerina/data.csv; +import ballerina/io; + +type Book record {| + string name; + string author; +|}; + +public function main() returns error? { + record {}[] csvContent = [{ + "name": "Clean Code", + "author": "Robert C. Martin", + "year": 2008, + "publisher": "Prentice Hall" + }, { + "name": "The Pragmatic Programmer", + "author": "Andrew Hunt and David Thomas", + "year": 1999, + "publisher": "Addison-Wesley" + }]; + + // The CSV data above contains publisher and year fields which are not + // required to be converted into a record field. + Book[] book = check csv:transform(csvContent); + io:println(book); +} +``` + +However, if the rest field is utilized (or if the record type is defined as an open record), all members in the CSV data will be transformed into record fields: + +```ballerina +type Book record { + string name; + string author; +} +``` + +In this instance, all other CSV header values, such as `year` and `publisher` will be transformed into `anydata-typed` fields with the corresponding CSV header as the key-value pair. + +This behavior extends to arrays as well. + +The process of projecting CSV data into a record supports various use cases, including the filtering out of unnecessary members. This functionality is anticipated to be enhanced in the future to accommodate more complex scenarios, such as filtering values based on regular expressions, among others. + +## Issues and projects + +Issues and Projects tabs are disabled for this repository as this is part of the Ballerina library. To report bugs, request new features, start new discussions, view project boards, etc. please visit Ballerina library [parent repository](https://github.com/ballerina-platform/ballerina-library). + +This repository only contains the source code for the package. + +## Useful links + +* Chat live with us via our [Discord server](https://discord.gg/ballerinalang). +* Post all technical questions on Stack Overflow with the [#ballerina](https://stackoverflow.com/questions/tagged/ballerina) tag. diff --git a/ballerina/build.gradle b/ballerina/build.gradle index 4c5f9c3..7cac14a 100644 --- a/ballerina/build.gradle +++ b/ballerina/build.gradle @@ -60,7 +60,7 @@ def stripBallerinaExtensionVersion(String extVersion) { apply plugin: 'io.ballerina.plugin' ballerina { - testCoverageParam = "--code-coverage --coverage-format=xml --includes=io.ballerina.stdlib.data.*:ballerina.*" + testCoverageParam = "--code-coverage --coverage-format=xml --includes=io.ballerina.lib.data.csvdata.*:ballerina.*" packageOrganization = packageOrg module = packageName langVersion = ballerinaLangVersion @@ -82,6 +82,8 @@ task updateTomlFiles { def stdlibDependentConstraintVersion = stripBallerinaExtensionVersion("${stdlibDependentConstraintNativeVersion}") def newConfig = ballerinaTomlFilePlaceHolder.text.replace("@project.version@", project.version) newConfig = newConfig.replace("@toml.version@", tomlVersion) + newConfig = newConfig.replace("@stdlib.constraintnative.version@", stdlibDependentConstraintNativeVersion) + newConfig = newConfig.replace("@constraint.version@", stdlibDependentConstraintVersion) ballerinaTomlFile.text = newConfig def newCompilerPluginToml = compilerPluginTomlFilePlaceHolder.text.replace("@project.version@", project.version) @@ -127,16 +129,15 @@ task deleteDependencyTomlFiles { } } - updateTomlFiles.dependsOn copyStdlibs build.dependsOn "generatePomFileForMavenPublication" -build.dependsOn ":${packageName}-native:build" build.dependsOn deleteDependencyTomlFiles +build.dependsOn ":${packageName}-native:build" build.dependsOn ":${packageName}-compiler-plugin:build" -build.dependsOn deleteDependencyTomlFiles -test.dependsOn ":${packageName}-native:build" +test.dependsOn "generatePomFileForMavenPublication" +test.dependsOn deleteDependencyTomlFiles test.dependsOn ":${packageName}-native:build" test.dependsOn ":${packageName}-compiler-plugin:build" diff --git a/ballerina/csv_api.bal b/ballerina/csv_api.bal index 7355b09..e39b2ce 100644 --- a/ballerina/csv_api.bal +++ b/ballerina/csv_api.bal @@ -16,99 +16,96 @@ import ballerina/jballerina.java; -# Converts CSV string to subtype of record array. +# Parse a CSV string as a subtype of `record {}[]` or `anydata[][]`. +# +# ```ballerina +# string csvString = string `id,name +# 1,John +# 3,Jane`; +# record {int id; string name;}[] csv1 = check csv:parseString(csvString); +# [int, string][] csv2 = check csv:parseString(csvString); +# record {|int id;|}[] csv3 = check csv:parseString(csvString); +# record {int id;}[] csv4 = check csv:parseString(csvString, {skipLines: [1]}); +# ``` # -# + s - Source CSV string value +# + csvString - Source CSV string value # + options - Options to be used for filtering in the projection # + t - Target type # + return - On success, value belonging to the given target type, else returns an `csv:Error` value. -public isolated function parseStringToRecord(string s, parseToRecordOption options = {}, typedesc t = <>) - returns t|Error = @java:Method {'class: "io.ballerina.stdlib.data.csvdata.csv.Native"} external; +public isolated function parseString(string csvString, ParseOptions options = {}, typedesc t = <>) + returns t|Error = @java:Method {'class: "io.ballerina.lib.data.csvdata.csv.Native"} external; -# Converts byte[] to subtype of record array. +# Parse a byte[] as a subtype of `record {}[]` or `anydata[][]`. +# +# ```ballerina +# byte[] csvBytes = check io:fileReadBytes("example.csv"); +# +# record {int id; string name;}[] csv1 = check csv:parseBytes(csvBytes); +# [int, string][] csv2 = check csv:parseBytes(csvBytes); +# record {|int id;|}[] csv3 = check csv:parseBytes(csvBytes); +# record {int id;}[] csv4 = check csv:parseBytes(csvBytes, {skipLines: [1]}); +# ``` # -# + s - Source CSV byte array +# + csvBytes - Source CSV byte array # + options - Options to be used for filtering in the projection # + t - Target type # + return - On success, value belonging to the given target type, else returns an `csv:Error` value. -public isolated function parseBytesToRecord(byte[] s, parseToRecordOption options = {}, typedesc t = <>) - returns t|Error = @java:Method {'class: "io.ballerina.stdlib.data.csvdata.csv.Native"} external; +public isolated function parseBytes(byte[] csvBytes, ParseOptions options = {}, typedesc t = <>) + returns t|Error = @java:Method {'class: "io.ballerina.lib.data.csvdata.csv.Native"} external; -# Converts CSV byte-block-stream to subtype of record array. -# -# + s - Source CSV byte-block-stream -# + options - Options to be used for filtering in the projection -# + t - Target type -# + return - On success, value belonging to the given target type, else returns an `csv:Error` value. -public isolated function parseStreamToRecord(stream s, - parseToRecordOption options = {}, typedesc t = <>) - returns t|Error = @java:Method {'class: "io.ballerina.stdlib.data.csvdata.csv.Native"} external; - -# Converts CSV string to subtype of anydata[][]. -# -# + s - Source CSV string value -# + options - Options to be used for filtering in the projection -# + t - Target type -# + return - On success, value belonging to the given target type, else returns an `csv:Error` value. -public isolated function parseStringToList(string s, ParseOption options = {}, typedesc t = <>) - returns t|Error = @java:Method {'class: "io.ballerina.stdlib.data.csvdata.csv.Native"} external; - -# Converts byte[] to subtype of anydata[][]. -# -# + s - Source CSV byte array +# Parse a CSV byte block stream as a subtype of `record {}[]` or `anydata[][]`. +# +# ```ballerina +# stream csvByteStream = check io:fileReadBlocksAsStream("example.csv"); +# record {int id; string name;}[] csv1 = check csv:parseStream(csvByteStream); +# +# stream csvByteStream2 = check io:fileReadBlocksAsStream("example.csv"); +# [int, string][] csv2 = check csv:parseStream(csvByteStream2); +# +# stream csvByteStream3 = check io:fileReadBlocksAsStream("example.csv"); +# record {|int id;|}[] csv3 = check csv:parseStream(csvByteStream3); +# +# stream csvByteStream4 = check io:fileReadBlocksAsStream("example.csv"); +# record {int id;}[] csv4 = check csv:parseStream(csvByteStream4, {skipLines: [1]}); +# ``` +# +# + csvByteStream - Source CSV byte block stream # + options - Options to be used for filtering in the projection # + t - Target type # + return - On success, value belonging to the given target type, else returns an `csv:Error` value. -public isolated function parseBytesToList(byte[] s, ParseOption options = {}, typedesc t = <>) - returns t|Error = @java:Method {'class: "io.ballerina.stdlib.data.csvdata.csv.Native"} external; +public isolated function parseStream(stream csvByteStream, + ParseOptions options = {}, typedesc t = <>) + returns t|Error = @java:Method {'class: "io.ballerina.lib.data.csvdata.csv.Native"} external; -# Converts CSV byte-block-stream to subtype of anydata[][]. +# Transform value of type record {}[] to subtype of `record {}[]` or `anydata[][]`. +# +# ```ballerina +# record {int id; string name;}[] csvRecords = [{id: 1, name: "John"}, {id: 2, name: "Jane"}]; +# [int, string][] csv1 = check csv:transform(csvRecords); +# record {|int id;|}[] csv2 = check csv:transform(csvRecords); +# record {int id;}[] csv3 = check csv:transform(csvRecords, {skipLines: [1]}); +# ``` # -# + s - Source CSV byte-block-stream -# + options - Options to be used for filtering in the projection -# + t - Target type -# + return - On success, value belonging to the given target type, else returns an `csv:Error` value. -public isolated function parseStreamToList(stream s, - ParseOption options = {}, typedesc t = <>) - returns t|Error = @java:Method {'class: "io.ballerina.stdlib.data.csvdata.csv.Native"} external; - -# Convert value of type record{}[] to subtype of record{}[]. -# -# + s - Source Ballerina record array value -# + options - Options to be used for filtering in the projection -# + t - Target type -# + return - On success, returns value belonging to the given target type, else returns an `csv:Error` value. -public isolated function parseRecordAsRecordType(record{}[] s, - RecordAsRecordOption options = {}, typedesc t = <>) - returns t|Error = @java:Method {'class: "io.ballerina.stdlib.data.csvdata.csv.Native"} external; - -# Convert value of type record{}[] to subtype of anydata[][]. -# -# + s - Source Ballerina record array value -# + headerNames - The order of the header names in the source +# + csvRecords - Source Ballerina record array value # + options - Options to be used for filtering in the projection # + t - Target type # + return - On success, returns value belonging to the given target type, else returns an `csv:Error` value. -public isolated function parseRecordAsListType(record{}[] s, string[] headerNames, - Options options = {}, typedesc t = <>) - returns t|Error = @java:Method {'class: "io.ballerina.stdlib.data.csvdata.csv.Native"} external; +public isolated function transform(record {}[] csvRecords, + TransformOptions options = {}, typedesc t = <>) + returns t|Error = @java:Method {'class: "io.ballerina.lib.data.csvdata.csv.Native"} external; -# Convert value of type string[][] to subtype of record{}[]. -# -# + s - Source Ballerina string array of array value -# + customHeaders - The order of the header names in the source -# + options - Options to be used for filtering in the projection -# + t - Target type -# + return - On success, returns value belonging to the given target type, else returns an `csv:Error` value. -public isolated function parseListAsRecordType(string[][] s, string[]? customHeaders = (), - ListAsRecordOption options = {}, typedesc t = <>) - returns t|Error = @java:Method {'class: "io.ballerina.stdlib.data.csvdata.csv.Native"} external; - -# Convert value of type string[][] to subtype of anydata[][]. -# -# + s - Source Ballerina string array of array value +# Parse a string array of array as a subtype of `record {}[]` or `anydata[][]`. +# +# ```ballerina +# string[][] csvList = [["1", "John"], ["2", "Jane"]]; +# [int, string][] csv1 = check csv:parseList(csvList); +# record {|int id;|}[] csv2 = check csv:parseList(csvList, {customHeaders: ["id", "name"]}); +# record {int id;}[] csv3 = check csv:parseList(csvList, {skipLines: [1], customHeaders: ["id", "name"]}); +# ``` +# +# + csvList - Source Ballerina string array of array value # + options - Options to be used for filtering in the projection # + t - Target type # + return - On success, returns value belonging to the given target type, else returns an `csv:Error` value. -public isolated function parseListAsListType(string[][] s, ListAsListOption options = {}, typedesc t = <>) - returns t|Error = @java:Method {'class: "io.ballerina.stdlib.data.csvdata.csv.Native"} external; +public isolated function parseList(string[][] csvList, ParseListOptions options = {}, typedesc t = <>) + returns t|Error = @java:Method {'class: "io.ballerina.lib.data.csvdata.csv.Native"} external; diff --git a/ballerina/init.bal b/ballerina/init.bal index a0a98cb..02fa1d9 100644 --- a/ballerina/init.bal +++ b/ballerina/init.bal @@ -21,5 +21,5 @@ isolated function init() { } isolated function setModule() = @java:Method { - 'class: "io.ballerina.stdlib.data.csvdata.utils.ModuleUtils" + 'class: "io.ballerina.lib.data.csvdata.utils.ModuleUtils" } external; diff --git a/ballerina/types.bal b/ballerina/types.bal index 11b0c38..c1f1095 100644 --- a/ballerina/types.bal +++ b/ballerina/types.bal @@ -1,3 +1,20 @@ +// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +# Represents an error. public type Error error; # Defines the name of the JSON Object key. @@ -8,66 +25,90 @@ public type NameConfig record {| |}; # The annotation is used to overwrite the existing record field name. -public const annotation NameConfig Name on record field; +public const annotation NameConfig Name on record field; +# Represents options for data projection. public type Options record { - record { - # If `true`, nil values will be considered as optional fields in the projection. - boolean nilAsOptionalField = false; // () assign to op - # If `true`, absent fields will be considered as nilable types in the projection. - boolean absentAsNilableType = false; // source haven't () && expected type contains op => () - }|false allowDataProjection = {}; - int[]|string skipLines = []; + # Allows data projection with specific settings. + # + # This configuration can be either a record or false. + # If it is a record, it contains `nilAsOptionalField` and `absentAsNilableType` options. + # If it is set to `false`, data projection is not allowed. + record { + # If `true`, nil values will be considered as optional fields in the projection. + boolean nilAsOptionalField = false; + # If `true`, absent fields will be considered as nilable types in the projection. + boolean absentAsNilableType = false; + }|false allowDataProjection = {}; + # Lines to skip during processing, specified either as an array of integers or a string. + int[]|string skipLines = []; + # If `true`, enables validation of constraints during processing. + boolean enableConstraintValidation = true; + # If `true`, when the result is a list it will contain headers as the first row. + boolean outputWithHeaders = false; }; -public type ParseOption record {| - *Options; - string:Char delimiter = ","; - string encoding = "UTF-8"; - string locale = "en_US"; -// TODO: Add " for Strings" - string:Char textEnclosure = "\""; - string:Char escapeChar = "\\"; - LineTerminator|LineTerminator[] lineTerminator = [CR, LF, CRLF]; - NilValue? nilValue = (); - // string commentStartingSequence = "#"; - string:Char comment = "#"; - false|int:Unsigned32 header = 0; -|}; - -public type parseToRecordOption record {| - *ParseOption; - - // if header = false and this value is null, Then compiler time error. - string[]? customHeaders = (); - boolean enableConstraintValidation = true; -|}; - -public type ListAsListOption record {| +# Represents the options for parsing data. +public type ParseOptions record {| *Options; - boolean stringConversion = true; + # The delimiter character used for separating fields in the data. + string:Char delimiter = ","; + # The character encoding of the data. + string encoding = "UTF-8"; + # The locale used for parsing. + string locale = "en_US"; + # The character used to enclose text fields. + string:Char textEnclosure = "\""; + # The character used for escaping. + string:Char escapeChar = "\\"; + # The line terminator(s) used in the data. + LineTerminator|LineTerminator[] lineTerminator = [LF, CRLF]; + # The value to represent nil. + NilValue? nilValue = (); + # The character used to indicate comments in the data. + string:Char comment = "#"; + # Specifies whether the header is present and, if so, the number of header lines. + int:Unsigned32? header = 0; + # Custom headers for the data, if headers are absent. + string[]? customHeadersIfHeadersAbsent = (); |}; -public type RecordAsRecordOption record {| +# Represents options for treating a list as a record. +public type ParseListOptions record {| *Options; - boolean enableConstraintValidation = true; + # If `0`, all the source data will treat as data rows. + # Otherwise specify the header rows(Starts from 1) in the source data. + int:Unsigned32 headerRows = 0; + # Specify the header names of the source data. + # This field will overwrite the header values in the header rows. + # This will be mandatory if the header row parameter is larger than one. + string[] customHeaders?; |}; -public type ListAsRecordOption record {| +# Represents options for treating a list as a record. +public type TransformOptions record {| *Options; - boolean enableConstraintValidation = true; - boolean stringConversion = true; + # Specify the order of the headers in the source data. + # If the expected type is a subset of `record {}[]` this parameter will be ignored. + string[]? headerOrder = (); |}; +# Enum representing possible line terminators. public enum LineTerminator { - CR = "\r", - LF = "\n", - CRLF = "\r\n" + # Line Feed (LF) line terminator: `\n` + LF = "\n", + # Carriage Return and Line Feed (CRLF) line terminator: `\r\n` + CRLF = "\r\n" }; +# Enum representing possible nil values. public enum NilValue { - NULL = "null", - EMPTY_STRING = "", - NOT_APPLICABLE = "N/A", - BAL_NULL = "()" + # Represents a nil value as the string "null". + NULL = "null", + # Represents a nil value as "N/A". + NOT_APPLICABLE = "N/A", + # Represents an empty string as a nil value. + EMPTY_STRING = "", + # Represents a nil value as Ballerina nil value `()`. + NIL = "()" }; diff --git a/build-config/checkstyle/build.gradle b/build-config/checkstyle/build.gradle index 857ad1e..62780e5 100644 --- a/build-config/checkstyle/build.gradle +++ b/build-config/checkstyle/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/build-config/resources/Ballerina.toml b/build-config/resources/Ballerina.toml index 3db6b47..d846308 100644 --- a/build-config/resources/Ballerina.toml +++ b/build-config/resources/Ballerina.toml @@ -8,12 +8,19 @@ repository = "https://github.com/ballerina-platform/module-ballerina-data.csv" license = ["Apache-2.0"] distribution = "2201.9.0" export = ["data.csv"] +readme = "Package.md" [platform.java17] graalvmCompatible = true [[platform.java17.dependency]] -groupId = "io.ballerina.stdlib" +groupId = "io.ballerina.lib" artifactId = "data.csv-native" version = "@toml.version@" path = "../native/build/libs/data.csv-native-@project.version@.jar" + +[[platform.java17.dependency]] +groupId = "io.ballerina.stdlib" +artifactId = "constraint-native" +version = "@constraint.version@" +path = "./lib/constraint-native-@stdlib.constraintnative.version@.jar" diff --git a/build-config/resources/BallerinaTest.toml b/build-config/resources/BallerinaTest.toml new file mode 100644 index 0000000..5b44930 --- /dev/null +++ b/build-config/resources/BallerinaTest.toml @@ -0,0 +1,16 @@ +[package] +org = "ballerina" +name = "@package.name@" +version = "@toml.version@" + +[[dependency]] +org = "ballerina" +name = "@test.common@" +repository = "local" +version = "@toml.version@" + +[platform.java17] +graalvmCompatible = true + +[build-options] +graalvmBuildOptions = "-H:+IncludeAllLocales" diff --git a/build-config/resources/CompilerPlugin.toml b/build-config/resources/CompilerPlugin.toml index 10f4b4a..e814d75 100644 --- a/build-config/resources/CompilerPlugin.toml +++ b/build-config/resources/CompilerPlugin.toml @@ -1,6 +1,6 @@ [plugin] -id = "constraint-compiler-plugin" -class = "io.ballerina.stdlib.data.csvdata.compiler.CsvDataCompilerPlugin" +id = "data.csv-compiler-plugin" +class = "io.ballerina.lib.data.csvdata.compiler.CsvDataCompilerPlugin" [[dependency]] path = "../compiler-plugin/build/libs/data.csv-compiler-plugin-@project.version@.jar" diff --git a/build-config/resources/CsvTestCommon.toml b/build-config/resources/CsvTestCommon.toml new file mode 100644 index 0000000..38b069a --- /dev/null +++ b/build-config/resources/CsvTestCommon.toml @@ -0,0 +1,7 @@ +[package] +org = "ballerina" +name = "@package.name@" +version = "@toml.version@" + +[platform.java17] +graalvmCompatible = true diff --git a/build.gradle b/build.gradle index dc3d177..44c60de 100644 --- a/build.gradle +++ b/build.gradle @@ -45,7 +45,7 @@ allprojects { } maven { - url = 'https://maven.pkg.github.com/ballerina-platform/ballerina-lang' + url = 'https://maven.pkg.github.com/ballerina-platform/*' credentials { username System.getenv("packageUser") password System.getenv("packagePAT") @@ -63,9 +63,13 @@ subprojects { configurations { ballerinaStdLibs + jbalTools } dependencies { + jbalTools ("org.ballerinalang:jballerina-tools:${ballerinaLangVersion}") { + transitive = false + } /* Standard libraries */ ballerinaStdLibs "io.ballerina.stdlib:file-ballerina:${stdlibFileVersion}" ballerinaStdLibs "io.ballerina.stdlib:io-ballerina:${stdlibIoVersion}" @@ -92,5 +96,9 @@ release { } task build { - dependsOn('data.csv-ballerina:build') + dependsOn(':data.csv-compiler-plugin:build') + dependsOn(':data.csv-native:build') + dependsOn(':data.csv-ballerina:build') + dependsOn(':data.csv-compiler-plugin-tests:test') + dependsOn(':data.csv-ballerina-tests:test') } diff --git a/codecov.yml b/codecov.yml index f60b970..25b7030 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,3 +1,10 @@ +fixes: + - "ballerina/data.csv/**/::../ballerina/" + +ignore: + - "ballerina-tests" + - "test-utils" + coverage: precision: 2 round: down diff --git a/compiler-plugin-test/build.gradle b/compiler-plugin-test/build.gradle index 6d5c7f2..1ad7000 100644 --- a/compiler-plugin-test/build.gradle +++ b/compiler-plugin-test/build.gradle @@ -105,4 +105,8 @@ jacocoTestReport { sourceSets project(':data.csv-compiler-plugin').sourceSets.main } +build { + dependsOn(test) +} + test.dependsOn ":data.csv-ballerina:build" diff --git a/compiler-plugin-test/src/test/java/io/ballerina/lib/data/csvdata/compiler/CompilerPluginTest.java b/compiler-plugin-test/src/test/java/io/ballerina/lib/data/csvdata/compiler/CompilerPluginTest.java index 69c212b..086355b 100644 --- a/compiler-plugin-test/src/test/java/io/ballerina/lib/data/csvdata/compiler/CompilerPluginTest.java +++ b/compiler-plugin-test/src/test/java/io/ballerina/lib/data/csvdata/compiler/CompilerPluginTest.java @@ -18,9 +18,302 @@ package io.ballerina.lib.data.csvdata.compiler; +import io.ballerina.projects.DiagnosticResult; +import io.ballerina.tools.diagnostics.Diagnostic; +import io.ballerina.tools.diagnostics.DiagnosticSeverity; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.stream.Collectors; + +import static io.ballerina.lib.data.csvdata.compiler.CsvDataDiagnosticCodes.UNSUPPORTED_FIELD_TYPE; +import static io.ballerina.lib.data.csvdata.compiler.CsvDataDiagnosticCodes.UNSUPPORTED_TUPLE_MEMBER_TYPE; + /** * This class includes tests for Ballerina Csv Data compiler plugin. + * + * @since 0.1.0 */ public class CompilerPluginTest { + static final String UNSUPPORTED_TYPE = "unsupported type: type is not supported"; + static final String DUPLICATE_FIELD = "invalid field: duplicate field found"; + static final String UNSUPPORTED_FIELD_TYPE = "Unsupported type in the field: Only basic types " + + "are supported for fields, and other types are not allowed."; + static final String UNSUPPORTED_TUPLE_MEMBER_TYPE = "Unsupported type in the tuple member: " + + "Tuple members can only be basic types, other types are not supported."; + static final String IGNORE_OUTPUT_HEADERS_FOR_RECORD_ARRAY = "The option 'outputWithHeaders' will be not allowed" + + " since the expected type is a subtype record array."; + static final String IGNORE_HEADERS_ORDER_FOR_RECORD_ARRAY = "The option 'headerOrder' will be not allowed" + + " since the expected type is a subtype record array."; + static final String IGNORE_CUSTOM_HEADERS_PARAMETER_WHEN_HEADER_PRESENT = "The option " + + "'customHeadersIfHeadersAbsent' will be not allowed since the header is present."; + static final String CUSTOM_HEADERS_SHOULD_BE_PROVIDED = "customHeaders parameter should be provided since the" + + " headerRows larger than 1."; + + @Test + public void testInvalidExpectedUnionType() { + DiagnosticResult diagnosticResult = + CompilerPluginTestUtils.loadPackage("sample_package_1").getCompilation().diagnosticResult(); + List errorDiagnosticsList = diagnosticResult.diagnostics().stream() + .filter(r -> r.diagnosticInfo().severity().equals(DiagnosticSeverity.ERROR)) + .collect(Collectors.toList()); + Assert.assertEquals(errorDiagnosticsList.size(), 21); + Assert.assertEquals(errorDiagnosticsList.get(0).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(1).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(2).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(3).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(4).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(5).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(6).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(7).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(8).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(9).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(10).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(11).diagnosticInfo().messageFormat(), UNSUPPORTED_FIELD_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(12).diagnosticInfo().messageFormat(), UNSUPPORTED_FIELD_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(13).diagnosticInfo().messageFormat(), UNSUPPORTED_FIELD_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(14).diagnosticInfo().messageFormat(), UNSUPPORTED_FIELD_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(15).diagnosticInfo().messageFormat(), UNSUPPORTED_FIELD_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(16).diagnosticInfo().messageFormat(), UNSUPPORTED_FIELD_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(17). + diagnosticInfo().messageFormat(), UNSUPPORTED_TUPLE_MEMBER_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(18). + diagnosticInfo().messageFormat(), UNSUPPORTED_TUPLE_MEMBER_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(19). + diagnosticInfo().messageFormat(), UNSUPPORTED_TUPLE_MEMBER_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(20). + diagnosticInfo().messageFormat(), UNSUPPORTED_TUPLE_MEMBER_TYPE); + } + + @Test + public void testInvalidRecordFields() { + DiagnosticResult diagnosticResult = + CompilerPluginTestUtils.loadPackage("sample_package_2").getCompilation().diagnosticResult(); + List errorDiagnosticsList = diagnosticResult.diagnostics().stream() + .filter(r -> r.diagnosticInfo().severity().equals(DiagnosticSeverity.ERROR)) + .collect(Collectors.toList()); + Assert.assertEquals(errorDiagnosticsList.size(), 4); + Assert.assertEquals(errorDiagnosticsList.get(0).diagnosticInfo().messageFormat(), DUPLICATE_FIELD); + Assert.assertEquals(errorDiagnosticsList.get(1).diagnosticInfo().messageFormat(), DUPLICATE_FIELD); + Assert.assertEquals(errorDiagnosticsList.get(2).diagnosticInfo().messageFormat(), DUPLICATE_FIELD); + Assert.assertEquals(errorDiagnosticsList.get(3).diagnosticInfo().messageFormat(), DUPLICATE_FIELD); + } + + @Test + public void testInvalidProgram() { + DiagnosticResult diagnosticResult = + CompilerPluginTestUtils.loadPackage("sample_package_3").getCompilation().diagnosticResult(); + List errorDiagnosticsList = diagnosticResult.diagnostics().stream() + .filter(r -> r.diagnosticInfo().severity().equals(DiagnosticSeverity.ERROR)) + .collect(Collectors.toList()); + Assert.assertEquals(errorDiagnosticsList.size(), 1); + } + + @Test + public void testModuleLevelInvalidExpectedUnionType() { + DiagnosticResult diagnosticResult = + CompilerPluginTestUtils.loadPackage("sample_package_4").getCompilation().diagnosticResult(); + List errorDiagnosticsList = diagnosticResult.diagnostics().stream() + .filter(r -> r.diagnosticInfo().severity().equals(DiagnosticSeverity.ERROR)) + .collect(Collectors.toList()); + Assert.assertEquals(errorDiagnosticsList.size(), 10); + Assert.assertEquals(errorDiagnosticsList.get(0).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(1).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(2).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(3).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(4).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(5).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(6).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(7).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(8).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(9).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + } + + @Test + public void testEmptyProject() { + DiagnosticResult diagnosticResult = + CompilerPluginTestUtils.loadPackage("sample_package_5").getCompilation().diagnosticResult(); + List errorDiagnosticsList = diagnosticResult.diagnostics().stream() + .filter(r -> r.diagnosticInfo().severity().equals(DiagnosticSeverity.ERROR)) + .collect(Collectors.toList()); + Assert.assertEquals(errorDiagnosticsList.size(), 0); + } + + @Test + public void testInvalidExpectedUnionType2() { + DiagnosticResult diagnosticResult = + CompilerPluginTestUtils.loadPackage("sample_package_6").getCompilation().diagnosticResult(); + List errorDiagnosticsList = diagnosticResult.diagnostics().stream() + .filter(r -> r.diagnosticInfo().severity().equals(DiagnosticSeverity.ERROR)) + .collect(Collectors.toList()); + Assert.assertEquals(errorDiagnosticsList.size(), 10); + Assert.assertEquals(errorDiagnosticsList.get(0).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(1).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(2).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(3).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(4).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(5).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(6).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(7).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(8).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + Assert.assertEquals(errorDiagnosticsList.get(9).diagnosticInfo().messageFormat(), UNSUPPORTED_TYPE); + } + + @Test + public void testIgnoredCustomHeaderIfAbsentOptions() { + DiagnosticResult diagnosticResult = + CompilerPluginTestUtils.loadPackage("sample_package_7").getCompilation().diagnosticResult(); + List errorDiagnosticsList = diagnosticResult.diagnostics().stream() + .filter(r -> r.diagnosticInfo().severity().equals(DiagnosticSeverity.ERROR)) + .collect(Collectors.toList()); + Assert.assertEquals(errorDiagnosticsList.size(), 12); + Assert.assertEquals(errorDiagnosticsList.get(0).diagnosticInfo() + .messageFormat(), IGNORE_CUSTOM_HEADERS_PARAMETER_WHEN_HEADER_PRESENT); + Assert.assertEquals(errorDiagnosticsList.get(1).diagnosticInfo() + .messageFormat(), IGNORE_CUSTOM_HEADERS_PARAMETER_WHEN_HEADER_PRESENT); + Assert.assertEquals(errorDiagnosticsList.get(2).diagnosticInfo() + .messageFormat(), IGNORE_CUSTOM_HEADERS_PARAMETER_WHEN_HEADER_PRESENT); + Assert.assertEquals(errorDiagnosticsList.get(3).diagnosticInfo() + .messageFormat(), IGNORE_CUSTOM_HEADERS_PARAMETER_WHEN_HEADER_PRESENT); + Assert.assertEquals(errorDiagnosticsList.get(4).diagnosticInfo() + .messageFormat(), IGNORE_CUSTOM_HEADERS_PARAMETER_WHEN_HEADER_PRESENT); + Assert.assertEquals(errorDiagnosticsList.get(5).diagnosticInfo() + .messageFormat(), IGNORE_CUSTOM_HEADERS_PARAMETER_WHEN_HEADER_PRESENT); + Assert.assertEquals(errorDiagnosticsList.get(6).diagnosticInfo() + .messageFormat(), IGNORE_CUSTOM_HEADERS_PARAMETER_WHEN_HEADER_PRESENT); + Assert.assertEquals(errorDiagnosticsList.get(7).diagnosticInfo() + .messageFormat(), IGNORE_CUSTOM_HEADERS_PARAMETER_WHEN_HEADER_PRESENT); + Assert.assertEquals(errorDiagnosticsList.get(8).diagnosticInfo() + .messageFormat(), IGNORE_CUSTOM_HEADERS_PARAMETER_WHEN_HEADER_PRESENT); + Assert.assertEquals(errorDiagnosticsList.get(9).diagnosticInfo() + .messageFormat(), IGNORE_CUSTOM_HEADERS_PARAMETER_WHEN_HEADER_PRESENT); + Assert.assertEquals(errorDiagnosticsList.get(10).diagnosticInfo() + .messageFormat(), IGNORE_CUSTOM_HEADERS_PARAMETER_WHEN_HEADER_PRESENT); + Assert.assertEquals(errorDiagnosticsList.get(11).diagnosticInfo() + .messageFormat(), IGNORE_CUSTOM_HEADERS_PARAMETER_WHEN_HEADER_PRESENT); + } + + @Test + public void testIgnoredOutputHeaderOptions() { + DiagnosticResult diagnosticResult = + CompilerPluginTestUtils.loadPackage("sample_package_8").getCompilation().diagnosticResult(); + List errorDiagnosticsList = diagnosticResult.diagnostics().stream() + .filter(r -> r.diagnosticInfo().severity().equals(DiagnosticSeverity.ERROR)) + .collect(Collectors.toList()); + Assert.assertEquals(errorDiagnosticsList.size(), 8); + Assert.assertEquals(errorDiagnosticsList.get(0).diagnosticInfo() + .messageFormat(), IGNORE_OUTPUT_HEADERS_FOR_RECORD_ARRAY); + Assert.assertEquals(errorDiagnosticsList.get(1).diagnosticInfo() + .messageFormat(), IGNORE_OUTPUT_HEADERS_FOR_RECORD_ARRAY); + Assert.assertEquals(errorDiagnosticsList.get(2).diagnosticInfo() + .messageFormat(), IGNORE_CUSTOM_HEADERS_PARAMETER_WHEN_HEADER_PRESENT); + Assert.assertEquals(errorDiagnosticsList.get(3).diagnosticInfo() + .messageFormat(), IGNORE_OUTPUT_HEADERS_FOR_RECORD_ARRAY); + Assert.assertEquals(errorDiagnosticsList.get(4).diagnosticInfo() + .messageFormat(), IGNORE_CUSTOM_HEADERS_PARAMETER_WHEN_HEADER_PRESENT); + Assert.assertEquals(errorDiagnosticsList.get(5).diagnosticInfo() + .messageFormat(), IGNORE_OUTPUT_HEADERS_FOR_RECORD_ARRAY); + Assert.assertEquals(errorDiagnosticsList.get(6).diagnosticInfo() + .messageFormat(), IGNORE_OUTPUT_HEADERS_FOR_RECORD_ARRAY); + Assert.assertEquals(errorDiagnosticsList.get(7).diagnosticInfo() + .messageFormat(), IGNORE_OUTPUT_HEADERS_FOR_RECORD_ARRAY); + } + + @Test + public void testIgnoredheaderOrderOptions() { + DiagnosticResult diagnosticResult = + CompilerPluginTestUtils.loadPackage("sample_package_9").getCompilation().diagnosticResult(); + List errorDiagnosticsList = diagnosticResult.diagnostics().stream() + .filter(r -> r.diagnosticInfo().severity().equals(DiagnosticSeverity.ERROR)) + .collect(Collectors.toList()); + Assert.assertEquals(errorDiagnosticsList.size(), 7); + Assert.assertEquals(errorDiagnosticsList.get(0).diagnosticInfo() + .messageFormat(), IGNORE_HEADERS_ORDER_FOR_RECORD_ARRAY); + Assert.assertEquals(errorDiagnosticsList.get(1).diagnosticInfo() + .messageFormat(), IGNORE_OUTPUT_HEADERS_FOR_RECORD_ARRAY); + Assert.assertEquals(errorDiagnosticsList.get(2).diagnosticInfo() + .messageFormat(), IGNORE_HEADERS_ORDER_FOR_RECORD_ARRAY); + Assert.assertEquals(errorDiagnosticsList.get(3).diagnosticInfo() + .messageFormat(), IGNORE_HEADERS_ORDER_FOR_RECORD_ARRAY); + Assert.assertEquals(errorDiagnosticsList.get(4).diagnosticInfo() + .messageFormat(), IGNORE_OUTPUT_HEADERS_FOR_RECORD_ARRAY); + Assert.assertEquals(errorDiagnosticsList.get(5).diagnosticInfo() + .messageFormat(), IGNORE_HEADERS_ORDER_FOR_RECORD_ARRAY); + Assert.assertEquals(errorDiagnosticsList.get(6).diagnosticInfo() + .messageFormat(), IGNORE_OUTPUT_HEADERS_FOR_RECORD_ARRAY); + } + + @Test + public void testIgnoredCustomHeaderOptions() { + DiagnosticResult diagnosticResult = + CompilerPluginTestUtils.loadPackage("sample_package_10").getCompilation().diagnosticResult(); + List errorDiagnosticsList = diagnosticResult.diagnostics().stream() + .filter(r -> r.diagnosticInfo().severity().equals(DiagnosticSeverity.ERROR)) + .collect(Collectors.toList()); + Assert.assertEquals(errorDiagnosticsList.size(), 12); + Assert.assertEquals(errorDiagnosticsList.get(0).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + Assert.assertEquals(errorDiagnosticsList.get(1).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + Assert.assertEquals(errorDiagnosticsList.get(2).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + Assert.assertEquals(errorDiagnosticsList.get(3).diagnosticInfo() + .messageFormat(), IGNORE_OUTPUT_HEADERS_FOR_RECORD_ARRAY); + Assert.assertEquals(errorDiagnosticsList.get(4).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + Assert.assertEquals(errorDiagnosticsList.get(5).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + Assert.assertEquals(errorDiagnosticsList.get(6).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + Assert.assertEquals(errorDiagnosticsList.get(7).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + Assert.assertEquals(errorDiagnosticsList.get(8).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + Assert.assertEquals(errorDiagnosticsList.get(9).diagnosticInfo() + .messageFormat(), IGNORE_OUTPUT_HEADERS_FOR_RECORD_ARRAY); + Assert.assertEquals(errorDiagnosticsList.get(10).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + Assert.assertEquals(errorDiagnosticsList.get(11).diagnosticInfo() + .messageFormat(), IGNORE_OUTPUT_HEADERS_FOR_RECORD_ARRAY); + } + + @Test + public void testNonCsvFunctionCall() { + DiagnosticResult diagnosticResult = + CompilerPluginTestUtils.loadPackage("sample_package_11").getCompilation().diagnosticResult(); + List errorDiagnosticsList = diagnosticResult.diagnostics().stream() + .filter(r -> r.diagnosticInfo().severity().equals(DiagnosticSeverity.ERROR)) + .collect(Collectors.toList()); + Assert.assertEquals(errorDiagnosticsList.size(), 0); + } + + @Test + public void testIgnoredCustomHeaderOptions2() { + DiagnosticResult diagnosticResult = + CompilerPluginTestUtils.loadPackage("sample_package_12").getCompilation().diagnosticResult(); + List errorDiagnosticsList = diagnosticResult.diagnostics().stream() + .filter(r -> r.diagnosticInfo().severity().equals(DiagnosticSeverity.ERROR)) + .collect(Collectors.toList()); + Assert.assertEquals(errorDiagnosticsList.size(), 9); + Assert.assertEquals(errorDiagnosticsList.get(0).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + Assert.assertEquals(errorDiagnosticsList.get(1).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + Assert.assertEquals(errorDiagnosticsList.get(2).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + Assert.assertEquals(errorDiagnosticsList.get(3).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + Assert.assertEquals(errorDiagnosticsList.get(4).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + Assert.assertEquals(errorDiagnosticsList.get(5).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + Assert.assertEquals(errorDiagnosticsList.get(6).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + Assert.assertEquals(errorDiagnosticsList.get(7).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + Assert.assertEquals(errorDiagnosticsList.get(8).diagnosticInfo() + .messageFormat(), CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + } } diff --git a/compiler-plugin-test/src/test/java/io/ballerina/lib/data/csvdata/compiler/CompilerPluginTestUtils.java b/compiler-plugin-test/src/test/java/io/ballerina/lib/data/csvdata/compiler/CompilerPluginTestUtils.java new file mode 100644 index 0000000..a3f15c4 --- /dev/null +++ b/compiler-plugin-test/src/test/java/io/ballerina/lib/data/csvdata/compiler/CompilerPluginTestUtils.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.data.csvdata.compiler; + +import io.ballerina.projects.Package; +import io.ballerina.projects.ProjectEnvironmentBuilder; +import io.ballerina.projects.directory.BuildProject; +import io.ballerina.projects.environment.Environment; +import io.ballerina.projects.environment.EnvironmentBuilder; + +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Utility functions related to compiler plugins tests. + * + * @since 0.1.0 + */ +public class CompilerPluginTestUtils { + private static final Path RESOURCE_DIRECTORY = Paths.get("src", "test", "resources", "ballerina_sources") + .toAbsolutePath(); + private static final Path DISTRIBUTION_PATH = Paths.get("../", "target", "ballerina-runtime") + .toAbsolutePath(); + + static Package loadPackage(String path) { + Path projectDirPath = RESOURCE_DIRECTORY.resolve(path); + Environment environment = EnvironmentBuilder.getBuilder().setBallerinaHome(DISTRIBUTION_PATH).build(); + ProjectEnvironmentBuilder projectEnvironmentBuilder = ProjectEnvironmentBuilder.getBuilder(environment); + BuildProject project = BuildProject.load(projectEnvironmentBuilder, projectDirPath); + return project.currentPackage(); + } +} diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_1/Ballerina.toml b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_1/Ballerina.toml new file mode 100644 index 0000000..80bcadd --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_1/Ballerina.toml @@ -0,0 +1,5 @@ +[package] +org = "ballerina" +name = "sample_package_1" +version = "0.1.0" +distribution = "2201.9.0" diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_1/main.bal b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_1/main.bal new file mode 100644 index 0000000..9052621 --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_1/main.bal @@ -0,0 +1,33 @@ +import ballerina/data.csv; +import ballerina/lang.regexp; + +type A [[int, string], [int, string]]; +public function main() returns error? { + stream s = ( [[1, 2, 3, 4, 5]]).toStream(); + + [[int, string], [int, string]]|error v = csv:parseString(string `a,b`, {}); + [[int, string], [int, string]]|[anydata...][]|error a = csv:parseString(string `a,b`, {}); + [record {}, record {}, record {}, record {}] _ = check csv:parseString(string `a,b`, {}); + [[int, string], [int, string]] _ = check csv:parseStream(s, {}); + [record {}, record {}, record {}, record {}] _ = check csv:parseStream(s, {}); + A _ = check csv:parseBytes([1,2,3], {}); + record {}[]|[record {}, record {}, record {}, record {}] _ = check csv:parseBytes([1,2,3], {}); + int[][]|[[int, string], [int, string]] _ = check csv:transform([{}], {}); + [[int, string], [int, string]] _ = check csv:parseList([], {}); + [record {}, record {}, record {}, record {}] _ = check csv:transform([{}], {}); + [record {}, record {}, record {}, record {}] _ = check csv:parseList([], {}); + record {record {} a;}[] _ = check csv:parseList([], {}); + record {int[] a;}[] _ = check csv:parseList([], {}); + record {[int...] a;}[] _ = check csv:parseList([], {}); + record {[int...] a;}[] _ = check csv:parseList([], {}); + record {record {} a;}[] _ = check csv:parseList([], {}); + record {xml a; regexp:RegExp b;}[] _ = check csv:parseList([], {}); + [xml, xml][] _ = check csv:parseList([], {}); + [regexp:RegExp, xml, int[]][]|csv:Error v2 = csv:parseList([], {}); + [regexp:RegExp, xml, int[]][]|int[][] _ = check csv:parseList([], {}); + int[][]|[regexp:RegExp, xml, int[]][] _ = check csv:parseList([], {}); + int[][]|[record {}|regexp:RegExp][] _ = check csv:parseList([], {}); + record {}[]|int[][] _ = check csv:parseList([], {}); + record {}[2] _ = check csv:parseList([], {}); + int[3][2] _ = check csv:parseList([], {}); +} diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_10/Ballerina.toml b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_10/Ballerina.toml new file mode 100644 index 0000000..5c24b99 --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_10/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "wso2" +name = "sample_package_10" +version = "0.1.0" +distribution = "2201.9.2" + +[build-options] +observabilityIncluded = true diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_10/main.bal b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_10/main.bal new file mode 100644 index 0000000..666f10f --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_10/main.bal @@ -0,0 +1,58 @@ +import ballerina/data.csv; + +string[]? headers = (); + +record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 0}); +record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: ["a", "b"]}); +record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: ()}); +record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: headers}); +record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 1}); +record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: ["a", "b"]}); +record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: ()}); +record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: headers}); +record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 2}); +record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: ["a", "b"]}); +record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: ()}); +record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: null, outputWithHeaders: false}); +record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: headers}); + +public function main() returns error? { + record {}[] val = check csv:parseList([["1", "2"]], {headerRows: 0}); + val = check csv:parseList([["1", "2"]], {headerRows: 0}); + + record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: ["a", "b"]}); + val = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: ["a", "b"]}); + + record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: ()}); + val = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: ()}); + + record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: headers}); + val = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: headers}); + + record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 1}); + val = check csv:parseList([["1", "2"]], {headerRows: 1}); + + record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: ["a", "b"]}); + val = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: ["a", "b"]}); + + record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: ()}); + val = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: ()}); + + record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: headers}); + val = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: headers}); + + record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 2}); + val = check csv:parseList([["1", "2"]], {headerRows: 2}); + + record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: ["a", "b"]}); + val = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: ["a", "b"]}); + + record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: ()}); + val = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: ()}); + + record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: null, outputWithHeaders: true}); + val = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: null, outputWithHeaders: true}); + + record {}[] _ = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: headers}); + val = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: headers}); +} diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_11/Ballerina.toml b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_11/Ballerina.toml new file mode 100644 index 0000000..26a2006 --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_11/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "wso2" +name = "sample_package_11" +version = "0.1.0" +distribution = "2201.9.2" + +[build-options] +observabilityIncluded = true diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_11/main.bal b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_11/main.bal new file mode 100644 index 0000000..d40a261 --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_11/main.bal @@ -0,0 +1,26 @@ +import ballerina/data.csv; + +type A record { + int:Unsigned32 headerRows = 0; + string[] customHeaders = []; + boolean outputWithHeaders = false; +}; + +public function main() returns error? { + record {}[] _ = check csv:parseString(string `a,b`, {}); + record {}[] _ = test1({headerRows: 2, outputWithHeaders: false}); + [int...][] _ = test2({headerRows: 2, outputWithHeaders: false}); + record {} _ = test3({headerRows: 2, outputWithHeaders: false}); +} + +function test1(A a) returns record {}[] { + return [{}]; +} + +function test2(A a) returns [int...][] { + return []; +} + +function test3(A a) returns record {} { + return {}; +} diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_12/Ballerina.toml b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_12/Ballerina.toml new file mode 100644 index 0000000..4c5b968 --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_12/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "wso2" +name = "sample_package_12" +version = "0.1.0" +distribution = "2201.9.2" + +[build-options] +observabilityIncluded = true diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_12/main.bal b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_12/main.bal new file mode 100644 index 0000000..a7a5241 --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_12/main.bal @@ -0,0 +1,58 @@ +import ballerina/data.csv; + +string[]? headers = (); + +[anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 0}); +[anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: ["a", "b"]}); +[anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: ()}); +[anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: headers}); +[anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 1}); +[anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: ["a", "b"]}); +[anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: ()}); +[anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: headers}); +[anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 2}); +[anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: ["a", "b"]}); +[anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: ()}); +[anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: null, outputWithHeaders: false}); +[anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: headers}); + +public function main() returns error? { + [anydata...][] val = check csv:parseList([["1", "2"]], {headerRows: 0}); + val = check csv:parseList([["1", "2"]], {headerRows: 0}); + + [anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: ["a", "b"]}); + val = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: ["a", "b"]}); + + [anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: ()}); + val = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: ()}); + + [anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: headers}); + val = check csv:parseList([["1", "2"]], {headerRows: 0, customHeaders: headers}); + + [anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 1}); + val = check csv:parseList([["1", "2"]], {headerRows: 1}); + + [anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: ["a", "b"]}); + val = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: ["a", "b"]}); + + [anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: ()}); + val = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: ()}); + + [anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: headers}); + val = check csv:parseList([["1", "2"]], {headerRows: 1, customHeaders: headers}); + + [anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 2}); + val = check csv:parseList([["1", "2"]], {headerRows: 2}); + + [anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: ["a", "b"]}); + val = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: ["a", "b"]}); + + [anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: ()}); + val = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: ()}); + + [anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: null, outputWithHeaders: true}); + val = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: null, outputWithHeaders: true}); + + [anydata...][] _ = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: headers}); + val = check csv:parseList([["1", "2"]], {headerRows: 2, customHeaders: headers}); +} diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_2/Ballerina.toml b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_2/Ballerina.toml new file mode 100644 index 0000000..80bcadd --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_2/Ballerina.toml @@ -0,0 +1,5 @@ +[package] +org = "ballerina" +name = "sample_package_1" +version = "0.1.0" +distribution = "2201.9.0" diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_2/main.bal b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_2/main.bal new file mode 100644 index 0000000..3de2239 --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_2/main.bal @@ -0,0 +1,42 @@ +import ballerina/data.csv as c; + +type A record { + @c:Name {value: "b"} + int a; + string b; +}; + +type B record { + @c:Name {value: "b"} + int a; + @c:Name {value: "a"} + string b; +}; + +type B2 record { + @c:Name {value: "b"} + int a; + @c:Name {value: "b"} + string b; +}; + +type B3 record { + @c:Name {value: "a"} + int a; + @c:Name {value: "a"} + string b2; +}; + +type B4 record { + @c:Name {value: "b"} + int a; + @c:Name {value: "a"} + string b2; +}; + +type C record { + @c:Name {value: "x"} + int a; + @c:Name {value: "x"} + string b; +}; diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_3/Ballerina.toml b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_3/Ballerina.toml new file mode 100644 index 0000000..ee11d8e --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_3/Ballerina.toml @@ -0,0 +1,5 @@ +[package] +org = "ballerina" +name = "sample_package_3" +version = "0.1.0" +distribution = "2201.9.0" diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_3/main.bal b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_3/main.bal new file mode 100644 index 0000000..3253727 --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_3/main.bal @@ -0,0 +1,5 @@ +// test when there is a compiler time error. +type A [[int1, string], [int, string]]; + +public function main() returns error? { +} diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_4/Ballerina.toml b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_4/Ballerina.toml new file mode 100644 index 0000000..b42c32c --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_4/Ballerina.toml @@ -0,0 +1,5 @@ +[package] +org = "ballerina" +name = "sample_package_4" +version = "0.1.0" +distribution = "2201.9.0" diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_4/main.bal b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_4/main.bal new file mode 100644 index 0000000..ebad20f --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_4/main.bal @@ -0,0 +1,17 @@ +import ballerina/data.csv; + +type A [[int, string], [int, string]]; +stream s = ( [[1, 2, 3, 4, 5]]).toStream(); + +[[int, string], [int, string]] val = check csv:parseString(string `a,b`, {}); +[record {}, record {}, record {}, record {}] val2 = check csv:parseString(string `a,b`, {}); +[[int, string], [int, string]] val3 = check csv:parseStream(s, {}); +[record {}, record {}, record {}, record {}] val4 = check csv:parseStream(s, {}); +A val5 = check csv:parseBytes([1,2,3], {}); +record {}[]|[record {}, record {}, record {}, record {}] val6 = check csv:parseBytes([1,2,3], {}); +int[][]|[[int, string], [int, string]] val7 = check csv:transform([{}], {}); +[[int, string], [int, string]] val8 = check csv:parseList([], {}); +[record {}, record {}, record {}, record {}] val9 = check csv:transform([{}], {}); +[record {}, record {}, record {}, record {}] val10 = check csv:parseList([], {}); +record {}[2] val11 = check csv:parseList([], {}); +int[3][2] val12 = check csv:parseList([], {}); diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_5/Ballerina.toml b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_5/Ballerina.toml new file mode 100644 index 0000000..d3bf9e5 --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_5/Ballerina.toml @@ -0,0 +1,5 @@ +[package] +org = "ballerina" +name = "sample_package_5" +version = "0.1.0" +distribution = "2201.9.0" diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_5/main.bal b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_5/main.bal new file mode 100644 index 0000000..62dcecc --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_5/main.bal @@ -0,0 +1,8 @@ +public function main() { + // test scenario without module import + test(); +} + +function test() { + +} diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_6/Ballerina.toml b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_6/Ballerina.toml new file mode 100644 index 0000000..c49ff82 --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_6/Ballerina.toml @@ -0,0 +1,5 @@ +[package] +org = "admin" +name = "sample_package_6" +version = "0.1.0" +distribution = "2201.9.0" diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_6/main.bal b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_6/main.bal new file mode 100644 index 0000000..2574a7f --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_6/main.bal @@ -0,0 +1,24 @@ +import ballerina/data.csv; + +type A [[int, string], [int, string]]; + +public function main() returns error? { + check test(); +} + +public function test() returns error? { + stream s = ( [[1, 2, 3, 4, 5]]).toStream(); + + [[int, string], [int, string]] val = check csv:parseString(string `a,b`, {}); + [record {}, record {}, record {}, record {}] val2 = check csv:parseString(string `a,b`, {}); + [[int, string], [int, string]] val3 = check csv:parseStream(s, {}); + [record {}, record {}, record {}, record {}] val4 = check csv:parseStream(s, {}); + A val5 = check csv:parseBytes([1,2,3], {}); + record {}[]|[record {}, record {}, record {}, record {}] val6 = check csv:parseBytes([1,2,3], {}); + int[][]|[[int, string], [int, string]] val7 = check csv:transform([{}], {}); + [[int, string], [int, string]] val8 = check csv:parseList([], {}); + [record {}, record {}, record {}, record {}] val9 = check csv:transform([{}], {}); + [record {}, record {}, record {}, record {}] val10 = check csv:parseList([], {}); + record {}[2] val11 = check csv:parseList([], {}); + int[3][2] val12 = check csv:parseList([], {}); +} diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_7/Ballerina.toml b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_7/Ballerina.toml new file mode 100644 index 0000000..1bd2166 --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_7/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "wso2" +name = "sample_package_7" +version = "0.1.0" +distribution = "2201.9.2" + +[build-options] +observabilityIncluded = true diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_7/main.bal b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_7/main.bal new file mode 100644 index 0000000..e56e0d0 --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_7/main.bal @@ -0,0 +1,121 @@ +import ballerina/data.csv; + +string[] customHeaders = ["a", "b"]; +int:Unsigned32 header = 0; +()|null header2 = null; + +record {}[] val = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); +record {}[] val2 = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: ()}); +record {}[] val3 = check csv:parseString(string `a, b`, {header: (), customHeadersIfHeadersAbsent: ["a", "b"]}); +record {}[] val4 = check csv:parseString(string `a, b`, {header: null, customHeadersIfHeadersAbsent: ()}); +record {}[] val5 = check csv:parseString(string `a, b`, {customHeadersIfHeadersAbsent: ()}); +record {}[] val6 = check csv:parseString(string `a, b`, {header: ()}); +record {}[] val7 = check csv:parseString(string `a, b`, {customHeadersIfHeadersAbsent: ["a", "b"]}); +record {}[] val8 = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: null}); +record {}[] val9 = check csv:parseString(string `a, b`, {header: (), customHeadersIfHeadersAbsent: null}); +record {}[] val10 = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: customHeaders}); +record {}[] val11 = check csv:parseString(string `a, b`, {header: header, customHeadersIfHeadersAbsent: customHeaders}); +record {}[] val12 = check csv:parseString(string ``, {header: header2, customHeadersIfHeadersAbsent: customHeaders}); +record {}[]|error val13 = csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); +record {}[]|[int...][]|error val14 = csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); + +anydata[][] arrVal = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); +anydata[][] arrVal2 = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: ()}); +anydata[][] arrVal3 = check csv:parseString(string `a, b`, {header: null, customHeadersIfHeadersAbsent: ["a", "b"]}); +anydata[][] arrVal4 = check csv:parseString(string `a, b`, {header: (), customHeadersIfHeadersAbsent: ()}); +anydata[][] arrVal5 = check csv:parseString(string `a, b`, {customHeadersIfHeadersAbsent: ()}); +anydata[][] arrVal6 = check csv:parseString(string `a, b`, {header: ()}); +anydata[][] arrVal7 = check csv:parseString(string `a, b`, {customHeadersIfHeadersAbsent: ["a", "b"]}); +anydata[][] arrVal8 = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: null}); +anydata[][] arrVal9 = check csv:parseString(string `a, b`, {header: null, customHeadersIfHeadersAbsent: null}); +anydata[][] arrVal10 = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: customHeaders}); +anydata[][] arrVal11 = check csv:parseString(string `a, b`, {header: header, customHeadersIfHeadersAbsent: customHeaders}); +anydata[][] arrVal12 = check csv:parseString(string ``, {header: header2, customHeadersIfHeadersAbsent: customHeaders}); +anydata[][]|error arrVal13 = csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); +anydata[][]|[int...][]|error arrVal14 = csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); + +public function main() returns error? { + record {}[] val = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); + val = check csv:parseString(string ``, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); + + record {}[]|error val_2 = csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); + val_2 = csv:parseString(string ``, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); + + record {}[]|record {int a;}[]|error val_3 = csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); + val_3 = csv:parseString(string ``, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); + + record {}[] val2 = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: ()}); + val2 = check csv:parseString(string ``, {header: 0, customHeadersIfHeadersAbsent: ()}); + + record {}[] val3 = check csv:parseString(string `a, b`, {header: (), customHeadersIfHeadersAbsent: ["a", "b"]}); + val3 = check csv:parseString(string ``, {header: (), customHeadersIfHeadersAbsent: ["a", "b"]}); + + record {}[] val4 = check csv:parseString(string `a, b`, {header: (), customHeadersIfHeadersAbsent: ()}); + val4 = check csv:parseString(string ``, {header: null, customHeadersIfHeadersAbsent: ()}); + + record {}[] val5 = check csv:parseString(string `a, b`, {customHeadersIfHeadersAbsent: ()}); + val5 = check csv:parseString(string ``, {customHeadersIfHeadersAbsent: ()}); + + record {}[] val6 = check csv:parseString(string `a, b`, {header: null}); + val6 = check csv:parseString(string ``, {header: ()}); + + record {}[] val7 = check csv:parseString(string `a, b`, {customHeadersIfHeadersAbsent: ["a", "b"]}); + val7 = check csv:parseString(string ``, {customHeadersIfHeadersAbsent: ["a", "b"]}); + + record {}[] val8 = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: null}); + val8 = check csv:parseString(string ``, {header: 0, customHeadersIfHeadersAbsent: null}); + + record {}[] val9 = check csv:parseString(string `a, b`, {header: (), customHeadersIfHeadersAbsent: null}); + val9 = check csv:parseString(string ``, {header: (), customHeadersIfHeadersAbsent: null}); + + record {}[] val10 = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: customHeaders}); + val10 = check csv:parseString(string ``, {header: 0, customHeadersIfHeadersAbsent: customHeaders}); + + record {}[] val11 = check csv:parseString(string `a, b`, {header: header, customHeadersIfHeadersAbsent: customHeaders}); + val11 = check csv:parseString(string `a, b`, {header: header, customHeadersIfHeadersAbsent: customHeaders}); + + record {}[] val12 = check csv:parseString(string ``, {header: header2, customHeadersIfHeadersAbsent: customHeaders}); + val11 = check csv:parseString(string ``, {header: header2, customHeadersIfHeadersAbsent: customHeaders}); + + anydata[][] arrVal = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); + val = check csv:parseString(string ``, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); + + anydata[][]|error arrVal_2 = csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); + arrVal_2 = csv:parseString(string ``, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); + + anydata[][]|[anydata...][]|error arrVal_3 = csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); + arrVal_3 = csv:parseString(string ``, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"]}); + + anydata[][] arrVal2 = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: ()}); + val2 = check csv:parseString(string ``, {header: 0, customHeadersIfHeadersAbsent: ()}); + + anydata[][] arrVal3 = check csv:parseString(string `a, b`, {header: (), customHeadersIfHeadersAbsent: ["a", "b"]}); + val3 = check csv:parseString(string ``, {header: (), customHeadersIfHeadersAbsent: ["a", "b"]}); + + anydata[][] arrVal4 = check csv:parseString(string `a, b`, {header: (), customHeadersIfHeadersAbsent: ()}); + val4 = check csv:parseString(string ``, {header: (), customHeadersIfHeadersAbsent: ()}); + + anydata[][] arrVal5 = check csv:parseString(string `a, b`, {customHeadersIfHeadersAbsent: ()}); + val5 = check csv:parseString(string ``, {customHeadersIfHeadersAbsent: ()}); + + anydata[][] arrVal6 = check csv:parseString(string `a, b`, {header: ()}); + val6 = check csv:parseString(string ``, {header: null}); + + anydata[][] arrVal7 = check csv:parseString(string `a, b`, {customHeadersIfHeadersAbsent: ["a", "b"]}); + val7 = check csv:parseString(string ``, {customHeadersIfHeadersAbsent: ["a", "b"]}); + + anydata[][] arrVal8 = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: null}); + val8 = check csv:parseString(string ``, {header: 0, customHeadersIfHeadersAbsent: null}); + + anydata[][] arrVal9 = check csv:parseString(string `a, b`, {header: null, customHeadersIfHeadersAbsent: null}); + val9 = check csv:parseString(string ``, {header: (), customHeadersIfHeadersAbsent: null}); + + anydata[][] arrVal10 = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: customHeaders}); + val10 = check csv:parseString(string ``, {header: 0, customHeadersIfHeadersAbsent: customHeaders}); + + anydata[][] arrVal11 = check csv:parseString(string `a, b`, {header: header, customHeadersIfHeadersAbsent: customHeaders}); + val11 = check csv:parseString(string `a, b`, {header: header, customHeadersIfHeadersAbsent: customHeaders}); + + anydata[][] arrVal12 = check csv:parseString(string ``, {header: header2, customHeadersIfHeadersAbsent: customHeaders}); + val11 = check csv:parseString(string ``, {header: header2, customHeadersIfHeadersAbsent: customHeaders}); +} diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_8/Ballerina.toml b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_8/Ballerina.toml new file mode 100644 index 0000000..52d54d8 --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_8/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "wso2" +name = "sample_package_8" +version = "0.1.0" +distribution = "2201.9.2" + +[build-options] +observabilityIncluded = true diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_8/main.bal b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_8/main.bal new file mode 100644 index 0000000..6e0de16 --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_8/main.bal @@ -0,0 +1,18 @@ +import ballerina/data.csv; + +boolean o = false; +record {}[] val = check csv:parseString(string `a, b`, {outputWithHeaders: false}); +record {}[] val2 = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: (), outputWithHeaders: true}); +record {}[] val3 = check csv:parseString(string `a, b`, {header: (), customHeadersIfHeadersAbsent: ["a", "b"]}); +record {}[] val4 = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: (), outputWithHeaders: o}); + +public function main() returns error? { + record {}[] val = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"], outputWithHeaders: false}); + val = check csv:parseString(string ``, {header: 0, customHeadersIfHeadersAbsent: ["a", "b"], outputWithHeaders: true}); + + record {}[] val2 = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: (), outputWithHeaders: false}); + val2 = check csv:parseString(string ``, {header: 0, customHeadersIfHeadersAbsent: (), outputWithHeaders: true}); + + record {}[] val3 = check csv:parseString(string `a, b`, {header: 0, customHeadersIfHeadersAbsent: (), outputWithHeaders: o}); + val3 = check csv:parseString(string ``, {header: 0, customHeadersIfHeadersAbsent: (), outputWithHeaders: o}); +} diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_9/Ballerina.toml b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_9/Ballerina.toml new file mode 100644 index 0000000..9f56b33 --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_9/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "wso2" +name = "sample_package_9" +version = "0.1.0" +distribution = "2201.9.2" + +[build-options] +observabilityIncluded = true diff --git a/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_9/main.bal b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_9/main.bal new file mode 100644 index 0000000..2a4dedf --- /dev/null +++ b/compiler-plugin-test/src/test/resources/ballerina_sources/sample_package_9/main.bal @@ -0,0 +1,22 @@ +import ballerina/data.csv; + +boolean o = false; +string[] headerOrder = ["a", "b"]; +record {}[] val = check csv:transform([{"a": 1, "b": 2}], {headerOrder: ["a", "b"], outputWithHeaders: false}); +record {}[] val2 = check csv:transform([{"a": 1, "b": 2}], {headerOrder: ["a", "b"]}); +record {}[] val3 = check csv:transform([{"a": 1, "b": 2}], {"header": false, headerOrder: headerOrder}); +record {}[] val4 = check csv:transform([{"a": 1, "b": 2}], {outputWithHeaders: o, headerOrder: ()}); + +public function main() returns error? { + record {}[] val = check csv:transform([{"a": 1, "b": 2}], {headerOrder: ()}); + val = check csv:transform([{"a": 1, "b": 2}], {headerOrder: ()}); + + record {}[] val2 = check csv:transform([{"a": 1, "b": 2}], {headerOrder: ["a", "b"], outputWithHeaders: false}); + val2 = check csv:transform([{"a": 1, "b": 2}], {headerOrder: ["a", "b"], outputWithHeaders: true}); + + record {}[] val3 = check csv:transform([{"a": 1, "b": 2}], {headerOrder: null}); + val3 = check csv:transform([{"a": 1, "b": 2}], {headerOrder: null}); + + record {}[] val4 = check csv:transform([{"a": 1, "b": 2}], {headerOrder}); + val4 = check csv:transform([{"a": 1, "b": 2}], {headerOrder}); +} diff --git a/compiler-plugin-test/src/test/resources/testng.xml b/compiler-plugin-test/src/test/resources/testng.xml index 02c1d8e..e5f9421 100644 --- a/compiler-plugin-test/src/test/resources/testng.xml +++ b/compiler-plugin-test/src/test/resources/testng.xml @@ -21,7 +21,7 @@ - + diff --git a/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/Constants.java b/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/Constants.java new file mode 100644 index 0000000..738e715 --- /dev/null +++ b/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/Constants.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.data.csvdata.compiler; + +/** + * Constants for CsvData's compiler plugin. + * + * @since 0.1.0 + */ +public final class Constants { + static final String PARSE_STRING = "parseString"; + static final String PARSE_BYTES = "parseBytes"; + static final String PARSE_STREAM = "parseStream"; + static final String TRANSFORM = "transform"; + static final String PARSE_LISTS = "parseList"; + static final String NAME = "Name"; + static final String CSVDATA = "csv"; + static final String BALLERINA = "ballerina"; + static final String DATA_CSVDATA = "data.csv"; + static final String NIL = "()"; + static final String NULL = "null"; + + static class UserConfigurations { + private UserConfigurations() { + } + + static final String OUTPUT_WITH_HEADERS = "outputWithHeaders"; + static final String HEADERS_ORDER = "headerOrder"; + static final String HEADER = "header"; + static final String CUSTOM_HEADERS = "customHeaders"; + static final String HEADERS_ROWS = "headerRows"; + static final String CUSTOM_HEADERS_IF_ABSENT = "customHeadersIfHeadersAbsent"; + } + + private Constants() { + } +} diff --git a/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/CsvDataCodeAnalyzer.java b/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/CsvDataCodeAnalyzer.java new file mode 100644 index 0000000..1e67372 --- /dev/null +++ b/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/CsvDataCodeAnalyzer.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.data.csvdata.compiler; + +import io.ballerina.compiler.syntax.tree.SyntaxKind; +import io.ballerina.projects.plugins.CodeAnalysisContext; +import io.ballerina.projects.plugins.CodeAnalyzer; + +import java.util.List; + +/** + * Csvdata Code Analyzer. + * + * @since 0.1.0 + */ +public class CsvDataCodeAnalyzer extends CodeAnalyzer { + @Override + public void init(CodeAnalysisContext codeAnalysisContext) { + codeAnalysisContext.addSyntaxNodeAnalysisTask(new CsvDataTypeValidator(), + List.of(SyntaxKind.MODULE_PART)); + } +} diff --git a/compiler-plugin/src/main/java/io/ballerina/stdlib/data/csvdata/compiler/CsvDataCompilerPlugin.java b/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/CsvDataCompilerPlugin.java similarity index 89% rename from compiler-plugin/src/main/java/io/ballerina/stdlib/data/csvdata/compiler/CsvDataCompilerPlugin.java rename to compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/CsvDataCompilerPlugin.java index 80f28de..430091e 100644 --- a/compiler-plugin/src/main/java/io/ballerina/stdlib/data/csvdata/compiler/CsvDataCompilerPlugin.java +++ b/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/CsvDataCompilerPlugin.java @@ -16,7 +16,7 @@ * under the License. */ -package io.ballerina.stdlib.data.csvdata.compiler; +package io.ballerina.lib.data.csvdata.compiler; import io.ballerina.projects.plugins.CompilerPlugin; import io.ballerina.projects.plugins.CompilerPluginContext; @@ -30,6 +30,6 @@ public class CsvDataCompilerPlugin extends CompilerPlugin { @Override public void init(CompilerPluginContext compilerPluginContext) { - + compilerPluginContext.addCodeAnalyzer(new CsvDataCodeAnalyzer()); } } diff --git a/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/CsvDataDiagnosticCodes.java b/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/CsvDataDiagnosticCodes.java new file mode 100644 index 0000000..5361b38 --- /dev/null +++ b/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/CsvDataDiagnosticCodes.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.data.csvdata.compiler; + +import io.ballerina.tools.diagnostics.DiagnosticSeverity; + +import static io.ballerina.tools.diagnostics.DiagnosticSeverity.ERROR; + +/** + * Diagnostic codes for Csv data's compiler plugin. + * + * @since 0.1.0 + */ +public enum CsvDataDiagnosticCodes { + DUPLICATE_FIELD("CSV_ERROR_1", "invalid field: duplicate field found", ERROR), + UNSUPPORTED_TYPE("CSV_ERROR_2", "unsupported type: type is not supported", ERROR), + UNSUPPORTED_FIELD_TYPE("CSV_ERROR_3", "Unsupported type in the field: Only basic types are supported for fields, " + + "and other types are not allowed.", ERROR), + UNSUPPORTED_TUPLE_MEMBER_TYPE("CSV_ERROR_4", "Unsupported type in the tuple member: Tuple members can only " + + "be basic types, other types are not supported.", ERROR), + IGNORE_OUTPUT_HEADERS_FOR_RECORD_ARRAY("CSV_ERROR_5", "The option 'outputWithHeaders' will " + + "be not allowed since the expected type is a subtype record array.", ERROR), + IGNORE_HEADERS_ORDER_FOR_RECORD_ARRAY("CSV_ERROR_5", "The option 'headerOrder' will " + + "be not allowed since the expected type is a subtype record array.", ERROR), + IGNORE_CUSTOM_HEADERS_PARAMETER_WHEN_HEADER_PRESENT("CSV_ERROR_6", + "The option 'customHeadersIfHeadersAbsent' will be not " + + "allowed since the header is present.", ERROR), + CUSTOM_HEADERS_SHOULD_BE_PROVIDED("CSV_ERROR_7", + "customHeaders parameter should be provided since the headerRows larger than 1.", ERROR); + + private final String code; + private final String message; + private final DiagnosticSeverity severity; + + CsvDataDiagnosticCodes(String code, String message, DiagnosticSeverity severity) { + this.code = code; + this.message = message; + this.severity = severity; + } + + public String getCode() { + return code; + } + + public String getMessage() { + return message; + } + + public DiagnosticSeverity getSeverity() { + return severity; + } +} diff --git a/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/CsvDataTypeValidator.java b/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/CsvDataTypeValidator.java new file mode 100644 index 0000000..49ff126 --- /dev/null +++ b/compiler-plugin/src/main/java/io/ballerina/lib/data/csvdata/compiler/CsvDataTypeValidator.java @@ -0,0 +1,640 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.data.csvdata.compiler; + +import io.ballerina.compiler.api.ModuleID; +import io.ballerina.compiler.api.SemanticModel; +import io.ballerina.compiler.api.symbols.AnnotationAttachmentSymbol; +import io.ballerina.compiler.api.symbols.AnnotationSymbol; +import io.ballerina.compiler.api.symbols.ArrayTypeSymbol; +import io.ballerina.compiler.api.symbols.IntersectionTypeSymbol; +import io.ballerina.compiler.api.symbols.ModuleSymbol; +import io.ballerina.compiler.api.symbols.RecordFieldSymbol; +import io.ballerina.compiler.api.symbols.RecordTypeSymbol; +import io.ballerina.compiler.api.symbols.Symbol; +import io.ballerina.compiler.api.symbols.SymbolKind; +import io.ballerina.compiler.api.symbols.TupleTypeSymbol; +import io.ballerina.compiler.api.symbols.TypeDefinitionSymbol; +import io.ballerina.compiler.api.symbols.TypeDescKind; +import io.ballerina.compiler.api.symbols.TypeReferenceTypeSymbol; +import io.ballerina.compiler.api.symbols.TypeSymbol; +import io.ballerina.compiler.api.symbols.UnionTypeSymbol; +import io.ballerina.compiler.api.symbols.VariableSymbol; +import io.ballerina.compiler.syntax.tree.AssignmentStatementNode; +import io.ballerina.compiler.syntax.tree.BasicLiteralNode; +import io.ballerina.compiler.syntax.tree.CheckExpressionNode; +import io.ballerina.compiler.syntax.tree.ChildNodeList; +import io.ballerina.compiler.syntax.tree.ExpressionNode; +import io.ballerina.compiler.syntax.tree.FunctionArgumentNode; +import io.ballerina.compiler.syntax.tree.FunctionCallExpressionNode; +import io.ballerina.compiler.syntax.tree.FunctionDefinitionNode; +import io.ballerina.compiler.syntax.tree.IdentifierToken; +import io.ballerina.compiler.syntax.tree.ImportDeclarationNode; +import io.ballerina.compiler.syntax.tree.ListConstructorExpressionNode; +import io.ballerina.compiler.syntax.tree.MappingConstructorExpressionNode; +import io.ballerina.compiler.syntax.tree.MappingFieldNode; +import io.ballerina.compiler.syntax.tree.ModuleMemberDeclarationNode; +import io.ballerina.compiler.syntax.tree.ModulePartNode; +import io.ballerina.compiler.syntax.tree.ModuleVariableDeclarationNode; +import io.ballerina.compiler.syntax.tree.NameReferenceNode; +import io.ballerina.compiler.syntax.tree.NamedArgumentNode; +import io.ballerina.compiler.syntax.tree.NilLiteralNode; +import io.ballerina.compiler.syntax.tree.Node; +import io.ballerina.compiler.syntax.tree.PositionalArgumentNode; +import io.ballerina.compiler.syntax.tree.QualifiedNameReferenceNode; +import io.ballerina.compiler.syntax.tree.SeparatedNodeList; +import io.ballerina.compiler.syntax.tree.SpecificFieldNode; +import io.ballerina.compiler.syntax.tree.SyntaxKind; +import io.ballerina.compiler.syntax.tree.TypeDefinitionNode; +import io.ballerina.compiler.syntax.tree.VariableDeclarationNode; +import io.ballerina.projects.plugins.AnalysisTask; +import io.ballerina.projects.plugins.SyntaxNodeAnalysisContext; +import io.ballerina.tools.diagnostics.Diagnostic; +import io.ballerina.tools.diagnostics.DiagnosticFactory; +import io.ballerina.tools.diagnostics.DiagnosticInfo; +import io.ballerina.tools.diagnostics.DiagnosticSeverity; +import io.ballerina.tools.diagnostics.Location; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** + * CsvData Record Field Validator. + * + * @since 0.1.0 + */ +public class CsvDataTypeValidator implements AnalysisTask { + + private SemanticModel semanticModel; + private final HashMap allDiagnosticInfo = new HashMap<>(); + Location currentLocation; + private String modulePrefix = Constants.CSVDATA; + + @Override + public void perform(SyntaxNodeAnalysisContext ctx) { + semanticModel = ctx.semanticModel(); + List diagnostics = semanticModel.diagnostics(); + boolean erroneousCompilation = diagnostics.stream() + .anyMatch(d -> d.diagnosticInfo().severity().equals(DiagnosticSeverity.ERROR)); + if (erroneousCompilation) { + reset(); + return; + } + + ModulePartNode rootNode = (ModulePartNode) ctx.node(); + updateModulePrefix(rootNode); + + for (ModuleMemberDeclarationNode member : rootNode.members()) { + switch (member.kind()) { + case FUNCTION_DEFINITION -> processFunctionDefinitionNode((FunctionDefinitionNode) member, ctx); + case MODULE_VAR_DECL -> + processModuleVariableDeclarationNode((ModuleVariableDeclarationNode) member, ctx); + case TYPE_DEFINITION -> + processTypeDefinitionNode((TypeDefinitionNode) member, ctx); + } + } + reset(); + } + + private void reset() { + semanticModel = null; + allDiagnosticInfo.clear(); + currentLocation = null; + modulePrefix = Constants.CSVDATA; + } + + private void updateModulePrefix(ModulePartNode rootNode) { + for (ImportDeclarationNode importDeclarationNode : rootNode.imports()) { + semanticModel.symbol(importDeclarationNode) + .filter(moduleSymbol -> moduleSymbol.kind() == SymbolKind.MODULE) + .filter(moduleSymbol -> isCsvDataImport((ModuleSymbol) moduleSymbol)) + .ifPresent(moduleSymbol -> modulePrefix = ((ModuleSymbol) moduleSymbol).id().modulePrefix()); + } + } + + private void processFunctionDefinitionNode(FunctionDefinitionNode functionDefinitionNode, + SyntaxNodeAnalysisContext ctx) { + ChildNodeList childNodeList = functionDefinitionNode.functionBody().children(); + for (Node node : childNodeList) { + if (node.kind() == SyntaxKind.LOCAL_VAR_DECL) { + processLocalVarDeclNode((VariableDeclarationNode) node, ctx); + } else if (node.kind() == SyntaxKind.ASSIGNMENT_STATEMENT) { + processAssignmentStmtNode((AssignmentStatementNode) node, ctx); + } + } + } + + private void processAssignmentStmtNode(AssignmentStatementNode assignmentStatementNode, + SyntaxNodeAnalysisContext ctx) { + ExpressionNode expressionNode = assignmentStatementNode.expression(); + if (!isParseFunctionOfStringSource(expressionNode)) { + return; + } + currentLocation = assignmentStatementNode.location(); + semanticModel.symbol(assignmentStatementNode.varRef()) + .map(symbol -> ((VariableSymbol) symbol).typeDescriptor()) + .ifPresent(typeSymbol -> validateFunctionParameterTypes(expressionNode, typeSymbol, currentLocation, ctx)); + } + + private void processLocalVarDeclNode(VariableDeclarationNode variableDeclarationNode, + SyntaxNodeAnalysisContext ctx) { + Optional initializer = variableDeclarationNode.initializer(); + if (initializer.isEmpty()) { + return; + } + + currentLocation = variableDeclarationNode.typedBindingPattern().typeDescriptor().location(); + Optional symbol = semanticModel.symbol(variableDeclarationNode.typedBindingPattern()); + if (symbol.isEmpty()) { + return; + } + + TypeSymbol typeSymbol = ((VariableSymbol) symbol.get()).typeDescriptor(); + ExpressionNode expressionNode = initializer.get(); + if (!isParseFunctionOfStringSource(expressionNode)) { + checkTypeAndDetectDuplicateFields(typeSymbol, ctx); + return; + } + validateExpectedType(typeSymbol, currentLocation, ctx); + validateFunctionParameterTypes(expressionNode, typeSymbol, expressionNode.location(), ctx); + } + + private void checkTypeAndDetectDuplicateFields(TypeSymbol typeSymbol, SyntaxNodeAnalysisContext ctx) { + switch (typeSymbol.typeKind()) { + case RECORD -> detectDuplicateFields((RecordTypeSymbol) typeSymbol, ctx); + case ARRAY -> checkTypeAndDetectDuplicateFields(((ArrayTypeSymbol) typeSymbol).memberTypeDescriptor(), ctx); + case TUPLE -> { + for (TypeSymbol memberType : ((TupleTypeSymbol) typeSymbol).memberTypeDescriptors()) { + checkTypeAndDetectDuplicateFields(memberType, ctx); + } + } + case UNION -> { + for (TypeSymbol memberType : ((UnionTypeSymbol) typeSymbol).memberTypeDescriptors()) { + checkTypeAndDetectDuplicateFields(memberType, ctx); + } + } + case TYPE_REFERENCE -> checkTypeAndDetectDuplicateFields( + ((TypeReferenceTypeSymbol) typeSymbol).typeDescriptor(), ctx); + case INTERSECTION -> checkTypeAndDetectDuplicateFields(getRawType(typeSymbol), ctx); + } + } + + private FunctionCallExpressionNode getFunctionCallExpressionNodeIfPresent(ExpressionNode expressionNode) { + return switch(expressionNode.kind()) { + case CHECK_EXPRESSION -> getFunctionCallExpressionNodeIfPresent( + ((CheckExpressionNode) expressionNode).expression()); + case FUNCTION_CALL -> (FunctionCallExpressionNode) expressionNode; + default -> null; + }; + } + + private Optional getFunctionName(FunctionCallExpressionNode node) { + NameReferenceNode nameReferenceNode = node.functionName(); + if (nameReferenceNode.kind() != SyntaxKind.QUALIFIED_NAME_REFERENCE) { + return Optional.empty(); + } + QualifiedNameReferenceNode qualifiedNameReferenceNode = (QualifiedNameReferenceNode) nameReferenceNode; + String prefix = qualifiedNameReferenceNode.modulePrefix().text(); + if (!prefix.equals(modulePrefix)) { + return Optional.empty(); + } + return Optional.of(qualifiedNameReferenceNode.identifier().text()); + } + + private boolean isParseFunctionOfStringSource(ExpressionNode expressionNode) { + FunctionCallExpressionNode node = getFunctionCallExpressionNodeIfPresent(expressionNode); + if (node == null) { + return false; + } + Optional functionName = getFunctionName(node); + return functionName + .map(fn -> fn.contains(Constants.PARSE_STRING) || fn.contains(Constants.PARSE_BYTES) || + fn.contains(Constants.PARSE_STREAM) || fn.contains(Constants.TRANSFORM) || + fn.contains(Constants.PARSE_LISTS)) + .orElse(false); + } + + private void validateFunctionParameterTypes(ExpressionNode expressionNode, + TypeSymbol expType, + Location currentLocation, SyntaxNodeAnalysisContext ctx) { + FunctionCallExpressionNode node = getFunctionCallExpressionNodeIfPresent(expressionNode); + if (node == null) { + return; + } + Optional functionName = getFunctionName(node); + SeparatedNodeList args = node.arguments(); + functionName.ifPresent(fn -> + validateFunctionParameterTypesWithExpType(expType, currentLocation, ctx, fn, args)); + } + + private void validateFunctionParameterTypesWithExpType(TypeSymbol expType, Location currentLocation, + SyntaxNodeAnalysisContext ctx, String functionName, SeparatedNodeList args) { + switch (expType.typeKind()) { + case ARRAY -> validateFunctionParameterTypesWithArrayType( + (ArrayTypeSymbol) expType, currentLocation, ctx, functionName, args); + case TYPE_REFERENCE -> validateFunctionParameterTypesWithExpType( + ((TypeReferenceTypeSymbol) expType).typeDescriptor(), currentLocation, ctx, functionName, args); + case INTERSECTION -> validateFunctionParameterTypesWithExpType( + getRawType(expType), currentLocation, ctx, functionName, args); + case UNION -> { + List memberTypes = ((UnionTypeSymbol) expType).memberTypeDescriptors(); + + // only handles the A|error scenarios + if (memberTypes.size() == 2) { + if (isUnionContainsError(memberTypes)) { + TypeSymbol nonErrorTypeSymbol = ignoreErrorTypeFromUnionTypeSymbolAndReturn(memberTypes); + if (nonErrorTypeSymbol != null) { + validateFunctionParameterTypesWithExpType(nonErrorTypeSymbol, currentLocation, + ctx, functionName, args); + } + } + } + } + } + } + + private boolean isUnionContainsError(List memberTypes) { + for (TypeSymbol memberSymbol : memberTypes) { + if (memberSymbol.typeKind() == TypeDescKind.TYPE_REFERENCE) { + memberSymbol = ((TypeReferenceTypeSymbol) memberSymbol).typeDescriptor(); + } + if (memberSymbol.typeKind() == TypeDescKind.ERROR) { + // ignore error type + return true; + } + } + return false; + } + + private TypeSymbol ignoreErrorTypeFromUnionTypeSymbolAndReturn(List memberTypes) { + for (TypeSymbol memberSymbol : memberTypes) { + if (memberSymbol.typeKind() == TypeDescKind.TYPE_REFERENCE) { + memberSymbol = ((TypeReferenceTypeSymbol) memberSymbol).typeDescriptor(); + } + if (memberSymbol.typeKind() == TypeDescKind.ERROR) { + // ignore error type + continue; + } + return memberSymbol; + } + return null; + } + + private void validateFunctionParameterTypesWithArrayType(ArrayTypeSymbol expType, Location currentLocation, + SyntaxNodeAnalysisContext ctx, String functionName, + SeparatedNodeList args) { + TypeSymbol memberTypeSymbol = expType.memberTypeDescriptor(); + if (memberTypeSymbol.typeKind() == TypeDescKind.TYPE_REFERENCE) { + memberTypeSymbol = ((TypeReferenceTypeSymbol) memberTypeSymbol).typeDescriptor(); + } + switch (memberTypeSymbol.typeKind()) { + case RECORD, MAP -> validateFunctionParameterTypesWithOptions( + currentLocation, ctx, functionName, args, true); + case ARRAY, TUPLE -> validateFunctionParameterTypesWithOptions( + currentLocation, ctx, functionName, args, false); + } + } + + private void validateFunctionParameterTypesWithOptions(Location currentLocation, SyntaxNodeAnalysisContext ctx, + String functionName, SeparatedNodeList args, boolean isRecord) { + ExpressionNode expression; + SeparatedNodeList fields; + String header = null, headerRows = null, customHeaders = null, + customHeadersIfHeaderAbsent = null, outputWithHeaders = null, headerOrder = null; + boolean isCustomHeaderPresent = false; + for (FunctionArgumentNode arg : args) { + int mappingConstructorExprNodeCount = 0; + if (arg instanceof PositionalArgumentNode positionalArgumentNode) { + expression = positionalArgumentNode.expression(); + } else if (arg instanceof NamedArgumentNode namedArgumentNode) { + expression = namedArgumentNode.expression(); + } else { + continue; + } + if (expression instanceof MappingConstructorExpressionNode mappingConstructorExpressionNode) { + checkAndAssertMappingConstructorArguments(mappingConstructorExprNodeCount); + fields = mappingConstructorExpressionNode.fields(); + for (MappingFieldNode field : fields) { + if (field instanceof SpecificFieldNode specificFieldNode) { + Node node = specificFieldNode.fieldName(); + if (node instanceof IdentifierToken identifierToken) { + String fieldName = identifierToken.text(); + switch (fieldName) { + case Constants.UserConfigurations.HEADER -> + header = getTheValueOfTheUserConfigOption(specificFieldNode); + case Constants.UserConfigurations.CUSTOM_HEADERS_IF_ABSENT -> + customHeadersIfHeaderAbsent = + getTheValueOfTheUserConfigOption(specificFieldNode); + case Constants.UserConfigurations.HEADERS_ROWS -> + headerRows = getTheValueOfTheUserConfigOption(specificFieldNode); + case Constants.UserConfigurations.CUSTOM_HEADERS -> { + customHeaders = getTheValueOfTheUserConfigOption(specificFieldNode); + isCustomHeaderPresent = true; + } + case Constants.UserConfigurations.HEADERS_ORDER -> { + if (isRecord) { + headerOrder = getTheValueOfTheUserConfigOption(specificFieldNode); + } + } + case Constants.UserConfigurations.OUTPUT_WITH_HEADERS -> { + if (isRecord) { + outputWithHeaders = getTheValueOfTheUserConfigOption(specificFieldNode); + } + } + } + } + } + } + } + } + throwErrorsIfIgnoredFieldFoundForOutputs(header, customHeadersIfHeaderAbsent, headerRows, + customHeaders, isCustomHeaderPresent, headerOrder, + outputWithHeaders, ctx, currentLocation, functionName, isRecord); + } + + private void checkAndAssertMappingConstructorArguments(int mappingConstructorExprNodeCount) { + if (mappingConstructorExprNodeCount > 1) { + assert false : "MappingConstructorExpressionNode count in the function " + + "arguments should be less than or equal to 1"; + } + } + + private void throwErrorsIfIgnoredFieldFoundForOutputs(String header, String customHeadersIfHeaderAbsent, + String headerRows, String customHeaders, boolean isCustomHeaderPresent, String headerOrder, + String outputWithHeaders, SyntaxNodeAnalysisContext ctx, Location currentLocation, + String functionName, boolean isRecord) { + switch (functionName) { + case Constants.PARSE_STRING -> { + if (header != null && !(header.equals(Constants.NIL) || header.equals(Constants.NULL)) + && customHeadersIfHeaderAbsent != null && !customHeadersIfHeaderAbsent.equals(Constants.NIL) + && !customHeadersIfHeaderAbsent.equals(Constants.NULL)) { + reportDiagnosticInfo(ctx, Optional.ofNullable(currentLocation), + CsvDataDiagnosticCodes.IGNORE_CUSTOM_HEADERS_PARAMETER_WHEN_HEADER_PRESENT); + } + } + case Constants.PARSE_LISTS -> { + if (headerRows != null && !headerRows.equals("0") && !headerRows.equals("1") + && (!isCustomHeaderPresent || (customHeaders != null && + (customHeaders.equals(Constants.NIL) || customHeaders.equals(Constants.NULL))))) { + reportDiagnosticInfo(ctx, Optional.ofNullable(currentLocation), + CsvDataDiagnosticCodes.CUSTOM_HEADERS_SHOULD_BE_PROVIDED); + } + } + case Constants.TRANSFORM -> { + if (isRecord && headerOrder != null && !headerOrder.equals(Constants.NIL) + && !headerOrder.equals(Constants.NULL)) { + reportDiagnosticInfo(ctx, Optional.ofNullable(currentLocation), + CsvDataDiagnosticCodes.IGNORE_HEADERS_ORDER_FOR_RECORD_ARRAY); + } + } + } + if (isRecord && outputWithHeaders != null) { + reportDiagnosticInfo(ctx, Optional.ofNullable(currentLocation), + CsvDataDiagnosticCodes.IGNORE_OUTPUT_HEADERS_FOR_RECORD_ARRAY); + } + } + + private String getTheValueOfTheUserConfigOption(SpecificFieldNode specificFieldNode) { + return specificFieldNode.valueExpr().map(expNode -> { + if (expNode instanceof BasicLiteralNode basicLiteralNode) { + return basicLiteralNode.literalToken().text(); + } + if (expNode instanceof ListConstructorExpressionNode listConstructorExpressionNode) { + return listConstructorExpressionNode.expressions().toString(); + } + if (expNode instanceof NilLiteralNode) { + return Constants.NIL; + } + return null; + }).orElse(null); + } + + private void validateExpectedType(TypeSymbol typeSymbol, Location currentLocation, SyntaxNodeAnalysisContext ctx) { + switch (typeSymbol.typeKind()) { + case UNION -> validateUnionType((UnionTypeSymbol) typeSymbol, currentLocation, ctx); + case ARRAY -> validateArrayType((ArrayTypeSymbol) typeSymbol, currentLocation, ctx); + case TUPLE -> validateTupleType(currentLocation, ctx); + case TYPE_REFERENCE -> validateExpectedType(((TypeReferenceTypeSymbol) typeSymbol) + .typeDescriptor(), currentLocation, ctx); + case INTERSECTION -> validateExpectedType(getRawType(typeSymbol), currentLocation, ctx); + } + } + + private void validateTupleType(Location currentLocation, SyntaxNodeAnalysisContext ctx) { + // Currently, this is unsupported. + reportDiagnosticInfo(ctx, Optional.ofNullable(currentLocation), CsvDataDiagnosticCodes.UNSUPPORTED_TYPE); + } + + private void validateArrayType(ArrayTypeSymbol typeSymbol, + Location currentLocation, SyntaxNodeAnalysisContext ctx) { + if (!isSupportedArrayMemberType(ctx, currentLocation, typeSymbol.memberTypeDescriptor())) { + reportDiagnosticInfo(ctx, Optional.ofNullable(currentLocation), CsvDataDiagnosticCodes.UNSUPPORTED_TYPE); + } + } + + private void validateUnionType(UnionTypeSymbol unionTypeSymbol, + Location currentLocation, SyntaxNodeAnalysisContext ctx) { + List memberTypeSymbols = unionTypeSymbol.memberTypeDescriptors(); + for (TypeSymbol memberTypeSymbol : memberTypeSymbols) { + validateExpectedType(memberTypeSymbol, currentLocation, ctx); + } + } + + private boolean isSupportedArrayMemberType(SyntaxNodeAnalysisContext ctx, + Location currentLocation, TypeSymbol typeSymbol) { + if (typeSymbol.typeKind() == TypeDescKind.TYPE_REFERENCE) { + typeSymbol = ((TypeReferenceTypeSymbol) typeSymbol).typeDescriptor(); + } + if (typeSymbol.typeKind() == TypeDescKind.INTERSECTION) { + typeSymbol = getRawType(typeSymbol); + } + TypeDescKind kind = typeSymbol.typeKind(); + if (kind == TypeDescKind.TYPE_REFERENCE) { + kind = ((TypeReferenceTypeSymbol) typeSymbol).typeDescriptor().typeKind(); + } + + switch (kind) { + case ARRAY, MAP, UNION, INTERSECTION -> { + return true; + } + case RECORD -> validateRecordFields(ctx, currentLocation, typeSymbol); + case TUPLE -> validateTupleMembers(ctx, currentLocation, typeSymbol); + default -> { + return false; + } + } + return true; + } + + private void validateTupleMembers(SyntaxNodeAnalysisContext ctx, Location currentLocation, TypeSymbol typeSymbol) { + if (typeSymbol.typeKind() == TypeDescKind.TYPE_REFERENCE) { + typeSymbol = ((TypeReferenceTypeSymbol) typeSymbol).typeDescriptor(); + } + TupleTypeSymbol tupleTypeSymbol = (TupleTypeSymbol) typeSymbol; + tupleTypeSymbol.memberTypeDescriptors().forEach(symbol -> + validateNestedTypeSymbols(ctx, currentLocation, symbol, false)); + tupleTypeSymbol.restTypeDescriptor().ifPresent(restSym -> + validateNestedTypeSymbols(ctx, currentLocation, restSym, false)); + } + + private void validateRecordFields(SyntaxNodeAnalysisContext ctx, Location currentLocation, TypeSymbol typeSymbol) { + if (typeSymbol.typeKind() == TypeDescKind.TYPE_REFERENCE) { + typeSymbol = ((TypeReferenceTypeSymbol) typeSymbol).typeDescriptor(); + } + RecordTypeSymbol recordTypeSymbol = (RecordTypeSymbol) typeSymbol; + + recordTypeSymbol.typeInclusions().forEach(symbol -> + validateNestedTypeSymbols(ctx, currentLocation, symbol, true)); + + recordTypeSymbol.fieldDescriptors().values().forEach(field -> validateNestedTypeSymbols(ctx, + currentLocation, field.typeDescriptor(), true)); + + recordTypeSymbol.restTypeDescriptor().ifPresent(restSym -> + validateNestedTypeSymbols(ctx, currentLocation, restSym, true)); + } + + private void validateNestedTypeSymbols(SyntaxNodeAnalysisContext ctx, + Location location, TypeSymbol typeSymbol, boolean isField) { + TypeDescKind kind = typeSymbol.typeKind(); + if (kind == TypeDescKind.TYPE_REFERENCE) { + kind = ((TypeReferenceTypeSymbol) typeSymbol).typeDescriptor().typeKind(); + } + + switch (kind) { + case ARRAY, OBJECT, RECORD, MAP, ERROR, FUNCTION, TUPLE, STREAM, FUTURE, TYPEDESC, + TYPE_REFERENCE, XML, XML_ELEMENT, XML_PROCESSING_INSTRUCTION, XML_COMMENT, + XML_TEXT, HANDLE, TABLE, NEVER, REGEXP -> + reportDiagnosticInfo(ctx, Optional.ofNullable(location), + isField ? CsvDataDiagnosticCodes.UNSUPPORTED_FIELD_TYPE + : CsvDataDiagnosticCodes.UNSUPPORTED_TUPLE_MEMBER_TYPE); + } + } + + public static TypeSymbol getRawType(TypeSymbol typeDescriptor) { + if (typeDescriptor.typeKind() == TypeDescKind.INTERSECTION) { + return getRawType(((IntersectionTypeSymbol) typeDescriptor).effectiveTypeDescriptor()); + } + if (typeDescriptor.typeKind() == TypeDescKind.TYPE_REFERENCE) { + TypeReferenceTypeSymbol typeRef = (TypeReferenceTypeSymbol) typeDescriptor; + TypeSymbol refType = typeRef.typeDescriptor(); + return switch (refType.typeKind()) { + case TYPE_REFERENCE, INTERSECTION -> getRawType(refType); + default -> refType; + }; + } + return typeDescriptor; + } + + private void reportDiagnosticInfo(SyntaxNodeAnalysisContext ctx, Optional location, + CsvDataDiagnosticCodes diagnosticsCodes) { + Location pos = location.orElseGet(() -> currentLocation); + DiagnosticInfo diagnosticInfo = new DiagnosticInfo(diagnosticsCodes.getCode(), + diagnosticsCodes.getMessage(), diagnosticsCodes.getSeverity()); + if (pos == null || Objects.equals(allDiagnosticInfo.get(pos), diagnosticInfo)) { + return; + } + + allDiagnosticInfo.put(pos, diagnosticInfo); + ctx.reportDiagnostic(DiagnosticFactory.createDiagnostic(diagnosticInfo, pos)); + } + + private void processModuleVariableDeclarationNode(ModuleVariableDeclarationNode moduleVariableDeclarationNode, + SyntaxNodeAnalysisContext ctx) { + Optional initializer = moduleVariableDeclarationNode.initializer(); + currentLocation = moduleVariableDeclarationNode.typedBindingPattern().typeDescriptor().location(); + if (initializer.isEmpty()) { + return; + } + ExpressionNode expressionNode = initializer.get(); + if (!isParseFunctionOfStringSource(expressionNode)) { + return; + } + + Optional symbol = semanticModel.symbol(moduleVariableDeclarationNode.typedBindingPattern()); + symbol.map(s -> (VariableSymbol) s).map(VariableSymbol::typeDescriptor) + .ifPresent(s -> { + validateExpectedType(s, currentLocation, ctx); + validateFunctionParameterTypes(expressionNode, s, expressionNode.location(), ctx); + }); + } + + private void processTypeDefinitionNode(TypeDefinitionNode typeDefinitionNode, SyntaxNodeAnalysisContext ctx) { + Node typeDescriptor = typeDefinitionNode.typeDescriptor(); + currentLocation = typeDefinitionNode.typeDescriptor().location(); + if (typeDescriptor.kind() != SyntaxKind.RECORD_TYPE_DESC) { + return; + } + validateRecordTypeDefinition(typeDefinitionNode, ctx); + } + + private void validateRecordTypeDefinition(TypeDefinitionNode typeDefinitionNode, SyntaxNodeAnalysisContext ctx) { + semanticModel.symbol(typeDefinitionNode) + .map(symbol -> (TypeDefinitionSymbol) symbol) + .ifPresent(typeDefinitionSymbol -> + detectDuplicateFields((RecordTypeSymbol) typeDefinitionSymbol.typeDescriptor(), ctx)); + } + + private void detectDuplicateFields(RecordTypeSymbol recordTypeSymbol, SyntaxNodeAnalysisContext ctx) { + Set fieldMembers = new HashSet<>(); + for (Map.Entry entry : recordTypeSymbol.fieldDescriptors().entrySet()) { + RecordFieldSymbol fieldSymbol = entry.getValue(); + String name = getNameFromAnnotation(entry.getKey(), fieldSymbol.annotAttachments()); + if (!fieldMembers.add(name)) { + reportDiagnosticInfo(ctx, fieldSymbol.getLocation(), CsvDataDiagnosticCodes.DUPLICATE_FIELD); + return; + } + } + } + + private String getNameFromAnnotation(String fieldName, + List annotationAttachments) { + for (AnnotationAttachmentSymbol annotAttSymbol : annotationAttachments) { + AnnotationSymbol annotation = annotAttSymbol.typeDescriptor(); + if (!getAnnotModuleName(annotation).contains(Constants.CSVDATA)) { + continue; + } + Optional nameAnnot = annotation.getName(); + if (nameAnnot.isEmpty()) { + continue; + } + String value = nameAnnot.get(); + if (value.equals(Constants.NAME)) { + return ((LinkedHashMap) annotAttSymbol.attachmentValue().orElseThrow().value()) + .get("value").toString(); + } + } + return fieldName; + } + + private String getAnnotModuleName(AnnotationSymbol annotation) { + return annotation.getModule().flatMap(ms -> ms.getName()).orElse(""); + } + + private boolean isCsvDataImport(ModuleSymbol moduleSymbol) { + ModuleID moduleId = moduleSymbol.id(); + return Constants.BALLERINA.equals(moduleId.orgName()) + && Constants.DATA_CSVDATA.equals(moduleId.moduleName()); + } +} diff --git a/compiler-plugin/src/main/java/module-info.java b/compiler-plugin/src/main/java/module-info.java index 60ac711..cf8cc26 100644 --- a/compiler-plugin/src/main/java/module-info.java +++ b/compiler-plugin/src/main/java/module-info.java @@ -16,7 +16,7 @@ * under the License. */ -module io.ballerina.stdlib.csvdata.compiler { +module io.ballerina.lib.csvdata.compiler { requires io.ballerina.lang; requires io.ballerina.tools.api; requires io.ballerina.parser; diff --git a/gradle.properties b/gradle.properties index ee77c50..cda8444 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,7 +1,8 @@ org.gradle.caching=true -group=io.ballerina.stdlib +group=io.ballerina.lib version=0.1.0-SNAPSHOT -ballerinaLangVersion=2201.9.0 +ballerinaLangVersion=2201.10.0-20240801-104200-87df251c +ballerinaTomlParserVersion=1.2.2 checkstyleToolVersion=10.12.0 puppycrawlCheckstyleVersion=10.12.0 @@ -11,7 +12,7 @@ spotbugsVersion=5.0.14 shadowJarPluginVersion=8.1.1 downloadPluginVersion=4.0.4 releasePluginVersion=2.8.0 -ballerinaGradlePluginVersion=2.0.1 +ballerinaGradlePluginVersion=2.2.6 javaJsonPathVersion=2.9.0 javaJsonSmartVersion=2.4.11 javaAccessorsSmartVersion=2.4.7 diff --git a/native/build.gradle b/native/build.gradle index 54266fa..2fef0f3 100644 --- a/native/build.gradle +++ b/native/build.gradle @@ -26,7 +26,6 @@ description = 'Ballerina - Data.CSV Java Utils' dependencies { implementation 'junit:junit:4.13.1' -// checkstyle project(':checkstyle') checkstyle "com.puppycrawl.tools:checkstyle:${puppycrawlCheckstyleVersion}" implementation 'org.apache.commons:commons-lang3:3.6' @@ -35,7 +34,6 @@ dependencies { implementation group: 'org.ballerinalang', name: 'value', version: "${ballerinaLangVersion}" implementation group: 'org.ballerinalang', name: 'value', version: "${ballerinaLangVersion}" implementation group: 'io.ballerina.stdlib', name: 'constraint-native', version: "${stdlibConstraintVersion}" - // ballerinaStdLibs "io.ballerina.stdlib:constraint-ballerina:${stdlibConstraintVersion}" } checkstyle { @@ -70,6 +68,28 @@ spotbugsTest { enabled = false } +publishing { + publications { + mavenJava(MavenPublication) { + groupId project.group + artifactId "data.csv-native" + version = project.version + artifact jar + } + } + + repositories { + maven { + name = "GitHubPackages" + url = uri("https://maven.pkg.github.com/ballerina-platform/module-ballerina-data.csv") + credentials { + username = System.getenv("publishUser") + password = System.getenv("publishPAT") + } + } + } +} + compileJava { doFirst { options.compilerArgs = [ diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/FromString.java b/native/src/main/java/io/ballerina/lib/data/csvdata/FromString.java new file mode 100644 index 0000000..c831bf9 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/FromString.java @@ -0,0 +1,386 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.data.csvdata; + +import io.ballerina.lib.data.csvdata.utils.Constants; +import io.ballerina.lib.data.csvdata.utils.CsvConfig; +import io.ballerina.lib.data.csvdata.utils.CsvUtils; +import io.ballerina.runtime.api.PredefinedTypes; +import io.ballerina.runtime.api.TypeTags; +import io.ballerina.runtime.api.creators.ErrorCreator; +import io.ballerina.runtime.api.creators.TypeCreator; +import io.ballerina.runtime.api.creators.ValueCreator; +import io.ballerina.runtime.api.types.FiniteType; +import io.ballerina.runtime.api.types.IntersectionType; +import io.ballerina.runtime.api.types.ReferenceType; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.types.UnionType; +import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.utils.TypeUtils; +import io.ballerina.runtime.api.values.BDecimal; +import io.ballerina.runtime.api.values.BError; +import io.ballerina.runtime.api.values.BString; + +import java.math.BigDecimal; +import java.text.DecimalFormatSymbols; +import java.text.NumberFormat; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.WeakHashMap; + +/** + * Native implementation of data:fromStringWithType(string). + * + * @since 0.1.0 + */ +public final class FromString { + private static WeakHashMap localeAttributes = new WeakHashMap<>(); + + private static final List TYPE_PRIORITY_ORDER = List.of( + TypeTags.INT_TAG, + TypeTags.FLOAT_TAG, + TypeTags.DECIMAL_TAG, + TypeTags.NULL_TAG, + TypeTags.BOOLEAN_TAG, + TypeTags.JSON_TAG, + TypeTags.STRING_TAG + ); + + private static final List BASIC_JSON_MEMBER_TYPES = List.of( + PredefinedTypes.TYPE_NULL, + PredefinedTypes.TYPE_BOOLEAN, + PredefinedTypes.TYPE_INT, + PredefinedTypes.TYPE_FLOAT, + PredefinedTypes.TYPE_DECIMAL, + PredefinedTypes.TYPE_STRING + ); + private static final UnionType JSON_TYPE_WITH_BASIC_TYPES = TypeCreator.createUnionType(BASIC_JSON_MEMBER_TYPES); + public static final Integer BBYTE_MIN_VALUE = 0; + public static final Integer BBYTE_MAX_VALUE = 255; + public static final Integer SIGNED32_MAX_VALUE = 2147483647; + public static final Integer SIGNED32_MIN_VALUE = -2147483648; + public static final Integer SIGNED16_MAX_VALUE = 32767; + public static final Integer SIGNED16_MIN_VALUE = -32768; + public static final Integer SIGNED8_MAX_VALUE = 127; + public static final Integer SIGNED8_MIN_VALUE = -128; + public static final Long UNSIGNED32_MAX_VALUE = 4294967295L; + public static final Integer UNSIGNED16_MAX_VALUE = 65535; + public static final Integer UNSIGNED8_MAX_VALUE = 255; + + private FromString() { + } + + public static Object fromStringWithType(BString string, Type expType, CsvConfig config) { + String value = string.getValue(); + + try { + return switch (expType.getTag()) { + case TypeTags.INT_TAG -> stringToInt(value, config); + case TypeTags.BYTE_TAG -> stringToByte(value, config); + case TypeTags.SIGNED8_INT_TAG -> stringToSigned8Int(value, config); + case TypeTags.SIGNED16_INT_TAG -> stringToSigned16Int(value, config); + case TypeTags.SIGNED32_INT_TAG -> stringToSigned32Int(value, config); + case TypeTags.UNSIGNED8_INT_TAG -> stringToUnsigned8Int(value, config); + case TypeTags.UNSIGNED16_INT_TAG -> stringToUnsigned16Int(value, config); + case TypeTags.UNSIGNED32_INT_TAG -> stringToUnsigned32Int(value, config); + case TypeTags.FLOAT_TAG -> stringToFloat(value, config); + case TypeTags.DECIMAL_TAG -> stringToDecimal(value, config); + case TypeTags.CHAR_STRING_TAG -> stringToChar(value); + case TypeTags.STRING_TAG -> string; + case TypeTags.BOOLEAN_TAG -> stringToBoolean(value); + case TypeTags.NULL_TAG -> stringToNull(value, config); + case TypeTags.FINITE_TYPE_TAG -> stringToFiniteType(value, (FiniteType) expType, config); + case TypeTags.UNION_TAG -> stringToUnion(string, (UnionType) expType, config, false); + case TypeTags.JSON_TAG, TypeTags.ANYDATA_TAG -> + stringToUnion(string, JSON_TYPE_WITH_BASIC_TYPES, config, true); + case TypeTags.TYPE_REFERENCED_TYPE_TAG -> + fromStringWithType(string, ((ReferenceType) expType).getReferredType(), config); + case TypeTags.INTERSECTION_TAG -> + fromStringWithType(string, ((IntersectionType) expType).getEffectiveType(), config); + default -> returnError(value, expType.toString()); + }; + } catch (ParseException | RuntimeException e) { + return returnError(value, expType.toString()); + } + } + + private static Object stringToFiniteType(String value, FiniteType finiteType, CsvConfig config) { + return finiteType.getValueSpace().stream() + .filter(finiteValue -> !(convertToSingletonValue(value, finiteValue, config) instanceof BError)) + .findFirst() + .orElseGet(() -> returnError(value, finiteType.toString())); + } + + private static Object convertToSingletonValue(String str, Object singletonValue, CsvConfig config) { + String singletonStr = String.valueOf(singletonValue); + if (str.equals(singletonStr)) { + return fromStringWithType(StringUtils.fromString(str), TypeUtils.getType(singletonValue), config); + } else { + return returnError(str, singletonStr); + } + } + + private static Long stringToInt(String value, CsvConfig config) throws NumberFormatException, ParseException { + Number number = parseNumberValue(value, config); + if (isIntegerValue(value, number, config.locale)) { + return number.longValue(); + } + throw new NumberFormatException(); + } + + private static int stringToByte(String value, CsvConfig config) throws NumberFormatException, ParseException { + Number number = parseNumberValue(value, config); + if (isIntegerValue(value, number, config.locale)) { + int intValue = number.intValue(); + if (isByteLiteral(intValue)) { + return intValue; + } + } + throw new NumberFormatException(); + } + + private static long stringToSigned8Int(String value, CsvConfig config) throws NumberFormatException, + ParseException { + Number number = parseNumberValue(value, config); + if (isIntegerValue(value, number, config.locale)) { + long intValue = number.longValue(); + if (isSigned8LiteralValue(intValue)) { + return intValue; + } + } + throw new NumberFormatException(); + } + + private static long stringToSigned16Int(String value, CsvConfig config) throws NumberFormatException, + ParseException { + + Number number = parseNumberValue(value, config); + if (isIntegerValue(value, number, config.locale)) { + long intValue = number.longValue(); + if (isSigned16LiteralValue(intValue)) { + return intValue; + } + } + throw new NumberFormatException(); + } + + private static long stringToSigned32Int(String value, CsvConfig config) throws NumberFormatException, + ParseException { + Number number = parseNumberValue(value, config); + if (isIntegerValue(value, number, config.locale)) { + long intValue = number.longValue(); + if (isSigned32LiteralValue(intValue)) { + return intValue; + } + } + throw new NumberFormatException(); + } + + private static long stringToUnsigned8Int(String value, CsvConfig config) + throws NumberFormatException, ParseException { + Number number = parseNumberValue(value, config); + if (isIntegerValue(value, number, config.locale)) { + long intValue = number.longValue(); + if (isUnsigned8LiteralValue(intValue)) { + return intValue; + } + } + throw new NumberFormatException(); + } + + private static long stringToUnsigned16Int(String value, CsvConfig config) + throws NumberFormatException, ParseException { + Number number = parseNumberValue(value, config); + if (isIntegerValue(value, number, config.locale)) { + long intValue = number.longValue(); + if (isUnsigned16LiteralValue(intValue)) { + return intValue; + } + } + throw new NumberFormatException(); + } + + private static long stringToUnsigned32Int(String value, CsvConfig config) + throws NumberFormatException, ParseException { + Number number = parseNumberValue(value, config); + if (isIntegerValue(value, number, config.locale)) { + long intValue = number.longValue(); + if (isUnsigned32LiteralValue(intValue)) { + return intValue; + } + } + throw new NumberFormatException(); + } + + private static BString stringToChar(String value) throws RuntimeException { + if (isCharLiteralValue(value)) { + return StringUtils.fromString(value); + } + throw new RuntimeException(); + } + + private static Double stringToFloat(String value, CsvConfig config) throws NumberFormatException, ParseException { + Number number = parseNumberValue(value, config); + if (isDoubleValue(value, number, config.locale)) { + return number.doubleValue(); + } + throw new NumberFormatException(); + } + + private static BDecimal stringToDecimal(String value, CsvConfig config) throws NumberFormatException, + ParseException { + Number number = parseNumberValue(value, config); + if (isDoubleValue(value, number, config.locale)) { + BigDecimal decimalValue = BigDecimal.valueOf(number.doubleValue()); + return ValueCreator.createDecimalValue(decimalValue); + } + throw new NumberFormatException(); + } + + private static Object stringToBoolean(String value) throws NumberFormatException { + if ("true".equalsIgnoreCase(value)) { + return true; + } + + if ("false".equalsIgnoreCase(value)) { + return false; + } + return returnError(value, "boolean"); + } + + private static Object stringToNull(String value, CsvConfig config) throws NumberFormatException { + Object nullValue = config.nilValue; + if (CsvUtils.isNullValue(nullValue, value)) { + return null; + } + return returnError(value, nullValue == null ? Constants.Values.BALLERINA_NULL : nullValue.toString()); + } + + private static Object stringToUnion(BString string, UnionType expType, CsvConfig config, boolean isJsonOrAnydata) + throws NumberFormatException { + List memberTypes = new ArrayList<>(expType.getMemberTypes()); + if (isJsonOrAnydata) { + memberTypes.sort(Comparator.comparingInt(t -> { + int index = TYPE_PRIORITY_ORDER.indexOf(TypeUtils.getReferredType(t).getTag()); + return index == -1 ? Integer.MAX_VALUE : index; + })); + } else { + memberTypes.sort(Comparator.comparingInt(t -> { + int tag = TypeUtils.getReferredType(t).getTag(); + return tag == TypeTags.NULL_TAG ? Integer.MIN_VALUE : memberTypes.indexOf(t); + })); + } + for (Type memberType : memberTypes) { + try { + Object result = fromStringWithType(string, memberType, config); + if (result instanceof BError) { + continue; + } + return result; + } catch (Exception e) { + // Skip + } + } + return returnError(string.getValue(), expType.toString()); + } + + private static boolean isByteLiteral(long longValue) { + return (longValue >= BBYTE_MIN_VALUE && longValue <= BBYTE_MAX_VALUE); + } + + private static boolean isSigned32LiteralValue(Long longObject) { + return (longObject >= SIGNED32_MIN_VALUE && longObject <= SIGNED32_MAX_VALUE); + } + + private static boolean isSigned16LiteralValue(Long longObject) { + return (longObject.intValue() >= SIGNED16_MIN_VALUE && longObject.intValue() <= SIGNED16_MAX_VALUE); + } + + private static boolean isSigned8LiteralValue(Long longObject) { + return (longObject.intValue() >= SIGNED8_MIN_VALUE && longObject.intValue() <= SIGNED8_MAX_VALUE); + } + + private static boolean isUnsigned32LiteralValue(Long longObject) { + return (longObject >= 0 && longObject <= UNSIGNED32_MAX_VALUE); + } + + private static boolean isUnsigned16LiteralValue(Long longObject) { + return (longObject.intValue() >= 0 && longObject.intValue() <= UNSIGNED16_MAX_VALUE); + } + + private static boolean isUnsigned8LiteralValue(Long longObject) { + return (longObject.intValue() >= 0 && longObject.intValue() <= UNSIGNED8_MAX_VALUE); + } + + private static boolean isCharLiteralValue(String value) { + return value.codePoints().count() == 1; + } + + private static boolean isIntegerValue(String value, Number number, String localeStr) { + return isNumericType(number) && value.matches(getLocale(localeStr).intRegex()); + } + + private static boolean isDoubleValue(String value, Number number, String localeStr) { + return (number instanceof Double || isNumericType(number)) + && value.matches(getLocale(localeStr).doubleRegex()); + } + + private static BError returnError(String value, String expType) { + return ErrorCreator.createError(StringUtils + .fromString("Cannot convert " + value + " to the expected type: " + expType)); + } + + private static Number parseNumberValue(String numberString, CsvConfig config) throws ParseException { + NumberFormat numberFormat = NumberFormat.getInstance(getLocale(config.locale).locale()); + return numberFormat.parse(numberString); + } + + private static LocaleInfo getLocale(String localeStr) { + if (!localeAttributes.containsKey(localeStr)) { + localeAttributes.put(localeStr, computeLocaleIfAbsent(localeStr)); + } + return localeAttributes.get(localeStr); + } + + private static LocaleInfo computeLocaleIfAbsent(String localeStr) { + Locale locale = CsvUtils.createLocaleFromString(localeStr); + DecimalFormatSymbols dfs = new DecimalFormatSymbols(locale); + char decimalSeparator = dfs.getDecimalSeparator(); + char minusSign = dfs.getMinusSign(); + char zeroDigit = dfs.getZeroDigit(); + String numRange = "[" + zeroDigit + "-9]"; + String exponentSeparator = dfs.getExponentSeparator(); + String intRegex = "^[" + minusSign + "+]?" + numRange + "+$"; + String doubleRegex = "^[" + minusSign + "+]?" + numRange + "+(" + + (decimalSeparator == '.' ? "\\." : decimalSeparator) + + numRange + "*)?(" + exponentSeparator + "[+-]?" + numRange + "+)?$"; + return new LocaleInfo(locale, intRegex, doubleRegex); + } + + private static boolean isNumericType(Object value) { + return value instanceof Integer || value instanceof Long || + value instanceof Short || value instanceof Byte; + } + + private record LocaleInfo(Locale locale, String intRegex, String doubleRegex) { + } +} diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvCreator.java b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvCreator.java new file mode 100644 index 0000000..7411e3e --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvCreator.java @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.data.csvdata.csv; + +import io.ballerina.lib.data.csvdata.FromString; +import io.ballerina.lib.data.csvdata.utils.CsvConfig; +import io.ballerina.lib.data.csvdata.utils.CsvUtils; +import io.ballerina.lib.data.csvdata.utils.DiagnosticErrorCode; +import io.ballerina.lib.data.csvdata.utils.DiagnosticLog; +import io.ballerina.runtime.api.PredefinedTypes; +import io.ballerina.runtime.api.TypeTags; +import io.ballerina.runtime.api.creators.ValueCreator; +import io.ballerina.runtime.api.flags.SymbolFlags; +import io.ballerina.runtime.api.types.ArrayType; +import io.ballerina.runtime.api.types.Field; +import io.ballerina.runtime.api.types.MapType; +import io.ballerina.runtime.api.types.TupleType; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.utils.TypeUtils; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BError; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; +import org.ballerinalang.langlib.value.CloneReadOnly; + +import java.util.Map; + +import static io.ballerina.lib.data.csvdata.utils.CsvUtils.getUpdatedHeaders; + +/** + * Create objects for partially parsed csv. + * + * @since 0.1.0 + */ +public final class CsvCreator { + private CsvCreator() { + } + + static Object initRowValue(Type expectedType) { + expectedType = TypeUtils.getReferredType(expectedType); + + return switch (expectedType.getTag()) { + case TypeTags.RECORD_TYPE_TAG -> + ValueCreator.createRecordValue(expectedType.getPackage(), expectedType.getName()); + case TypeTags.MAP_TAG -> ValueCreator.createMapValue((MapType) expectedType); + case TypeTags.TUPLE_TAG -> ValueCreator.createTupleValue((TupleType) expectedType); + case TypeTags.ARRAY_TAG -> ValueCreator.createArrayValue((ArrayType) expectedType); + default -> throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_TYPE, expectedType); + }; + } + + static void convertAndUpdateCurrentCsvNode(CsvParser.StateMachine sm, + String value, Type type, CsvConfig config, Type exptype, + Field currentField) { + Object currentCsv = sm.currentCsvNode; + Object nilValue = config.nilValue; + if (sm.config.nilAsOptionalField && !type.isNilable() + && CsvUtils.isNullValue(nilValue, value) + && currentField != null && SymbolFlags.isFlagOn(currentField.getFlags(), SymbolFlags.OPTIONAL)) { + return; + } + Object convertedValue = convertToExpectedType(StringUtils.fromString(value), type, config); + sm.isCurrentCsvNodeEmpty = false; + if (convertedValue instanceof BError || convertedValue instanceof CsvUtils.UnMappedValue) { + if (ignoreIncompatibilityErrorsForMaps(sm, exptype)) { + return; + } + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_CAST, value, type); + } + + Type currentCsvNodeType = TypeUtils.getType(currentCsv); + switch (currentCsvNodeType.getTag()) { + case TypeTags.MAP_TAG, TypeTags.RECORD_TYPE_TAG -> { + ((BMap) currentCsv).put(StringUtils.fromString(getHeaderValueForColumnIndex(sm)), + convertedValue); + sm.currentCsvNodeLength++; + } + case TypeTags.ARRAY_TAG -> { + ArrayType arrayType = (ArrayType) currentCsvNodeType; + if (arrayType.getState() == ArrayType.ArrayState.CLOSED && + arrayType.getSize() - 1 < sm.columnIndex) { + sm.isColumnMaxSizeReached = true; + } + ((BArray) currentCsv).add(sm.columnIndex, convertedValue); + sm.currentCsvNodeLength++; + } + case TypeTags.TUPLE_TAG -> { + ((BArray) currentCsv).add(sm.columnIndex, convertedValue); + sm.currentCsvNodeLength++; + } + } + } + + public static String getHeaderValueForColumnIndex(CsvParser.StateMachine sm) { + if (sm.config.customHeadersIfHeadersAbsent == null && (sm.config.header == null)) { + String header = String.valueOf(sm.columnIndex + 1); + Map fieldHierarchy = sm.fieldHierarchy; + fieldHierarchy.remove(header); + return header; + } + if (sm.columnIndex >= sm.headers.size()) { + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_CUSTOM_HEADER_LENGTH); + } + String header = sm.headers.get(sm.columnIndex); + return getUpdatedHeaders(sm.updatedRecordFieldNames, header, + sm.fields.contains(header)); + } + + public static void addCustomHeadersIfNotNull(CsvParser.StateMachine sm, Object customHeader) { + BArray customHeaders = (BArray) customHeader; + for (int i = 0; i < customHeaders.size(); i++) { + String header = StringUtils.getStringValue(customHeaders.get(i)); + Map fieldHierarchy = sm.fieldHierarchy; + sm.headers.add(header); + if (fieldHierarchy.containsKey(header)) { + Field field = fieldHierarchy.get(header); + sm.fieldNames.put(header, field); + fieldHierarchy.remove(header); + } + } + } + + private static boolean ignoreIncompatibilityErrorsForMaps(CsvParser.StateMachine sm, Type exptype) { + if (exptype.getTag() == TypeTags.RECORD_TYPE_TAG) { + String header = getHeaderValueForColumnIndex(sm); + Map fields = sm.fieldNames; + return !fields.containsKey(header); + } + return exptype.getTag() == TypeTags.MAP_TAG; + } + + public static Object convertToExpectedType(BString value, Type type, CsvConfig config) { + if (type.getTag() == TypeTags.ANYDATA_TAG) { + return FromString.fromStringWithType(value, PredefinedTypes.TYPE_JSON, config); + } + return FromString.fromStringWithType(value, type, config); + } + + public static Object constructReadOnlyValue(Object value) { + return CloneReadOnly.cloneReadOnly(value); + } +} diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvParser.java b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvParser.java new file mode 100644 index 0000000..1f9f650 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvParser.java @@ -0,0 +1,1163 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.data.csvdata.csv; + +import io.ballerina.lib.data.csvdata.utils.Constants; +import io.ballerina.lib.data.csvdata.utils.CsvConfig; +import io.ballerina.lib.data.csvdata.utils.CsvUtils; +import io.ballerina.lib.data.csvdata.utils.DataUtils; +import io.ballerina.lib.data.csvdata.utils.DiagnosticErrorCode; +import io.ballerina.lib.data.csvdata.utils.DiagnosticLog; +import io.ballerina.runtime.api.PredefinedTypes; +import io.ballerina.runtime.api.TypeTags; +import io.ballerina.runtime.api.creators.TypeCreator; +import io.ballerina.runtime.api.creators.ValueCreator; +import io.ballerina.runtime.api.flags.SymbolFlags; +import io.ballerina.runtime.api.types.ArrayType; +import io.ballerina.runtime.api.types.Field; +import io.ballerina.runtime.api.types.IntersectionType; +import io.ballerina.runtime.api.types.MapType; +import io.ballerina.runtime.api.types.RecordType; +import io.ballerina.runtime.api.types.TupleType; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.utils.TypeUtils; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BError; +import io.ballerina.runtime.api.values.BTypedesc; +import org.apache.commons.lang3.StringEscapeUtils; + +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import static io.ballerina.lib.data.csvdata.utils.Constants.EscapeChar.BACKSLASH_CHAR; +import static io.ballerina.lib.data.csvdata.utils.Constants.EscapeChar.BACKSPACE_CHAR; +import static io.ballerina.lib.data.csvdata.utils.Constants.EscapeChar.CARRIAGE_RETURN_CHAR; +import static io.ballerina.lib.data.csvdata.utils.Constants.EscapeChar.DOUBLE_QUOTES_CHAR; +import static io.ballerina.lib.data.csvdata.utils.Constants.EscapeChar.FORM_FEED_CHAR; +import static io.ballerina.lib.data.csvdata.utils.Constants.EscapeChar.NEWLINE_CHAR; +import static io.ballerina.lib.data.csvdata.utils.Constants.EscapeChar.SLASH_CHAR; +import static io.ballerina.lib.data.csvdata.utils.Constants.EscapeChar.TAB_CHAR; +import static io.ballerina.lib.data.csvdata.utils.Constants.EscapeChar.UNICODE_START_CHAR; + +/** + * Convert Csv string to a ballerina record. + * + * @since 0.1.0 + */ +public final class CsvParser { + + private static final char CR = 0x000D; + private static final char HZ_TAB = 0x0009; + private static final char SPACE = 0x0020; + private static final char BACKSPACE = 0x0008; + private static final char FORMFEED = 0x000C; + private static final char QUOTES = '"'; + private static final char REV_SOL = '\\'; + private static final char SOL = '/'; + private static final char EOF = (char) -1; + private static final char NEWLINE = 0x000A; + + private CsvParser() { + } + + // TODO: Add this implementation after creating the object pool implementation + // private static final ThreadLocal LOCAL_THREAD_STATE_MACHINE + // = ThreadLocal.withInitial(StateMachine::new); + + public static Object parse(Reader reader, BTypedesc type, CsvConfig config) + throws BError { + // TODO: Add this implementation after creating the object pool implementation + // StateMachine sm = LOCAL_THREAD_STATE_MACHINE.get(); + StateMachine sm = new StateMachine(); + try { + CsvUtils.validateConfigs(config); + Object convertedValue = sm.execute(reader, TypeUtils.getReferredType(type.getDescribingType()), + config, type); + return DataUtils.validateConstraints(convertedValue, type, config.enableConstraintValidation); + } finally { + // Need to reset the state machine before leaving. Otherwise, references to the created + // CSV values will be maintained and the java GC will not happen properly. + sm.reset(); + } + } + + /** + * Represents the state machine used for CSV parsing. + */ + + static class StateMachine { + + private static final State HEADER_START_STATE = new HeaderStartState(); + private static final State HEADER_END_STATE = new HeaderEndState(); + private static final State ROW_START_STATE = new RowStartState(); + private static final State ROW_END_STATE = new RowEndState(); + private static final State STRING_ESCAPE_VALUE_STATE = new StringValueEscapedCharacterProcessingState(); + private static final State STRING_UNICODE_CHAR_STATE = new StringValueUnicodeHexProcessingState(); + private static final State HEADER_UNICODE_CHAR_STATE = new HeaderUnicodeHexProcessingState(); + private static final State HEADER_ESCAPE_CHAR_STATE = new HeaderEscapedCharacterProcessingState(); + private static final State STRING_QUOTE_CHAR_STATE = new StringQuoteValueState(); + private static final State HEADER_QUOTE_CHAR_STATE = new HeaderQuoteValueState(); + private static final char LINE_BREAK = '\n'; + + Object currentCsvNode; + ArrayList headers = new ArrayList<>(); + BArray rootCsvNode; + Map fieldHierarchy = new HashMap<>(); + Map updatedRecordFieldNames = new HashMap<>(); + HashSet fields = new HashSet<>(); + Map fieldNames = new HashMap<>(); + private char[] charBuff = new char[1024]; + private int charBuffIndex; + private int index; + private int line; + private int column; + Type restType; + Type expectedArrayElementType; + int columnIndex = 0; + int rowIndex = 1; + int lineNumber = 0; + ArrayType rootArrayType = null; + CsvConfig config = null; + boolean skipTheRow = false; + boolean insideComment = false; + boolean isCurrentCsvNodeEmpty = true; + boolean isHeaderConfigExceedLineNumber = false; + boolean isQuoteClosed = false; + private StringBuilder hexBuilder = new StringBuilder(4); + boolean isValueStart = false; + State prevState; + int arraySize = 0; + boolean addHeadersForOutput = false; + int currentCsvNodeLength = 0; + boolean isColumnMaxSizeReached = false; + boolean isRowMaxSizeReached = false; + + boolean isCarriageTokenPresent = false; + + StateMachine() { + reset(); + } + + public void reset() { + currentCsvNode = null; + headers = new ArrayList<>(); + rootCsvNode = null; + fieldHierarchy.clear(); + updatedRecordFieldNames.clear(); + fields.clear(); + fieldNames.clear(); + charBuff = new char[1024]; + charBuffIndex = 0; + index = 0; + line = 1; + column = 0; + restType = null; + expectedArrayElementType = null; + columnIndex = 0; + rowIndex = 1; + lineNumber = 0; + rootArrayType = null; + config = null; + skipTheRow = false; + insideComment = false; + isCurrentCsvNodeEmpty = true; + isHeaderConfigExceedLineNumber = false; + isQuoteClosed = false; + hexBuilder = new StringBuilder(4); + isValueStart = false; + prevState = null; + arraySize = 0; + addHeadersForOutput = false; + currentCsvNodeLength = 0; + isColumnMaxSizeReached = false; + isRowMaxSizeReached = false; + isCarriageTokenPresent = false; + } + + private boolean isWhitespace(char ch, Object lineTerminator) { + return ch == SPACE || ch == HZ_TAB || ch == CR + || CsvUtils.isCharContainsInLineTerminatorUserConfig(ch, lineTerminator, isCarriageTokenPresent); + } + + private static void throwExpected(String... chars) throws CsvParserException { + throw new CsvParserException("expected '" + String.join("' or '", chars) + "'"); + } + + private void processLocation(char ch) { + if (ch == LINE_BREAK) { + this.line++; + this.column = 0; + } else { + this.column++; + } + } + + private String value() { + if (this.charBuffIndex == 0) { + return ""; + } + String result = new String(this.charBuff, 0, this.charBuffIndex); + this.charBuffIndex = 0; + return result; + } + + private void clear() { + this.charBuffIndex = 0; + } + + private String peek() { + return new String(this.charBuff, 0, this.charBuffIndex); + } + + public Object execute(Reader reader, Type type, CsvConfig config, BTypedesc bTypedesc) throws BError { + this.config = config; + Type referredType = TypeUtils.getReferredType(type); + if (referredType.getTag() == TypeTags.INTERSECTION_TAG) { + for (Type constituentType : ((IntersectionType) referredType).getConstituentTypes()) { + if (constituentType.getTag() == TypeTags.READONLY_TAG) { + continue; + } + return CsvCreator.constructReadOnlyValue(execute(reader, constituentType, config, bTypedesc)); + } + } + + if (referredType.getTag() == TypeTags.UNION_TAG) { + expectedArrayElementType = referredType; + } else if (referredType.getTag() != TypeTags.ARRAY_TAG) { + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_TYPE, type); + } else { + rootArrayType = (ArrayType) referredType; + expectedArrayElementType = TypeUtils.getReferredType(rootArrayType.getElementType()); + rootCsvNode = ValueCreator.createArrayValue(rootArrayType); + } + + switch (expectedArrayElementType.getTag()) { + case TypeTags.RECORD_TYPE_TAG: + RecordType recordType = (RecordType) expectedArrayElementType; + restType = (recordType).getRestFieldType(); + fieldHierarchy = new HashMap<>(recordType.getFields()); + fields = new HashSet<>(recordType.getFields().keySet()); + updatedRecordFieldNames = CsvUtils + .processNameAnnotationsAndBuildCustomFieldMap(recordType, fieldHierarchy); + break; + case TypeTags.TUPLE_TAG: + restType = ((TupleType) expectedArrayElementType).getRestType(); + break; + case TypeTags.MAP_TAG: + case TypeTags.ARRAY_TAG: + break; + case TypeTags.INTERSECTION_TAG: + for (Type constituentType : ((IntersectionType) expectedArrayElementType).getConstituentTypes()) { + if (constituentType.getTag() == TypeTags.READONLY_TAG) { + continue; + } + Object mapValue = execute(reader, TypeCreator.createArrayType( + TypeCreator.createMapType(PredefinedTypes.TYPE_STRING) + ), CsvConfig.createConfigOptionsForUnion(config), bTypedesc); + config.stringConversion = true; + return CsvCreator.constructReadOnlyValue(CsvTraversal + .traverse((BArray) mapValue, config, bTypedesc, + TypeCreator.createArrayType(constituentType))); + } + throw DiagnosticLog.error(DiagnosticErrorCode.SOURCE_CANNOT_CONVERT_INTO_EXP_TYPE, + expectedArrayElementType); + case TypeTags.UNION_TAG: + boolean outputHeaders = config.outputWithHeaders; + Object customHeaders = config.customHeadersIfHeadersAbsent; + Object mapValue = execute(reader, TypeCreator.createArrayType( + TypeCreator.createMapType(PredefinedTypes.TYPE_STRING) + ), CsvConfig.createConfigOptionsForUnion(config), bTypedesc); + config.stringConversion = true; + config.outputWithHeaders = outputHeaders; + if (config.outputWithHeaders && customHeaders == null) { + config.customHeadersIfHeadersAbsent = this.headers; + } + if (customHeaders != null) { + config.customHeadersIfHeadersAbsent = customHeaders; + } + return CsvTraversal.traverse((BArray) mapValue, config, bTypedesc); + default: + throw DiagnosticLog.error(DiagnosticErrorCode.SOURCE_CANNOT_CONVERT_INTO_EXP_TYPE, + expectedArrayElementType); + } + + State currentState; + if (config.header != null) { + currentState = HEADER_START_STATE; + } else { + Object customHeadersIfHeadersAbsent = config.customHeadersIfHeadersAbsent; + if (customHeadersIfHeadersAbsent != null) { + CsvCreator.addCustomHeadersIfNotNull(this, customHeadersIfHeadersAbsent); + } + currentState = ROW_START_STATE; + addFieldNamesForNonHeaderState(); + } + try { + char[] buff = new char[1024]; + int count; + while ((count = reader.read(buff)) > 0) { + this.index = 0; + while (this.index < count) { + currentState = currentState.transition(this, buff, this.index, count); + } + } + currentState = currentState.transition(this, new char[]{EOF}, 0, 1); + if (currentState != ROW_END_STATE && currentState != HEADER_END_STATE) { + if (!this.isHeaderConfigExceedLineNumber) { + throw new CsvParserException("Invalid token found"); + } + } + return rootCsvNode; + } catch (IOException | CsvParserException e) { + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_TOKEN, e.getMessage(), line, column); + } + } + + private void addFieldNamesForNonHeaderState() { + this.fieldNames.putAll(this.fieldHierarchy); + } + + private void append(char ch) { + try { + this.charBuff[this.charBuffIndex] = ch; + this.charBuffIndex++; + } catch (ArrayIndexOutOfBoundsException e) { + /* this approach is faster than checking for the size by ourself */ + this.growCharBuff(); + this.charBuff[this.charBuffIndex++] = ch; + } + } + + private boolean isNewLineOrEof(char ch) { + return ch == EOF || CsvUtils.isCharContainsInLineTerminatorUserConfig(ch, + config.lineTerminator, isCarriageTokenPresent); + } + + private void growCharBuff() { + char[] newBuff = new char[charBuff.length * 2]; + System.arraycopy(this.charBuff, 0, newBuff, 0, this.charBuff.length); + this.charBuff = newBuff; + } + + /** + * A specific state in the CSV parsing state machine. + */ + interface State { + + State transition(StateMachine sm, char[] buff, int i, int count) throws CsvParserException; + } + + /** + * Represents the CSV header start state. + */ + private static class HeaderStartState implements State { + + @Override + public State transition(StateMachine sm, char[] buff, int i, int count) throws CsvParserException { + char ch; + State state = HEADER_START_STATE; + char separator = sm.config.delimiter; + int headerStartRowNumber = getHeaderStartRowWhenHeaderIsPresent(sm.config.header); + for (; i < count; i++) { + ch = buff[i]; + sm.processLocation(ch); + if (ch == EOF) { + handleEndOfTheHeader(sm); + return HEADER_END_STATE; + } + if (ch == Constants.LineTerminator.CR) { + sm.isCarriageTokenPresent = true; + continue; + } else if (!(sm.isCarriageTokenPresent && ch == Constants.LineTerminator.LF)) { + sm.isCarriageTokenPresent = true; + } + + if (sm.lineNumber < headerStartRowNumber) { + sm.isHeaderConfigExceedLineNumber = true; + if (sm.isNewLineOrEof(ch)) { + sm.lineNumber++; + } + continue; + } + sm.isHeaderConfigExceedLineNumber = false; + + if (ch == sm.config.comment) { + sm.insideComment = true; + state = this; + } else if (!sm.insideComment && ch == separator) { + addHeader(sm); + sm.columnIndex++; + state = this; + continue; + } else if (!sm.insideComment && ch == sm.config.textEnclosure) { + sm.prevState = this; + state = HEADER_QUOTE_CHAR_STATE; + break; + } else if (!sm.insideComment && ch == sm.config.escapeChar) { + sm.prevState = this; + state = HEADER_ESCAPE_CHAR_STATE; + break; + } else if (sm.insideComment && sm.isNewLineOrEof(ch)) { + sm.insideComment = false; + handleEndOfTheHeader(sm); + state = HEADER_END_STATE; + } else if (!sm.insideComment && isEndOfTheHeaderRow(sm, ch)) { + handleEndOfTheHeader(sm); + state = HEADER_END_STATE; + } else if (sm.isWhitespace(ch, sm.config.lineTerminator)) { + if (sm.isValueStart) { + sm.append(ch); + } + state = this; + continue; + } else { + if (!sm.insideComment) { + sm.append(ch); + sm.isValueStart = true; + } + state = this; + continue; + } + break; + } + sm.index = i + 1; + return state; + } + } + + private static void handleEndOfTheHeader(StateMachine sm) throws CsvParserException { + handleEndOfTheHeader(sm, true); + } + + private static void handleEndOfTheHeader(StateMachine sm, boolean trim) throws CsvParserException { + sm.isValueStart = false; + addHeader(sm, trim); + finalizeHeaders(sm); + sm.columnIndex = 0; + sm.lineNumber++; + } + + private static int getHeaderStartRowWhenHeaderIsPresent(Object header) { + return ((Long) header).intValue(); + } + + private static void finalizeHeaders(StateMachine sm) throws CsvParserException { + if (sm.headers.isEmpty()) { + throw DiagnosticLog.error(DiagnosticErrorCode.HEADER_CANNOT_BE_EMPTY); + } + Type expType = sm.expectedArrayElementType; + if (expType instanceof RecordType) { + validateRemainingRecordFields(sm); + } else if (expType instanceof ArrayType arrayType) { + CsvUtils.validateExpectedArraySize(arrayType.getSize(), sm.headers.size()); + } else if (expType instanceof MapType) { + //ignore + } else if (expType instanceof TupleType tupleType) { + validateTupleTypes(tupleType, sm.restType, sm.headers.size()); + } else { + throw new CsvParserException("Invalid expected type"); + } + } + + private static void validateTupleTypes(TupleType tupleType, Type restType, int currentSize) { + if (restType != null && tupleType.getTupleTypes().size() > currentSize) { + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_EXPECTED_TUPLE_SIZE, currentSize); + } + } + + private static void validateRemainingRecordFields(StateMachine sm) { + if (sm.restType == null) { + for (Field field : sm.fieldHierarchy.values()) { + if (sm.config.absentAsNilableType && field.getFieldType().isNilable()) { + return; + } + if (SymbolFlags.isFlagOn(field.getFlags(), SymbolFlags.REQUIRED)) { + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_FIELD_IN_CSV, field.getFieldName()); + } + } + } + } + + private static void addHeader(StateMachine sm) { + addHeader(sm, true); + } + + private static void addHeader(StateMachine sm, boolean trim) { + sm.isValueStart = false; + String value = sm.value(); + if (trim) { + value = value.trim(); + } + if (value.isEmpty()) { + throw DiagnosticLog.error(DiagnosticErrorCode.HEADER_CANNOT_BE_EMPTY); + } + if (sm.expectedArrayElementType instanceof RecordType) { + String fieldName = CsvUtils.getUpdatedHeaders( + sm.updatedRecordFieldNames, value, sm.fields.contains(value)); + Field field = sm.fieldHierarchy.get(fieldName); + if (field != null) { + sm.fieldNames.put(fieldName, field); + sm.fieldHierarchy.remove(fieldName); + } + } + if (sm.headers.contains(value)) { + throw DiagnosticLog.error(DiagnosticErrorCode.DUPLICATE_HEADER, value); + } + sm.headers.add(value); + } + + /** + * Represents the CSV header end state. + */ + private static class HeaderEndState implements State { + + @Override + public State transition(StateMachine sm, char[] buff, int i, int count) { + return ROW_START_STATE; + } + } + + /** + * Represents the CSV row start state. + */ + private static class RowStartState implements State { + + char ch; + State state = ROW_START_STATE; + + @Override + public State transition(StateMachine sm, char[] buff, int i, int count) { + char separator = sm.config.delimiter; + long[] skipLines = CsvUtils.getSkipDataRows(sm.config.skipLines); + + for (; i < count; i++) { + ch = buff[i]; + sm.processLocation(ch); + if (sm.isRowMaxSizeReached) { + if (ch == EOF) { + state = ROW_END_STATE; + break; + } + continue; + } + if (ch == Constants.LineTerminator.CR) { + sm.isCarriageTokenPresent = true; + continue; + } else if (!(sm.isCarriageTokenPresent && ch == Constants.LineTerminator.LF)) { + sm.isCarriageTokenPresent = false; + } + + if (sm.skipTheRow) { + if (isEndOfTheRowAndValueIsNotEmpty(sm, ch)) { + sm.insideComment = false; + sm.skipTheRow = false; + sm.clear(); + if (ch == EOF) { + return ROW_END_STATE; + } + } else { + sm.append(ch); + sm.isValueStart = true; + } + continue; + } + + if (sm.isCurrentCsvNodeEmpty) { + if (ignoreRow(skipLines, sm.rowIndex)) { + updateLineAndColumnIndexes(sm); + sm.skipTheRow = true; + continue; + } + initiateNewRowType(sm); + addHeadersAsTheFirstElementForArraysIfApplicable(sm); + } + if (!sm.insideComment && ch == sm.config.comment) { + handleEndOfTheRow(sm); + sm.insideComment = true; + state = this; + } else if (!sm.insideComment && ch == separator) { + addRowValue(sm); + state = this; + } else if (!sm.insideComment && ch == sm.config.textEnclosure) { + sm.prevState = this; + state = STRING_QUOTE_CHAR_STATE; + break; + } else if (!sm.insideComment && ch == sm.config.escapeChar) { + sm.prevState = this; + state = STRING_ESCAPE_VALUE_STATE; + break; + } else if (sm.insideComment && sm.isNewLineOrEof(ch)) { + sm.insideComment = false; + if (ch == EOF) { + state = ROW_END_STATE; + break; + } + } else if (isEndOfTheRowAndValueIsNotEmpty(sm, ch)) { + handleEndOfTheRow(sm); + if (ch == EOF) { + state = ROW_END_STATE; + break; + } + } else if (sm.isWhitespace(ch, sm.config.lineTerminator)) { + if (sm.isValueStart) { + sm.append(ch); + } + state = this; + // ignore + } else { + if (!sm.insideComment) { + sm.append(ch); + sm.isValueStart = true; + } + state = this; + } + } + if (state == null) { + state = this; + } + sm.index = i + 1; + return state; + } + } + + private static void handleEndOfTheRow(StateMachine sm) { + handleEndOfTheRow(sm, true); + } + + private static void handleEndOfTheRow(StateMachine sm, boolean trim) { + sm.isValueStart = false; + handleCsvRow(sm, trim); + CsvUtils.checkRequiredFieldsAndLogError(sm.fieldHierarchy, sm.config.absentAsNilableType); + } + + private static void handleCsvRow(StateMachine sm, boolean trim) { + String value = sm.peek(); + if (trim) { + value = value.trim(); + } + if (!(value.isBlank() && sm.currentCsvNodeLength == 0) + && !sm.isColumnMaxSizeReached && !sm.isRowMaxSizeReached) { + addRowValue(sm, trim); + } + if (!sm.isCurrentCsvNodeEmpty) { + finalizeTheRow(sm); + updateLineAndColumnIndexes(sm); + } else { + updateLineAndColumnIndexesWithoutRowIndexes(sm); + } + } + + private static void updateLineAndColumnIndexes(StateMachine sm) { + sm.rowIndex++; + updateLineAndColumnIndexesWithoutRowIndexes(sm); + } + + private static void updateLineAndColumnIndexesWithoutRowIndexes(StateMachine sm) { + sm.lineNumber++; + sm.currentCsvNode = null; + sm.isCurrentCsvNodeEmpty = true; + sm.columnIndex = 0; + sm.isColumnMaxSizeReached = false; + sm.clear(); + } + + private static boolean ignoreRow(long[] skipLines, int lineNumber) { + for (long skipLine : skipLines) { + if (skipLine == lineNumber) { + return true; + } + } + return false; + } + + private static void initiateNewRowType(StateMachine sm) { + sm.currentCsvNode = CsvCreator.initRowValue(sm.expectedArrayElementType); + } + + private static void addHeadersAsTheFirstElementForArraysIfApplicable(StateMachine sm) { + if (!sm.addHeadersForOutput && CsvUtils + .isExpectedTypeIsArray(sm.expectedArrayElementType) && sm.config.outputWithHeaders) { + ArrayList headers = sm.headers; + if (!headers.isEmpty()) { + for (String header : headers) { + addHeaderAsRowValue(sm, header); + } + if (!sm.isCurrentCsvNodeEmpty) { + finalizeTheRow(sm); + initiateNewRowType(sm); + } + } + sm.addHeadersForOutput = true; + sm.columnIndex = 0; + } + } + + private static void finalizeTheRow(StateMachine sm) { + int rootArraySize = sm.rootArrayType.getSize(); + if (rootArraySize == -1) { + sm.rootCsvNode.append(sm.currentCsvNode); + } else if (sm.arraySize < rootArraySize) { + sm.rootCsvNode.add(sm.arraySize, sm.currentCsvNode); + } + sm.arraySize++; + sm.currentCsvNodeLength = 0; + if (sm.arraySize == rootArraySize) { + sm.isRowMaxSizeReached = true; + } + } + + private static void addRowValue(StateMachine sm) { + addRowValue(sm, true); + } + + private static void addRowValue(StateMachine sm, boolean trim) { + if (sm.isColumnMaxSizeReached || sm.isRowMaxSizeReached) { + return; + } + Field currentField = null; + sm.isValueStart = false; + Type exptype = sm.expectedArrayElementType; + String value = sm.value(); + if (trim) { + value = value.trim(); + } + + Type type = getExpectedRowType(sm, exptype); + + if (exptype instanceof RecordType) { + currentField = getCurrentField(sm); + } + + if (type != null) { + CsvCreator.convertAndUpdateCurrentCsvNode(sm, + value, type, sm.config, exptype, currentField); + } + sm.columnIndex++; + } + + private static void addHeaderAsRowValue(StateMachine sm, String value) { + Type exptype = sm.expectedArrayElementType; + Field currentField = null; + Type type = getExpectedRowType(sm, exptype); + + if (exptype instanceof RecordType) { + currentField = getCurrentField(sm); + } + + if (type != null) { + CsvCreator.convertAndUpdateCurrentCsvNode(sm, + value, type, sm.config, exptype, currentField); + } + sm.columnIndex++; + } + + private static Type getExpectedRowType(StateMachine sm, Type exptype) { + if (exptype instanceof RecordType) { + return getExpectedRowTypeOfRecord(sm); + } else if (exptype instanceof MapType mapType) { + return (mapType.getConstrainedType()); + } else if (exptype instanceof ArrayType arrayType) { + return getExpectedRowTypeOfArray(sm, arrayType); + } else if (exptype instanceof TupleType tupleType) { + return getExpectedRowTypeOfTuple(sm, tupleType); + } + return null; + } + + private static Type getExpectedRowTypeOfTuple(StateMachine sm, TupleType tupleType) { + List tupleTypes = tupleType.getTupleTypes(); + if (tupleTypes.size() > sm.columnIndex) { + return tupleTypes.get(sm.columnIndex); + } else { + Type restType = sm.restType; + if (restType != null) { + return restType; + } else { + sm.charBuffIndex = 0; + if (sm.config.allowDataProjection) { + sm.isColumnMaxSizeReached = true; + return null; + } + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_EXPECTED_TUPLE_SIZE, tupleTypes.size()); + } + } + } + + private static Type getExpectedRowTypeOfArray(StateMachine sm, ArrayType arrayType) { + if (arrayType.getSize() != -1 && arrayType.getSize() <= sm.columnIndex) { + sm.charBuffIndex = 0; + if (sm.config.allowDataProjection) { + sm.isColumnMaxSizeReached = true; + return null; + } + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_EXPECTED_ARRAY_SIZE, arrayType.getSize()); + } + return arrayType.getElementType(); + } + + private static Type getExpectedRowTypeOfRecord(StateMachine sm) { + String header = CsvCreator.getHeaderValueForColumnIndex(sm); + Map fields = sm.fieldNames; + if (fields.containsKey(header)) { + return fields.get(header).getFieldType(); + } else { + Type restType = sm.restType; + if (restType != null) { + return restType; + } else { + sm.charBuffIndex = 0; + if (sm.config.allowDataProjection) { + return null; + } + throw DiagnosticLog.error(DiagnosticErrorCode.NO_FIELD_FOR_HEADER, header); + } + } + } + + private static Field getCurrentField(StateMachine sm) { + String header = CsvCreator.getHeaderValueForColumnIndex(sm); + Map fields = sm.fieldNames; + if (fields.containsKey(header)) { + return fields.get(header); + } + return null; + } + + /** + * Represents the CSV row end state. + */ + private static class RowEndState implements State { + + @Override + public State transition(StateMachine sm, char[] buff, int i, int count) { + return ROW_END_STATE; + } + } + + /** + * Represents the CSV row value with quote state. + */ + private static class StringQuoteValueState implements State { + + @Override + public State transition(StateMachine sm, char[] buff, int i, int count) { + State state = this; + char ch; + for (; i < count; i++) { + ch = buff[i]; + sm.processLocation(ch); + if (ch == EOF) { + handleEndOfTheRow(sm, false); + return ROW_END_STATE; + } + if (ch == Constants.LineTerminator.CR) { + sm.isCarriageTokenPresent = true; + continue; + } else if (!(sm.isCarriageTokenPresent && ch == Constants.LineTerminator.LF)) { + sm.isCarriageTokenPresent = false; + } + + if (ch == sm.config.textEnclosure) { + if (sm.isQuoteClosed) { + sm.append(ch); + sm.isValueStart = true; + continue; + } + sm.isQuoteClosed = true; + } else if (ch == sm.config.delimiter && sm.isQuoteClosed) { + addRowValue(sm, false); + state = ROW_START_STATE; + sm.isQuoteClosed = false; + break; + } else if (sm.isNewLineOrEof(ch) && sm.isQuoteClosed) { + handleEndOfTheRow(sm, false); + state = ROW_START_STATE; + sm.isQuoteClosed = false; + break; + } else if (ch == sm.config.escapeChar) { + state = STRING_ESCAPE_VALUE_STATE; + sm.prevState = this; + sm.isQuoteClosed = false; + break; + } else { + if (!sm.isQuoteClosed) { + sm.append(ch); + } else { + sm.append(sm.config.textEnclosure); + sm.append(ch); + sm.isQuoteClosed = false; + } + sm.isValueStart = true; + state = this; + } + } + if (state == null) { + state = this; + } + sm.index = i + 1; + return state; + } + } + + /** + * Represents the CSV header value with quote state. + */ + private static class HeaderQuoteValueState implements State { + + @Override + public State transition(StateMachine sm, char[] buff, int i, int count) + throws CsvParserException { + State state = this; + char ch; + for (; i < count; i++) { + ch = buff[i]; + sm.processLocation(ch); + if (ch == EOF) { + handleEndOfTheRow(sm); + return ROW_END_STATE; + } + if (ch == Constants.LineTerminator.CR) { + sm.isCarriageTokenPresent = true; + continue; + } else if (!(sm.isCarriageTokenPresent && ch == Constants.LineTerminator.LF)) { + sm.isCarriageTokenPresent = false; + } + + if (ch == sm.config.textEnclosure) { + sm.isQuoteClosed = true; + } else if (ch == sm.config.delimiter && sm.isQuoteClosed) { + addHeader(sm, false); + sm.columnIndex++; + sm.isQuoteClosed = false; + state = HEADER_START_STATE; + break; + } else if (sm.isNewLineOrEof(ch) && sm.isQuoteClosed) { + handleEndOfTheHeader(sm, false); + state = HEADER_END_STATE; + sm.isQuoteClosed = false; + break; + } else if (!sm.isQuoteClosed && ch == sm.config.escapeChar) { + sm.isQuoteClosed = false; + sm.prevState = this; + state = HEADER_ESCAPE_CHAR_STATE; + break; + } else { + if (!sm.isQuoteClosed) { + sm.append(ch); + } else { + sm.append(sm.config.textEnclosure); + sm.append(ch); + sm.isQuoteClosed = false; + } + sm.isValueStart = true; + state = this; + continue; + } + break; + } + sm.index = i + 1; + return state; + } + } + + /** + * Represents the state where an escaped unicode character in hex format is processed from a row value. + */ + private static class StringValueUnicodeHexProcessingState extends UnicodeHexProcessingState { + + @Override + protected State getSourceState() { + return STRING_UNICODE_CHAR_STATE; + } + + } + + /** + * Represents the state where an escaped unicode character in hex format is processed from a header name. + */ + private static class HeaderUnicodeHexProcessingState extends UnicodeHexProcessingState { + + @Override + protected State getSourceState() { + return HEADER_UNICODE_CHAR_STATE; + } + } + + /** + * Represents the state where an escaped unicode character in hex format is processed. + */ + private abstract static class UnicodeHexProcessingState implements State { + + protected abstract State getSourceState(); + + @Override + public State transition(StateMachine sm, char[] buff, int i, int count) throws CsvParserException { + State state = null; + char ch; + for (; i < count; i++) { + ch = buff[i]; + sm.processLocation(ch); + if (ch == EOF) { + handleEndOfTheRow(sm); + return ROW_END_STATE; + } + if (ch == Constants.LineTerminator.CR) { + sm.isCarriageTokenPresent = true; + continue; + } else if (!(sm.isCarriageTokenPresent && ch == Constants.LineTerminator.LF)) { + sm.isCarriageTokenPresent = false; + } + + if ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'F') || (ch >= 'a' && ch <= 'f')) { + sm.hexBuilder.append(ch); + sm.isValueStart = true; + if (sm.hexBuilder.length() >= 4) { + sm.append(this.extractUnicodeChar(sm)); + this.reset(sm); + state = sm.prevState; + sm.prevState = this; + break; + } + state = this; + continue; + } + this.reset(sm); + StateMachine.throwExpected("hexadecimal value of an unicode character"); + break; + } + if (state == null) { + state = this; + } + sm.index = i + 1; + return state; + } + + private void reset(StateMachine sm) { + sm.hexBuilder.setLength(0); + } + + private char extractUnicodeChar(StateMachine sm) { + return StringEscapeUtils.unescapeJava("\\u" + sm.hexBuilder.toString()).charAt(0); + } + } + + /** + * Represents the state where an escaped character is processed in a header value. + */ + private static class HeaderEscapedCharacterProcessingState extends EscapedCharacterProcessingState { + + @Override + protected State getSourceState() { + return HEADER_ESCAPE_CHAR_STATE; + } + } + + /** + * Represents the state where an escaped character is processed in a row value. + */ + private static class StringValueEscapedCharacterProcessingState extends EscapedCharacterProcessingState { + + @Override + protected State getSourceState() { + return STRING_ESCAPE_VALUE_STATE; + } + } + + /** + * Represents the state where an escaped character is processed in a header or row value. + */ + private abstract static class EscapedCharacterProcessingState implements State { + + static final Map ESCAPE_CHAR_MAP = Map.of(DOUBLE_QUOTES_CHAR, QUOTES, + BACKSLASH_CHAR, REV_SOL, SLASH_CHAR, SOL, BACKSPACE_CHAR, BACKSPACE, FORM_FEED_CHAR, + FORMFEED, NEWLINE_CHAR, NEWLINE, CARRIAGE_RETURN_CHAR, CR, TAB_CHAR, HZ_TAB); + + protected abstract State getSourceState(); + + @Override + public State transition(StateMachine sm, char[] buff, int i, int count) throws CsvParserException { + State state = null; + char ch; + if (i < count) { + ch = buff[i]; + sm.processLocation(ch); + if (ch == Constants.LineTerminator.CR) { + sm.isCarriageTokenPresent = true; + } else if (!(sm.isCarriageTokenPresent && ch == Constants.LineTerminator.LF)) { + sm.isCarriageTokenPresent = false; + } + if (ch == EOF) { + handleEndOfTheRow(sm); + return ROW_END_STATE; + } + switch (ch) { + case DOUBLE_QUOTES_CHAR: + case BACKSLASH_CHAR: + case SLASH_CHAR: + case BACKSPACE_CHAR: + case FORM_FEED_CHAR: + case NEWLINE_CHAR: + case CARRIAGE_RETURN_CHAR: + case TAB_CHAR: + sm.append(ESCAPE_CHAR_MAP.get(ch)); + state = sm.prevState; + break; + case UNICODE_START_CHAR: + if (this.getSourceState() == STRING_ESCAPE_VALUE_STATE) { + state = STRING_UNICODE_CHAR_STATE; + } else if (this.getSourceState() == HEADER_ESCAPE_CHAR_STATE) { + state = HEADER_UNICODE_CHAR_STATE; + } else { + throw new CsvParserException("unknown source '" + this.getSourceState() + + "' in escape char processing state"); + } + break; + default: + StateMachine.throwExpected("escaped characters"); + } + } + if (state == null) { + state = this; + } + sm.index = i + 1; + return state; + } + } + + public static boolean isEndOfTheRowAndValueIsNotEmpty(CsvParser.StateMachine sm, char ch) { + return sm.isNewLineOrEof(ch) && (ch == EOF || !(sm.isCurrentCsvNodeEmpty && sm.peek().isBlank())); + } + + public static boolean isEndOfTheHeaderRow(CsvParser.StateMachine sm, char ch) { + return sm.isNewLineOrEof(ch); + } + + public static class CsvParserException extends Exception { + + public CsvParserException(String msg) { + super(msg); + } + } + } +} diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvTraversal.java b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvTraversal.java new file mode 100644 index 0000000..cc80b24 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvTraversal.java @@ -0,0 +1,837 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.data.csvdata.csv; + +import io.ballerina.lib.data.csvdata.utils.CsvConfig; +import io.ballerina.lib.data.csvdata.utils.CsvUtils; +import io.ballerina.lib.data.csvdata.utils.DataUtils; +import io.ballerina.lib.data.csvdata.utils.DiagnosticErrorCode; +import io.ballerina.lib.data.csvdata.utils.DiagnosticLog; +import io.ballerina.runtime.api.PredefinedTypes; +import io.ballerina.runtime.api.TypeTags; +import io.ballerina.runtime.api.creators.TypeCreator; +import io.ballerina.runtime.api.creators.ValueCreator; +import io.ballerina.runtime.api.flags.SymbolFlags; +import io.ballerina.runtime.api.types.ArrayType; +import io.ballerina.runtime.api.types.Field; +import io.ballerina.runtime.api.types.IntersectionType; +import io.ballerina.runtime.api.types.MapType; +import io.ballerina.runtime.api.types.RecordType; +import io.ballerina.runtime.api.types.TupleType; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.types.UnionType; +import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.utils.TypeUtils; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BError; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; +import io.ballerina.runtime.api.values.BTypedesc; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Convert Csv value to a ballerina record. + * + * @since 0.1.0 + */ +public final class CsvTraversal { + private static final ThreadLocal tlCsvTree = ThreadLocal.withInitial(CsvTree::new); + + private CsvTraversal() { + } + + public static Object traverse(BArray csv, CsvConfig config, BTypedesc type) { + CsvTree csvTree = tlCsvTree.get(); + try { + CsvUtils.validateConfigs(config); + Object convertedValue = csvTree.traverseCsv(csv, config, type.getDescribingType()); + return DataUtils.validateConstraints(convertedValue, type, config.enableConstraintValidation); + } catch (BError e) { + return e; + } finally { + csvTree.reset(); + } + } + + public static Object traverse(BArray csv, CsvConfig config, BTypedesc typed, Type type) { + CsvTree csvTree = tlCsvTree.get(); + try { + Object convertedValue = csvTree.traverseCsv(csv, config, type); + return DataUtils.validateConstraints(convertedValue, typed, config.enableConstraintValidation); + } catch (BError e) { + return e; + } finally { + csvTree.reset(); + } + } + + private static class CsvTree { + Object currentCsvNode; + Field currentField; + Map fieldHierarchy = new HashMap<>(); + Map updatedRecordFieldNames = new HashMap<>(); + Map headerFieldHierarchy = new HashMap<>(); + HashSet fields = new HashSet<>(); + Type restType; + Deque fieldNames = new ArrayDeque<>(); + BArray rootCsvNode; + Type expectedArrayElementType; + Type sourceArrayElementType; + CsvConfig config; + String[] headers = null; + int arraySize = 0; + BString[] headersForArrayConversion = null; + boolean addHeadersForOutput = false; + boolean isFirstRowIsHeader = false; + boolean isFirstRowInserted = false; + + void reset() { + currentCsvNode = null; + currentField = null; + fieldHierarchy.clear(); + updatedRecordFieldNames.clear(); + headerFieldHierarchy.clear(); + fields.clear(); + restType = null; + fieldNames.clear(); + rootCsvNode = null; + expectedArrayElementType = null; + sourceArrayElementType = null; + config = null; + headers = null; + arraySize = 0; + headersForArrayConversion = null; + addHeadersForOutput = false; + isFirstRowIsHeader = false; + isFirstRowInserted = false; + } + + + void resetForUnionTypes() { + currentCsvNode = null; + currentField = null; + fieldHierarchy.clear(); + updatedRecordFieldNames.clear(); + headerFieldHierarchy.clear(); + fields.clear(); + restType = null; + fieldNames.clear(); + rootCsvNode = null; + expectedArrayElementType = null; + headers = null; + arraySize = 0; + headersForArrayConversion = null; + addHeadersForOutput = false; + isFirstRowIsHeader = false; + isFirstRowInserted = false; + } + + void resetForUnionMemberTypes() { + currentCsvNode = null; + currentField = null; + fieldHierarchy.clear(); + updatedRecordFieldNames.clear(); + headerFieldHierarchy.clear(); + fields.clear(); + restType = null; + fieldNames.clear(); + headers = null; + headersForArrayConversion = null; + } + + CsvTree() { + reset(); + } + + @SuppressWarnings("unchecked") + public Object traverseCsv(BArray csv, CsvConfig config, Type type) { + this.config = config; + sourceArrayElementType = TypeUtils.getReferredType(getSourceElementTypeForLists(csv)); + Type referredType = TypeUtils.getReferredType(type); + int sourceArraySize = (int) csv.getLength(); + if (referredType.getTag() == TypeTags.INTERSECTION_TAG) { + Optional mutableType = CsvUtils.getMutableType((IntersectionType) referredType); + if (mutableType.isPresent()) { + return CsvCreator.constructReadOnlyValue(traverseCsv(csv, config, mutableType.get())); + } + } + + if (referredType.getTag() != TypeTags.UNION_TAG) { + Optional intersectionValue = handleNonUnionIntersections(referredType, csv, config); + if (intersectionValue.isPresent()) { + return intersectionValue.get(); + } + int expectedArraySize = ((ArrayType) referredType).getSize(); + setRootCsvNodeForNonUnionArrays(referredType, type); + CsvUtils.validateExpectedArraySize(expectedArraySize, sourceArraySize); + traverseCsvWithExpectedType(sourceArraySize, csv, type); + } else { + traverseCsvWithUnionExpectedType(referredType, type, sourceArraySize, csv); + } + return rootCsvNode; + } + + private Optional handleNonUnionIntersections(Type referredType, BArray csv, CsvConfig config) { + if (referredType.getTag() == TypeTags.ARRAY_TAG) { + Type arrayElementType = TypeUtils.getReferredType(((ArrayType) referredType).getElementType()); + if (arrayElementType.getTag() == TypeTags.INTERSECTION_TAG) { + return CsvUtils.getMutableType((IntersectionType) arrayElementType) + .map(mutableType -> CsvCreator.constructReadOnlyValue( + traverseCsv(csv, config, TypeCreator.createArrayType(mutableType)))); + } + } + return Optional.empty(); + } + + private void traverseCsvWithUnionExpectedType(Type referredType, Type type, int sourceArraySize, + BArray csv) { + for (Type memberType: ((UnionType) referredType).getMemberTypes()) { + Type mType = TypeUtils.getReferredType(memberType); + if (mType.getTag() == TypeTags.ARRAY_TAG) { + int expectedArraySize = ((ArrayType) mType).getSize(); + resetForUnionTypes(); + try { + setRootCsvNodeForNonUnionArrays(mType, mType); + CsvUtils.validateExpectedArraySize(expectedArraySize, sourceArraySize); + traverseCsvWithExpectedType(sourceArraySize, csv, type); + return; + } catch (Exception ex) { + // ignore + } + } + } + throw DiagnosticLog.error(DiagnosticErrorCode.SOURCE_CANNOT_CONVERT_INTO_EXP_TYPE, type); + } + + private void traverseCsvWithExpectedType(int sourceArraySize, + BArray csv, Type type) { + boolean isIntersection = false; + this.isFirstRowIsHeader = false; + if (expectedArrayElementType.getTag() == TypeTags.INTERSECTION_TAG) { + Optional mutableType = CsvUtils.getMutableType((IntersectionType) expectedArrayElementType); + if (mutableType.isPresent()) { + isIntersection = true; + expectedArrayElementType = mutableType.get(); + } + } + + switch (expectedArrayElementType.getTag()) { + case TypeTags.RECORD_TYPE_TAG, TypeTags.MAP_TAG -> + traverseCsvWithMappingAsExpectedType(sourceArraySize, csv, + expectedArrayElementType, isIntersection); + case TypeTags.ARRAY_TAG, TypeTags.TUPLE_TAG -> traverseCsvWithListAsExpectedType(sourceArraySize, csv, + expectedArrayElementType, isIntersection); + case TypeTags.UNION_TAG -> traverseCsvWithUnionExpectedType(csv, + (UnionType) expectedArrayElementType, type); + default -> throw DiagnosticLog.error(DiagnosticErrorCode.SOURCE_CANNOT_CONVERT_INTO_EXP_TYPE, type); + } + } + + public void traverseCsvWithMappingAsExpectedType(long length, BArray csv, Type expectedArrayType, + boolean isIntersection) { + Object rowValue; + ArrayType arrayType = (ArrayType) rootCsvNode.getType(); + int rowNumber = 0; + for (int i = 0; i < length; i++) { + if (arrayType.getState() == ArrayType.ArrayState.CLOSED && + arrayType.getSize() - 1 < this.arraySize) { + break; + } + Object o = csv.get(i); + + if (i < config.headerRows && i != config.headerRows - 1) { + continue; + } + + if (i >= config.headerRows && ignoreRow(rowNumber + 1, config.skipLines)) { + rowNumber++; + continue; + } + + rowValue = initStatesForCsvRowWithMappingAsExpectedType(o, expectedArrayType); + if (isIntersection) { + rowValue = CsvCreator.constructReadOnlyValue(rowValue); + } + + if (!this.isFirstRowIsHeader) { + rootCsvNode.add(this.arraySize, rowValue); + this.arraySize++; + } + if (i >= config.headerRows) { + rowNumber++; + } + } + } + + public void traverseCsvWithListAsExpectedType(long length, BArray csv, Type expectedArrayType, + boolean isIntersection) { + Object rowValue; + expectedArrayType = TypeUtils.getReferredType(expectedArrayType); + ArrayType arrayType = (ArrayType) rootCsvNode.getType(); + int rowNumber = 0; + for (int i = 0; i < length; i++) { + if (arrayType.getState() == ArrayType.ArrayState.CLOSED && + arrayType.getSize() - 1 < this.arraySize) { + break; + } + + Object o = csv.get(i); + if (!addHeadersForOutput && config.outputWithHeaders + && (o instanceof BMap || (config.customHeaders != null || i == config.headerRows - 1))) { + // Headers will add to the list only in the first iteration + insertHeaderValuesForTheCsvIfApplicable(o, expectedArrayType); + } + if (i < config.headerRows) { + continue; + } + + if (ignoreRow(rowNumber + 1, config.skipLines)) { + rowNumber++; + continue; + } + + rowValue = initStatesForCsvRowWithListAsExpectedType(o, expectedArrayType); + if (isIntersection) { + rowValue = CsvCreator.constructReadOnlyValue(rowValue); + } + if (!this.isFirstRowIsHeader) { + rootCsvNode.add(this.arraySize, rowValue); + this.arraySize++; + } + rowNumber++; + } + } + + public void traverseCsvWithUnionExpectedType(BArray csv, + UnionType expectedArrayType, Type type) { + + for (Type memberType: expectedArrayType.getMemberTypes()) { + try { + memberType = TypeCreator.createArrayType(TypeUtils.getReferredType(memberType)); + traverseCsv(csv, config, memberType); + return; + } catch (Exception ex) { + resetForUnionTypes(); + } + } + throw DiagnosticLog.error(DiagnosticErrorCode.SOURCE_CANNOT_CONVERT_INTO_EXP_TYPE, type); + } + + private static boolean ignoreRow(int index, Object skipLinesConfig) { + long[] skipLines = CsvUtils.getSkipDataRows(skipLinesConfig); + for (long skipLine: skipLines) { + if (skipLine == index) { + return true; + } + } + return false; + } + + public Object initStatesForCsvRowWithMappingAsExpectedType(Object csvElement, Type expectedType) { + expectedType = TypeUtils.getReferredType(expectedType); + switch (expectedType.getTag()) { + case TypeTags.RECORD_TYPE_TAG -> { + RecordType recordType = (RecordType) expectedType; + this.fieldHierarchy = new HashMap<>(recordType.getFields()); + fields = new HashSet<>(recordType.getFields().keySet()); + this.updatedRecordFieldNames = CsvUtils.processNameAnnotationsAndBuildCustomFieldMap( + recordType, fieldHierarchy); + this.headerFieldHierarchy = new HashMap<>(recordType.getFields()); + this.restType = recordType.getRestFieldType(); + currentCsvNode = ValueCreator.createRecordValue(recordType.getPackage(), recordType.getName()); + traverseCsvRowWithMappingAsExpectedType(csvElement, expectedType, false); + } + case TypeTags.MAP_TAG -> { + MapType mapType = (MapType) expectedType; + currentCsvNode = ValueCreator.createMapValue(mapType); + traverseCsvRowWithMappingAsExpectedType(csvElement, expectedType, true); + } + default -> throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_TYPE, expectedType); + } + return currentCsvNode; + } + + public Object initStatesForCsvRowWithListAsExpectedType(Object csvElement, Type expectedType) { + expectedType = TypeUtils.getReferredType(expectedType); + switch (expectedType.getTag()) { + case TypeTags.ARRAY_TAG -> { + ArrayType arrayType = (ArrayType) expectedType; + currentCsvNode = ValueCreator.createArrayValue(arrayType); + traverseCsvRowWithListAsExpectedType(csvElement, arrayType); + } + case TypeTags.TUPLE_TAG -> { + TupleType tupleType = (TupleType) expectedType; + this.restType = tupleType.getRestType(); + currentCsvNode = ValueCreator.createTupleValue(tupleType); + traverseCsvRowWithListAsExpectedType(csvElement, tupleType); + } + default -> throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_TYPE, expectedType); + } + return currentCsvNode; + } + + private void traverseCsvRowWithListAsExpectedType(Object csvElement, Type type) { + int expectedTypeSize = CsvUtils.getTheExpectedArraySize(type); + if (csvElement instanceof BMap map) { + constructCsvArrayFromMapping(map, type, expectedTypeSize == -1 ? map.size() : expectedTypeSize); + } else if (csvElement instanceof BArray array) { + constructCsvArrayFromNonMapping(array, type, expectedTypeSize == -1 ? array.size() : expectedTypeSize); + } + } + + private void constructCsvArrayFromMapping(BMap map, Type type, int expectedSize) { + int index = 0; + BString[] keys = generateCsvHeadersForMappingRow(map, config.headerOrder, map.size()); + for (BString key: keys) { + if (!map.containsKey(key)) { + throw DiagnosticLog.error(DiagnosticErrorCode.HEADERS_WITH_VARYING_LENGTH_NOT_SUPPORTED, key); + } + Object v = map.get(key); + if (config.allowDataProjection && index >= expectedSize) { + break; + } + Type memberType = getTheElementTypeFromList(type, index); + if (memberType != null) { + boolean isArrayActive = insertToListAndReturnFalseIfListEnds(v, memberType, index, currentCsvNode); + if (!isArrayActive) { + return; + } + } + index++; + } + } + + private void constructCsvArrayFromNonMapping(BArray csvElement, Type type, int expectedSize) { + int index = 0; + for (int i = 0; i < csvElement.getLength(); i++) { + if (config.allowDataProjection && index >= expectedSize) { + break; + } + Type memberType = getTheElementTypeFromList(type, index); + if (memberType != null) { + boolean isArrayActive = insertToListAndReturnFalseIfListEnds( + csvElement.get(i), memberType, index, currentCsvNode); + if (!isArrayActive) { + return; + } + } + index++; + } + } + + private void traverseCsvRowWithMappingAsExpectedType(Object csvElement, + Type expectedType, boolean mappingType) { + if (csvElement instanceof BMap map) { + constructCsvMapFromMapping(map, mappingType, expectedType); + } else if (csvElement instanceof BArray array) { + constructCsvMapFromNonMapping(array, mappingType, expectedType); + } else { + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_CSV_DATA_FORMAT); + } + } + + private void constructCsvMapFromNonMapping(BArray csvElement, + boolean mappingType, Type expectedType) { + this.isFirstRowIsHeader = false; + int arraySize = csvElement.size(); + if (this.headers == null) { + this.headers = CsvUtils.createHeadersForParseLists(csvElement, csvElement.size(), config); + if (!this.isFirstRowInserted && config.headerRows >= 1) { + // To skip the row at the position [config.headerRows - 1] from being aded to the result. + this.isFirstRowIsHeader = true; + this.isFirstRowInserted = true; + return; + } + } + boolean headersMatchWithExpType = validateHeaders(expectedType, csvElement, arraySize); + if (!headersMatchWithExpType) { + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_CONVERSION_FOR_ARRAY_TO_MAP, + csvElement, expectedType); + } + insertCsvMappingRow(csvElement, arraySize, mappingType, expectedType); + } + + private void constructCsvMapFromMapping( + BMap map, boolean mappingType, Type expType) { + Type currentFieldType; + for (BString key : map.getKeys()) { + if (!mappingType) { + if (!isMappingKeyBelongsToNonRestType(map.get(key), key)) { + continue; + } + currentFieldType = TypeUtils.getReferredType(currentField.getFieldType()); + } else { + addFieldInMapType(key); + currentFieldType = TypeUtils.getReferredType( + ((MapType) expType).getConstrainedType() + ); + } + insertCurrentFieldMemberIntoMapping(currentFieldType, map.get(key), key, mappingType); + } + CsvUtils.checkRequiredFieldsAndLogError(fieldHierarchy, config.absentAsNilableType); + } + + private void insertHeaderValuesForTheCsvIfApplicable(Object obj, Type type) { + if (config.outputWithHeaders && CsvUtils.isExpectedTypeIsArray(type)) { + if (this.headers == null && obj instanceof BArray array) { + this.headers = CsvUtils.createHeadersForParseLists(array, array.size(), config); + } + if (this.headers == null && obj instanceof BMap) { + BMap map = (BMap) obj; + int size = map.size(); + BString[] headerArray = generateCsvHeadersForMappingRow(map, config.headerOrder, size); + this.headers = new String[size]; + for (int i = 0; i < headerArray.length; i++) { + this.headers[i] = StringUtils.getStringValue(headerArray[i]); + } + } + + BArray headersArray; + if (type instanceof ArrayType arrayType) { + headersArray = ValueCreator.createArrayValue(arrayType); + } else { + headersArray = ValueCreator.createTupleValue((TupleType) type); + } + + for (int i = 0; i < this.headers.length; i++) { + Type memberType = getTheElementTypeFromList(type, i); + if (memberType != null) { + boolean isArrayActive = insertToListAndReturnFalseIfListEnds(StringUtils.fromString( + headers[i]), memberType, i, headersArray); + if (!isArrayActive) { + break; + } + } + } + + if (!this.isFirstRowIsHeader) { + rootCsvNode.add(this.arraySize, headersArray); + this.arraySize++; + addHeadersForOutput = true; + } + } + } + + private BString[] generateCsvHeadersForMappingRow(BMap map, Object headerOrder, int size) { + BString[] keys = new BString[size]; + if (headerOrder != null) { + String[] order = ((BArray) headerOrder).getStringArray(); + if (order.length != size) { + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_HEADER_NAMES_LENGTH); + } + for (int i = 0; i < size; i++) { + keys[i] = StringUtils.fromString(order[i]); + } + } else { + if (headersForArrayConversion == null) { + headersForArrayConversion = map.getKeys(); + } + keys = headersForArrayConversion; + } + return keys; + } + + private Type getTheElementTypeFromList(Type type, int index) { + if (type instanceof TupleType tupleType) { + List tupleTypes = tupleType.getTupleTypes(); + if (tupleTypes.size() >= index + 1) { + return tupleTypes.get(index); + } + Type res = tupleType.getRestType(); + if (res != null) { + return res; + } else { + if (config.allowDataProjection) { + return null; + } + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_EXPECTED_TUPLE_SIZE, tupleTypes.size()); + } + } + ArrayType arrayType = (ArrayType) type; + if (arrayType.getSize() != -1 && arrayType.getSize() <= index) { + if (config.allowDataProjection) { + return null; + } + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_EXPECTED_ARRAY_SIZE, arrayType.getSize()); + } + return arrayType.getElementType(); + } + + private boolean validateHeaders(Type expectedType, BArray csvElement, int arraySize) { + if (arraySize < this.headers.length) { + throw DiagnosticLog.error(DiagnosticErrorCode.HEADERS_WITH_VARYING_LENGTH_NOT_SUPPORTED); + } + if (expectedType instanceof MapType) { + return true; + } + Type type = csvElement.getType(); + if (type instanceof TupleType tupleType) { + return validateHeadersWithTupleDataRows(expectedType, tupleType); + } else { + return validateHeadersWithArrayDataRows(expectedType); + } + } + + private boolean validateHeadersWithTupleDataRows(Type expectedType, TupleType tupleType) { + Type type; + List tupleTypes = tupleType.getTupleTypes(); + Type tupleRestType = tupleType.getRestType(); + + if (expectedType instanceof RecordType) { + if (this.restType != null + && (this.restType.getTag() == TypeTags.ANYDATA_TAG + || this.restType.getTag() == TypeTags.JSON_TAG)) { + return true; + } + + for (int i = 0; i < this.headers.length; i++) { + if (i >= tupleTypes.size()) { + type = tupleRestType; + } else { + type = tupleTypes.get(i); + } + String header = this.headers[i]; + Field field = this.headerFieldHierarchy.remove(header); + + if (field != null) { + if (!config.stringConversion && type != null + && type.getTag() != field.getFieldType().getTag()) { + return false; + } + continue; + } + + if ((tupleRestType != null && (type == this.restType || this.restType == tupleRestType))) { + continue; + } + + if (CsvUtils.isHeaderFieldsEmpty(this.headerFieldHierarchy)) { + continue; + } + return false; + } + return true; + } + return false; + } + + private boolean validateHeadersWithArrayDataRows(Type expectedType) { + if (expectedType instanceof RecordType) { + if (this.restType != null) { + return true; + } + + for (String key: this.fieldHierarchy.keySet()) { + for (String header: this.headers) { + if (key.equals(this.updatedRecordFieldNames.get(header))) { + return true; + } + } + } + } + return false; + } + + private void insertCsvMappingRow(BArray csvElement, int arraySize, boolean mappingType, Type expectedType) { + Type fieldType; + BString key; + if (arraySize != this.headers.length) { + throw DiagnosticLog.error(DiagnosticErrorCode.HEADERS_WITH_VARYING_LENGTH_NOT_SUPPORTED); + } + + for (int i = 1; i <= arraySize; i++) { + key = StringUtils.fromString(this.headers[i - 1]); + if (!mappingType) { + if (!isMappingKeyBelongsToNonRestType(csvElement.get(i - 1), key)) { + continue; + } + fieldType = TypeUtils.getReferredType(currentField.getFieldType()); + } else { + addFieldInMapType(key); + fieldType = ((MapType) expectedType).getConstrainedType(); + } + insertCurrentFieldMemberIntoMapping(fieldType, csvElement.get(i - 1), key, mappingType); + } + CsvUtils.checkRequiredFieldsAndLogError(fieldHierarchy, config.absentAsNilableType); + } + + private boolean isMappingKeyBelongsToNonRestType(Object value, BString key) { + String keyStr = StringUtils.getStringValue(key); + String fieldName = CsvUtils.getUpdatedHeaders(this.updatedRecordFieldNames, + keyStr, this.fields.contains(keyStr)); + currentField = fieldHierarchy.remove(fieldName); + if (currentField == null) { + // Add to the rest field + if (restType != null) { + Type restFieldType = TypeUtils.getReferredType(restType); + insertRestFieldMemberIntoMapping(restFieldType, key, value); + return false; + } + if (config.allowDataProjection) { + return false; + } + throw DiagnosticLog.error(DiagnosticErrorCode.NO_FIELD_FOR_HEADER, key); + } + fieldNames.push(currentField.getFieldName()); + return true; + } + + private void addFieldInMapType(BString key) { + fieldNames.push(key.toString()); + } + + private Object convertCsvValueIntoExpectedType(Type type, Object csvMember, boolean isRecursive) { + Type fieldType = TypeUtils.getReferredType(type); + Object nilValue = config.nilValue; + if (!isRecursive && config.nilAsOptionalField && !fieldType.isNilable() + && CsvUtils.isNullValue(nilValue, csvMember) + && currentField != null && SymbolFlags.isFlagOn(currentField.getFlags(), SymbolFlags.OPTIONAL)) { + return CsvUtils.SkipMappedValue.VALUE; + } + if (config.stringConversion && csvMember instanceof BString str) { + Object convertedValue = CsvCreator.convertToExpectedType(str, type, config); + if (!(convertedValue instanceof BError)) { + return convertedValue; + } + } else { + switch (fieldType.getTag()) { + case TypeTags.NULL_TAG: + case TypeTags.BOOLEAN_TAG: + case TypeTags.INT_TAG: + case TypeTags.FLOAT_TAG: + case TypeTags.DECIMAL_TAG: + case TypeTags.STRING_TAG: + case TypeTags.JSON_TAG: + case TypeTags.ANYDATA_TAG: + case TypeTags.CHAR_STRING_TAG: + case TypeTags.BYTE_TAG: + case TypeTags.SIGNED8_INT_TAG: + case TypeTags.SIGNED16_INT_TAG: + case TypeTags.SIGNED32_INT_TAG: + case TypeTags.UNSIGNED8_INT_TAG: + case TypeTags.UNSIGNED16_INT_TAG: + case TypeTags.UNSIGNED32_INT_TAG: + case TypeTags.FINITE_TYPE_TAG: + if (CsvUtils.checkTypeCompatibility(fieldType, csvMember, config.stringConversion)) { + Object value = CsvUtils.convertToBasicType(csvMember, fieldType, config); + if (!(value instanceof BError)) { + return value; + } + } + break; + case TypeTags.UNION_TAG: + for (Type memberType : ((UnionType) fieldType).getMemberTypes()) { + memberType = TypeUtils.getReferredType(memberType); + if (!CsvUtils.isBasicType(memberType)) { + throw DiagnosticLog.error(DiagnosticErrorCode + .EXPECTED_TYPE_CAN_ONLY_CONTAIN_BASIC_TYPES, memberType); + } + Object value = convertCsvValueIntoExpectedType(memberType, csvMember, true); + if (!(value instanceof BError || value instanceof CsvUtils.UnMappedValue)) { + return value; + } + } + break; + case TypeTags.INTERSECTION_TAG: + Type effectiveType = ((IntersectionType) fieldType).getEffectiveType(); + effectiveType = TypeUtils.getReferredType(effectiveType); + if (!SymbolFlags.isFlagOn(SymbolFlags.READONLY, effectiveType.getFlags())) { + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_TYPE, type); + } + for (Type constituentType : ((IntersectionType) fieldType).getConstituentTypes()) { + constituentType = TypeUtils.getReferredType(constituentType); + if (constituentType.getTag() == TypeTags.READONLY_TAG) { + continue; + } + return CsvCreator.constructReadOnlyValue(convertCsvValueIntoExpectedType(constituentType, + csvMember, true)); + } + break; + default: + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_EXPECTED_TYPE, type); + } + } + return CsvUtils.UnMappedValue.VALUE; + } + + private void insertRestFieldMemberIntoMapping(Type type, BString key, Object csvMember) { + Object value = convertCsvValueIntoExpectedType(type, csvMember, false); + if (!(value instanceof CsvUtils.UnMappedValue)) { + ((BMap) currentCsvNode).put(key, value); + } + } + + private void insertCurrentFieldMemberIntoMapping(Type type, Object recValue, BString key, boolean isMapType) { + Object value = convertCsvValueIntoExpectedType(type, recValue, false); + if (!(value instanceof CsvUtils.UnMappedValue || value instanceof CsvUtils.SkipMappedValue)) { + ((BMap) currentCsvNode).put(StringUtils.fromString(fieldNames.pop()), value); + return; + } + + if (isMapType || value instanceof CsvUtils.SkipMappedValue) { + return; + } + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_TYPE_FOR_FIELD, recValue, key); + } + + public boolean insertToListAndReturnFalseIfListEnds(Object arrayValue, + Type type, int index, Object currentCsvNode) { + Object value = convertCsvValueIntoExpectedType(type, arrayValue, false); + boolean isArrayType = type instanceof ArrayType; + if (!(value instanceof CsvUtils.UnMappedValue)) { + if (isArrayType) { + ArrayType arrayType = (ArrayType) TypeUtils.getType(type); + if (arrayType.getState() == ArrayType.ArrayState.CLOSED && + arrayType.getSize() - 1 < index) { + return false; + } + } + ((BArray) currentCsvNode).add(index, value); + return true; + } + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_TYPE_FOR_ARRAY, arrayValue, index, type); + } + + private void setRootCsvNodeForNonUnionArrays(Type referredType, Type type) { + referredType = TypeUtils.getReferredType(referredType); + if (referredType instanceof ArrayType arrayType) { + rootCsvNode = ValueCreator.createArrayValue(arrayType); + expectedArrayElementType = TypeUtils.getReferredType((arrayType).getElementType()); + return; + } + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_TYPE, type, PredefinedTypes.TYPE_ANYDATA_ARRAY); + } + + private Type getSourceElementTypeForLists(BArray csv) { + if (csv.getType() instanceof TupleType tupleType) { + List memberTypes = new ArrayList<>(tupleType.getTupleTypes()); + return TypeCreator.createUnionType(memberTypes); + } + return csv.getElementType(); + } + } +} diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/csv/Native.java b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/Native.java new file mode 100644 index 0000000..619e01d --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/csv/Native.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.data.csvdata.csv; + +import io.ballerina.lib.data.csvdata.io.DataReaderTask; +import io.ballerina.lib.data.csvdata.io.DataReaderThreadPool; +import io.ballerina.lib.data.csvdata.utils.Constants; +import io.ballerina.lib.data.csvdata.utils.CsvConfig; +import io.ballerina.lib.data.csvdata.utils.DiagnosticErrorCode; +import io.ballerina.lib.data.csvdata.utils.DiagnosticLog; +import io.ballerina.runtime.api.Environment; +import io.ballerina.runtime.api.Future; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BError; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BObject; +import io.ballerina.runtime.api.values.BStream; +import io.ballerina.runtime.api.values.BString; +import io.ballerina.runtime.api.values.BTypedesc; + +import java.io.ByteArrayInputStream; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.nio.charset.Charset; + +/** + * Csv conversion. + * + * @since 0.1.0 + */ +public final class Native { + private Native() { + } + + public static Object parseString(BString csv, BMap options, BTypedesc type) { + try { + return CsvParser.parse(new StringReader(csv.getValue()), + type, CsvConfig.createParseOptions(options)); + } catch (BError e) { + return e; + } catch (Exception e) { + return DiagnosticLog.error(DiagnosticErrorCode.INVALID_CAST, csv, type); + } + } + + public static Object parseBytes(BArray csv, BMap options, BTypedesc type) { + try { + byte[] bytes = csv.getBytes(); + return CsvParser.parse(new InputStreamReader(new ByteArrayInputStream(bytes), + Charset.forName(options.getStringValue(Constants.ConfigConstants.ENCODING).toString())), + type, CsvConfig.createParseOptions(options)); + } catch (BError e) { + return e; + } catch (Exception e) { + return DiagnosticLog.error(DiagnosticErrorCode.INVALID_CAST, csv, type); + } + } + + public static Object parseStream(Environment env, BStream csv, + BMap options, BTypedesc type) { + try { + final BObject iteratorObj = csv.getIteratorObj(); + final Future future = env.markAsync(); + DataReaderTask task = new DataReaderTask(env, iteratorObj, future, type, + CsvConfig.createParseOptions(options), + options.getStringValue(Constants.ConfigConstants.ENCODING)); + DataReaderThreadPool.EXECUTOR_SERVICE.submit(task); + return null; + } catch (BError e) { + return e; + } catch (Exception e) { + return DiagnosticLog.error(DiagnosticErrorCode.INVALID_CAST, csv, type); + } + } + + public static Object transform(BArray csv, BMap options, BTypedesc type) { + try { + return CsvTraversal.traverse(csv, CsvConfig.createTransformOptions(options), type); + } catch (Exception e) { + return DiagnosticLog.getCsvError(e.getMessage()); + } + } + public static Object parseList(BArray csv, BMap options, BTypedesc type) { + try { + CsvConfig config = CsvConfig.createParseListOptions(options); + config.stringConversion = true; + return CsvTraversal.traverse(csv, config, type); + } catch (Exception e) { + return DiagnosticLog.getCsvError(e.getMessage()); + } + } +} diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/io/BallerinaByteBlockInputStream.java b/native/src/main/java/io/ballerina/lib/data/csvdata/io/BallerinaByteBlockInputStream.java new file mode 100644 index 0000000..63c3945 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/io/BallerinaByteBlockInputStream.java @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.data.csvdata.io; + +import io.ballerina.lib.data.csvdata.utils.DiagnosticLog; +import io.ballerina.runtime.api.Environment; +import io.ballerina.runtime.api.async.Callback; +import io.ballerina.runtime.api.async.StrandMetadata; +import io.ballerina.runtime.api.types.MethodType; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BError; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BObject; +import io.ballerina.runtime.api.values.BString; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; + +/** + * Java Input Stream based on Ballerina byte block stream. stream + * + * @since 0.1.0 + */ +public class BallerinaByteBlockInputStream extends InputStream { + + private final BObject iterator; + private final Environment env; + private final String nextMethodName; + private final Type returnType; + private final String strandName; + private final StrandMetadata metadata; + private final Map properties; + private final AtomicBoolean done = new AtomicBoolean(false); + private final MethodType closeMethod; + private final Consumer futureResultConsumer; + + private byte[] currentChunk = new byte[0]; + private int nextChunkIndex = 0; + + public BallerinaByteBlockInputStream(Environment env, BObject iterator, MethodType nextMethod, + MethodType closeMethod, Consumer futureResultConsumer) { + this.env = env; + this.iterator = iterator; + this.nextMethodName = nextMethod.getName(); + this.returnType = nextMethod.getReturnType(); + this.closeMethod = closeMethod; + this.strandName = env.getStrandName().orElse(""); + this.metadata = env.getStrandMetadata(); + this.properties = Map.of(); + this.futureResultConsumer = futureResultConsumer; + } + + @Override + public int read() { + if (done.get()) { + return -1; + } + if (hasBytesInCurrentChunk()) { + return currentChunk[nextChunkIndex++]; + } + // Need to get a new block from the stream, before reading again. + nextChunkIndex = 0; + try { + if (readNextChunk()) { + return read(); + } + } catch (InterruptedException e) { + BError error = DiagnosticLog.getCsvError("Cannot read the stream, interrupted error"); + futureResultConsumer.accept(error); + return -1; + } + return -1; + } + + @Override + public void close() throws IOException { + super.close(); + Semaphore semaphore = new Semaphore(0); + if (closeMethod != null) { + env.getRuntime().invokeMethodAsyncSequentially(iterator, closeMethod.getName(), strandName, metadata, + new Callback() { + @Override + public void notifyFailure(BError bError) { + semaphore.release(); + } + + @Override + public void notifySuccess(Object result) { + semaphore.release(); + } + }, properties, returnType); + } + try { + semaphore.acquire(); + } catch (InterruptedException e) { + throw new IOException("Error while closing the stream", e); + } + } + + private boolean hasBytesInCurrentChunk() { + return currentChunk.length != 0 && nextChunkIndex < currentChunk.length; + } + + private boolean readNextChunk() throws InterruptedException { + Semaphore semaphore = new Semaphore(0); + Callback callback = new Callback() { + + @Override + public void notifyFailure(BError bError) { + // Panic with an error + done.set(true); + futureResultConsumer.accept(bError); + currentChunk = new byte[0]; + semaphore.release(); + } + + @Override + public void notifySuccess(Object result) { + if (result == null) { + done.set(true); + currentChunk = new byte[0]; + semaphore.release(); + return; + } + if (result instanceof BMap) { + BMap valueRecord = (BMap) result; + final BString value = Arrays.stream(valueRecord.getKeys()).findFirst().get(); + final BArray arrayValue = valueRecord.getArrayValue(value); + currentChunk = arrayValue.getByteArray(); + semaphore.release(); + } else { + // Case where Completes with an error + done.set(true); + semaphore.release(); + } + } + + }; + env.getRuntime().invokeMethodAsyncSequentially(iterator, nextMethodName, strandName, metadata, callback, + properties, returnType); + semaphore.acquire(); + return !done.get(); + } +} diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/io/DataReaderTask.java b/native/src/main/java/io/ballerina/lib/data/csvdata/io/DataReaderTask.java new file mode 100644 index 0000000..b4186bf --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/io/DataReaderTask.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.ballerina.lib.data.csvdata.io; + +import io.ballerina.lib.data.csvdata.csv.CsvParser; +import io.ballerina.lib.data.csvdata.utils.CsvConfig; +import io.ballerina.lib.data.csvdata.utils.DiagnosticLog; +import io.ballerina.runtime.api.Environment; +import io.ballerina.runtime.api.Future; +import io.ballerina.runtime.api.types.MethodType; +import io.ballerina.runtime.api.types.ObjectType; +import io.ballerina.runtime.api.utils.TypeUtils; +import io.ballerina.runtime.api.values.BObject; +import io.ballerina.runtime.api.values.BString; +import io.ballerina.runtime.api.values.BTypedesc; + +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.util.function.Consumer; + +/** + * This class will read data from a Ballerina Stream of byte blocks, in non-blocking manner. + * + * @since 0.1.0 + */ +public class DataReaderTask implements Runnable { + + private static final String METHOD_NAME_NEXT = "next"; + private static final String METHOD_NAME_CLOSE = "close"; + + private final Environment env; + private final BObject iteratorObj; + private final Future future; + private final BTypedesc typed; + private final CsvConfig config; + private final BString encoding; + + public DataReaderTask(Environment env, BObject iteratorObj, Future future, BTypedesc typed, + CsvConfig config, BString encoding) { + this.env = env; + this.iteratorObj = iteratorObj; + this.future = future; + this.typed = typed; + this.config = config; + this.encoding = encoding; + } + + static MethodType resolveNextMethod(BObject iterator) { + MethodType method = getMethodType(iterator, METHOD_NAME_NEXT); + if (method != null) { + return method; + } + throw new IllegalStateException("next method not found in the iterator object"); + } + + static MethodType resolveCloseMethod(BObject iterator) { + return getMethodType(iterator, METHOD_NAME_CLOSE); + } + + private static MethodType getMethodType(BObject iterator, String methodNameClose) { + ObjectType objectType = (ObjectType) TypeUtils.getReferredType(iterator.getOriginalType()); + MethodType[] methods = objectType.getMethods(); + // Assumes compile-time validation of the iterator object + for (MethodType method : methods) { + if (method.getName().equals(methodNameClose)) { + return method; + } + } + return null; + } + + @Override + public void run() { + ResultConsumer resultConsumer = new ResultConsumer<>(future); + try (var byteBlockSteam = new BallerinaByteBlockInputStream(env, iteratorObj, resolveNextMethod(iteratorObj), + resolveCloseMethod(iteratorObj), resultConsumer)) { + Object result = CsvParser.parse(new InputStreamReader(byteBlockSteam, + Charset.forName(this.encoding.toString())), + typed, this.config); + future.complete(result); + } catch (Exception e) { + future.complete(DiagnosticLog.getCsvError("Error occurred while reading the stream: " + e.getMessage())); + } + } + + /** + * This class will hold module related utility functions. + * + * @param The type of the result + * @param future The future to complete + * @since 0.1.0 + */ + public record ResultConsumer(Future future) implements Consumer { + + @Override + public void accept(T t) { + future.complete(t); + } + } +} diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/io/DataReaderThreadPool.java b/native/src/main/java/io/ballerina/lib/data/csvdata/io/DataReaderThreadPool.java new file mode 100644 index 0000000..379a6e3 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/io/DataReaderThreadPool.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.ballerina.lib.data.csvdata.io; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Thread pool for data reader. + * + * @since 0.1.0 + */ +public final class DataReaderThreadPool { + private static final int CORE_POOL_SIZE = 0; + private static final int MAX_POOL_SIZE = 50; + private static final long KEEP_ALIVE_TIME = 60L; + private static final String THREAD_NAME = "bal-data-csv-thread"; + private static final AtomicLong THREAD_ID = new AtomicLong(1); + public static final ExecutorService EXECUTOR_SERVICE = new ThreadPoolExecutor(CORE_POOL_SIZE, + MAX_POOL_SIZE, KEEP_ALIVE_TIME, TimeUnit.SECONDS, new SynchronousQueue<>(), new DataThreadFactory()); + + private DataReaderThreadPool() { + } + + /** + * Thread factory for data reader. + */ + static class DataThreadFactory implements ThreadFactory { + + @Override + public Thread newThread(Runnable runnable) { + Thread ballerinaData = new Thread(runnable); + ballerinaData.setName(THREAD_NAME + "-" + THREAD_ID.getAndIncrement()); + return ballerinaData; + } + } +} diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/utils/Constants.java b/native/src/main/java/io/ballerina/lib/data/csvdata/utils/Constants.java new file mode 100644 index 0000000..f2ad47b --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/utils/Constants.java @@ -0,0 +1,80 @@ +package io.ballerina.lib.data.csvdata.utils; + +import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.values.BString; + +/* + * Constants used in the CSV data module. + * + * @since 0.1.0 + */ +public final class Constants { + public static final class ConfigConstants { + public static final BString DELIMITER = StringUtils.fromString("delimiter"); + public static final BString TEXT_ENCLOSURE = StringUtils.fromString("textEnclosure"); + public static final BString HEADER = StringUtils.fromString("header"); + public static final BString ESCAPE_CHAR = StringUtils.fromString("escapeChar"); + public static final BString LINE_TERMINATOR = StringUtils.fromString("lineTerminator"); + public static final BString SKIP_LINES = StringUtils.fromString("skipLines"); + public static final BString NIL_VALUE = StringUtils.fromString("nilValue"); + public static final BString COMMENT_CHAR = StringUtils.fromString("comment"); + public static final BString LOCALE = StringUtils.fromString("locale"); + public static final BString ENCODING = StringUtils.fromString("encoding"); + public static final BString NIL_AS_OPTIONAL = StringUtils.fromString("nilAsOptionalField"); + public static final BString ABSENT_AS_NILABLE = StringUtils.fromString("absentAsNilableType"); + public static final BString ALLOW_DATA_PROJECTION = StringUtils.fromString("allowDataProjection"); + public static final BString STRING_CONVERSION = StringUtils.fromString("stringConversion"); + public static final BString ENABLE_CONSTRAINT_VALIDATION = StringUtils. + fromString("enableConstraintValidation"); + public static final BString OUTPUT_WITH_HEADERS = StringUtils.fromString("outputWithHeaders"); + public static final BString HEADER_ROWS = StringUtils.fromString("headerRows"); + public static final BString CUSTOM_HEADERS_IF_HEADER_ABSENT = + StringUtils.fromString("customHeadersIfHeadersAbsent"); + public static final BString CUSTOM_HEADERS = StringUtils.fromString("customHeaders"); + public static final BString HEADERS_ORDER = StringUtils.fromString("headerOrder"); + + private ConfigConstants() { + } + } + + public static final class Values { + public static final String NULL = "null"; + public static final String BALLERINA_NULL = "()"; + + private Values() { + } + } + + public static final class LineTerminator { + public static final char LF = '\n'; + public static final char CR = '\r'; + public static final String CRLF = "\r\n"; + + private LineTerminator() { + } + } + + public static final class EscapeChar { + public static final char DOUBLE_QUOTES_CHAR = '"'; + public static final char BACKSLASH_CHAR = '\\'; + public static final char SLASH_CHAR = '/'; + public static final char BACKSPACE_CHAR = 'b'; + public static final char FORM_FEED_CHAR = 'f'; + public static final char NEWLINE_CHAR = 'n'; + public static final char CARRIAGE_RETURN_CHAR = 'r'; + public static final char TAB_CHAR = 't'; + public static final char UNICODE_START_CHAR = 'u'; + + private EscapeChar() { + } + } + + public static final String SKIP_LINE_RANGE_SEP = "-"; + public static final String FIELD = "$field$."; + public static final String NAME = "Name"; + public static final BString VALUE = StringUtils.fromString("value"); + public static final String UNDERSCORE = "_"; + + private Constants() { + } +} diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/utils/CsvConfig.java b/native/src/main/java/io/ballerina/lib/data/csvdata/utils/CsvConfig.java new file mode 100644 index 0000000..cf1c64c --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/utils/CsvConfig.java @@ -0,0 +1,157 @@ +package io.ballerina.lib.data.csvdata.utils; + +import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; + +/* + * Configuration for CSV operations. + * + * @since 0.1.0 + */ +public class CsvConfig { + public char delimiter = ','; + public char textEnclosure = '\\'; + public Object header = 0; + public char escapeChar = '\\'; + public Object lineTerminator = '\n'; + public Object skipLines = null; + public Object nilValue = null; + public char comment = '#'; + public String locale = "en_US"; + public boolean nilAsOptionalField = false; + public boolean absentAsNilableType = false; + public boolean allowDataProjection = true; + public Object headerOrder = null; + public boolean stringConversion = false; + public boolean enableConstraintValidation = false; + public boolean outputWithHeaders = false; + public Object customHeadersIfHeadersAbsent = null; + public long headerRows = 0; + public Object customHeaders = null; + + private CsvConfig(CsvConfig config) { + this.allowDataProjection = false; + this.header = config.header; + this.delimiter = config.delimiter; + this.textEnclosure = config.textEnclosure; + this.escapeChar = config.escapeChar; + this.lineTerminator = config.lineTerminator; + this.nilValue = config.nilValue; + this.comment = config.comment; + this.locale = config.locale; + } + + public static CsvConfig createParseListOptions(BMap options) { + updateDataProjectOptions(options); + return new CsvConfig( + options.getBooleanValue(Constants.ConfigConstants.NIL_AS_OPTIONAL), + options.getBooleanValue(Constants.ConfigConstants.ABSENT_AS_NILABLE), + options.getBooleanValue(Constants.ConfigConstants.ALLOW_DATA_PROJECTION), + options.get(Constants.ConfigConstants.SKIP_LINES), + options.getBooleanValue(Constants.ConfigConstants.ENABLE_CONSTRAINT_VALIDATION), + options.getBooleanValue(Constants.ConfigConstants.OUTPUT_WITH_HEADERS), + options.getIntValue(Constants.ConfigConstants.HEADER_ROWS), + options.get(Constants.ConfigConstants.CUSTOM_HEADERS) + ); + } + + public CsvConfig(boolean nilAsOptionalField, boolean absentAsNilableType, boolean allowDataProjection, + Object skipLines, boolean enableConstraintValidation, boolean outputWithHeaders, + long headerRows, Object headers) { + this.nilAsOptionalField = nilAsOptionalField; + this.absentAsNilableType = absentAsNilableType; + this.allowDataProjection = allowDataProjection; + this.skipLines = skipLines; + this.enableConstraintValidation = enableConstraintValidation; + this.outputWithHeaders = outputWithHeaders; + this.headerRows = headerRows; + this.customHeaders = headers; + } + + public static CsvConfig createTransformOptions(BMap options) { + updateDataProjectOptions(options); + return new CsvConfig( + options.getBooleanValue(Constants.ConfigConstants.NIL_AS_OPTIONAL), + options.getBooleanValue(Constants.ConfigConstants.ABSENT_AS_NILABLE), + options.getBooleanValue(Constants.ConfigConstants.ALLOW_DATA_PROJECTION), + options.get(Constants.ConfigConstants.SKIP_LINES), + options.getBooleanValue(Constants.ConfigConstants.ENABLE_CONSTRAINT_VALIDATION), + options.getBooleanValue(Constants.ConfigConstants.OUTPUT_WITH_HEADERS), + options.get(Constants.ConfigConstants.HEADERS_ORDER) + ); + } + + public CsvConfig(boolean nilAsOptionalField, boolean absentAsNilableType, boolean allowDataProjection, + Object skipLines, boolean enableConstraintValidation, boolean outputWithHeaders, + Object headerOrder) { + this.nilAsOptionalField = nilAsOptionalField; + this.absentAsNilableType = absentAsNilableType; + this.allowDataProjection = allowDataProjection; + this.skipLines = skipLines; + this.enableConstraintValidation = enableConstraintValidation; + this.outputWithHeaders = outputWithHeaders; + this.headerOrder = headerOrder; + } + + public static CsvConfig createParseOptions(BMap options) { + updateDataProjectOptions(options); + return new CsvConfig( + options.getBooleanValue(Constants.ConfigConstants.NIL_AS_OPTIONAL), + options.getBooleanValue(Constants.ConfigConstants.ABSENT_AS_NILABLE), + options.getBooleanValue(Constants.ConfigConstants.ALLOW_DATA_PROJECTION), + options.get(Constants.ConfigConstants.SKIP_LINES), + options.getBooleanValue(Constants.ConfigConstants.ENABLE_CONSTRAINT_VALIDATION), + options.getBooleanValue(Constants.ConfigConstants.OUTPUT_WITH_HEADERS), + StringUtils.getStringValue(options.getStringValue(Constants.ConfigConstants.DELIMITER)).charAt(0), + StringUtils.getStringValue(options.getStringValue(Constants.ConfigConstants.LOCALE)), + StringUtils.getStringValue(options.getStringValue(Constants.ConfigConstants.TEXT_ENCLOSURE)).charAt(0), + StringUtils.getStringValue(options.getStringValue(Constants.ConfigConstants.ESCAPE_CHAR)).charAt(0), + options.get(Constants.ConfigConstants.LINE_TERMINATOR), + options.get(Constants.ConfigConstants.NIL_VALUE), + StringUtils.getStringValue(options.getStringValue(Constants.ConfigConstants.COMMENT_CHAR)).charAt(0), + options.get(Constants.ConfigConstants.HEADER), + options.get(Constants.ConfigConstants.CUSTOM_HEADERS_IF_HEADER_ABSENT) + ); + } + + public CsvConfig(boolean nilAsOptionalField, boolean absentAsNilableType, boolean allowDataProjection, + Object skipLines, boolean enableConstraintValidation, boolean outputWithHeaders, char delimiter, + String locale, char textEnclosure, char escapeChar, Object lineTerminator, + Object nilValue, char comment, Object header, Object customHeadersIfHeadersAbsent) { + this.nilAsOptionalField = nilAsOptionalField; + this.absentAsNilableType = absentAsNilableType; + this.allowDataProjection = allowDataProjection; + this.skipLines = skipLines; + this.enableConstraintValidation = enableConstraintValidation; + this.outputWithHeaders = outputWithHeaders; + this.delimiter = delimiter; + this.locale = locale; + this.textEnclosure = textEnclosure; + this.escapeChar = escapeChar; + this.lineTerminator = lineTerminator; + this.nilValue = nilValue; + this.comment = comment; + this.header = header; + this.customHeadersIfHeadersAbsent = customHeadersIfHeadersAbsent; + } + + public static CsvConfig createConfigOptionsForUnion(CsvConfig config) { + return new CsvConfig(config); + } + + private static void updateDataProjectOptions(BMap options) { + Object allowProjections = options.get(Constants.ConfigConstants.ALLOW_DATA_PROJECTION); + if (allowProjections instanceof Boolean) { + options.put(Constants.ConfigConstants.NIL_AS_OPTIONAL, false); + options.put(Constants.ConfigConstants.ABSENT_AS_NILABLE, false); + return; + } + BMap projections = (BMap) allowProjections; + options.put(Constants.ConfigConstants.ALLOW_DATA_PROJECTION, true); + options.put(Constants.ConfigConstants.NIL_AS_OPTIONAL, projections.getBooleanValue( + Constants.ConfigConstants.NIL_AS_OPTIONAL)); + options.put(Constants.ConfigConstants.ABSENT_AS_NILABLE, projections.getBooleanValue( + Constants.ConfigConstants.ABSENT_AS_NILABLE)); + } +} diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/utils/CsvUtils.java b/native/src/main/java/io/ballerina/lib/data/csvdata/utils/CsvUtils.java new file mode 100644 index 0000000..73f8dc1 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/utils/CsvUtils.java @@ -0,0 +1,341 @@ +package io.ballerina.lib.data.csvdata.utils; + +import io.ballerina.runtime.api.TypeTags; +import io.ballerina.runtime.api.flags.SymbolFlags; +import io.ballerina.runtime.api.types.ArrayType; +import io.ballerina.runtime.api.types.Field; +import io.ballerina.runtime.api.types.IntersectionType; +import io.ballerina.runtime.api.types.RecordType; +import io.ballerina.runtime.api.types.TupleType; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.utils.TypeUtils; +import io.ballerina.runtime.api.utils.ValueUtils; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BDecimal; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +import static io.ballerina.lib.data.csvdata.utils.Constants.SKIP_LINE_RANGE_SEP; + +/* + * Utility functions used in the CSV data module. + * + * @since 0.1.0 + */ +public class CsvUtils { + private static final long[] EMPTY_LONG_ARRAY = new long[]{}; + + public static void validateExpectedArraySize(int size, int currentSize) { + if (size != -1 && size > currentSize) { + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_EXPECTED_ARRAY_SIZE, currentSize); + } + } + + public static boolean isExpectedTypeIsArray(Type expectedType) { + expectedType = TypeUtils.getReferredType(expectedType); + + return switch (expectedType.getTag()) { + case TypeTags.TUPLE_TAG, TypeTags.ARRAY_TAG -> true; + default -> false; + }; + } + + public static boolean isBasicType(Type type) { + return switch (type.getTag()) { + case TypeTags.INT_TAG, TypeTags.STRING_TAG, TypeTags.BOOLEAN_TAG, TypeTags.DECIMAL_TAG, TypeTags.FLOAT_TAG, + TypeTags.NULL_TAG, TypeTags.JSON_TAG, TypeTags.ANYDATA_TAG, TypeTags.UNION_TAG, + TypeTags.INTERSECTION_TAG, TypeTags.CHAR_STRING_TAG, TypeTags.BYTE_TAG, TypeTags.SIGNED8_INT_TAG, + TypeTags.SIGNED16_INT_TAG, TypeTags.SIGNED32_INT_TAG, TypeTags.UNSIGNED8_INT_TAG, + TypeTags.UNSIGNED16_INT_TAG, TypeTags.UNSIGNED32_INT_TAG -> true; + default -> false; + }; + } + + public static String[] createHeadersForParseLists(BArray csvElement, int headerSize, CsvConfig config) { + String[] headers = new String[headerSize]; + Object customHeaders = config.customHeaders; + long headerRows = config.headerRows; + + int length = headers.length; + if (customHeaders instanceof BArray array) { + if (array.size() != length) { + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_CUSTOM_HEADER_LENGTH); + } + for (int i = 0; i < length; i++) { + headers[i] = array.get(i).toString(); + } + return headers; + } + + if (headerRows == 1) { + return csvElement.getStringArray(); + } + + if (headerRows > 1) { + throw DiagnosticLog.error(DiagnosticErrorCode.NO_CUSTOM_HEADER_PROVIDED); + } + + // when headerRows = 0 and customHeaders = null + for (int i = 0; i < length; i++) { + headers[i] = String.valueOf(i + 1); + } + return headers; + } + + public static Optional getMutableType(IntersectionType intersectionType) { + for (Type constituentType : intersectionType.getConstituentTypes()) { + if (constituentType.getTag() == TypeTags.READONLY_TAG) { + continue; + } + return Optional.of(constituentType); + } + return Optional.empty(); + } + + public static Object convertToBasicType(Object csv, Type targetType, CsvConfig config) { + if (csv == null) { + csv = config.nilValue; + } + return ValueUtils.convert(csv, targetType); + } + + public static void checkRequiredFieldsAndLogError(Map filedHierarchy, boolean absentAsNilableType) { + filedHierarchy.values().forEach(field -> { + if (absentAsNilableType && field.getFieldType().isNilable()) { + return; + } + if (SymbolFlags.isFlagOn(field.getFlags(), SymbolFlags.REQUIRED)) { + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_FIELD_IN_CSV, field.getFieldName()); + } + }); + } + + public static boolean isHeaderFieldsEmpty(Map currentField) { + for (Field field: currentField.values()) { + if (SymbolFlags.isFlagOn(field.getFlags(), SymbolFlags.REQUIRED)) { + return false; + } + } + return true; + } + + public static boolean checkTypeCompatibility(Type constraintType, Object csv, boolean stringConversion) { + int tag = constraintType.getTag(); + if (csv instanceof BString) { + return stringConversion || TypeTags.isStringTypeTag(tag) || isJsonOrAnyDataOrAny(tag); + } + if (csv instanceof Long) { + return TypeTags.isIntegerTypeTag(tag) || tag == TypeTags.FLOAT_TAG || tag == TypeTags.DECIMAL_TAG + || tag == TypeTags.BYTE_TAG || isJsonOrAnyDataOrAny(tag); + } + if (csv instanceof BDecimal) { + if ((tag == TypeTags.DECIMAL_TAG + || tag == TypeTags.FLOAT_TAG || TypeTags.isIntegerTypeTag(tag)) || isJsonOrAnyDataOrAny(tag)) { + return true; + } + } + if (csv instanceof Double) { + return (tag == TypeTags.FLOAT_TAG + || tag == TypeTags.DECIMAL_TAG || TypeTags.isIntegerTypeTag(tag)) || isJsonOrAnyDataOrAny(tag); + } + if (csv instanceof Boolean) { + return tag == TypeTags.BOOLEAN_TAG || isJsonOrAnyDataOrAny(tag); + } + if (csv == null) { + return tag == TypeTags.NULL_TAG || isJsonOrAnyDataOrAny(tag); + } + return false; + } + + private static boolean isJsonOrAnyDataOrAny(int tag) { + return tag == TypeTags.JSON_TAG || tag == TypeTags.ANYDATA_TAG || tag == TypeTags.ANY_TAG; + } + + public static int getTheExpectedArraySize(Type type) { + if (type instanceof TupleType tupleType) { + if (tupleType.getRestType() != null) { + return -1; + } + return tupleType.getTupleTypes().size(); + } else { + return ((ArrayType) type).getSize(); + } + } + + public static Map processNameAnnotationsAndBuildCustomFieldMap(RecordType recordType, + Map fieldHierarchy) { + BMap annotations = recordType.getAnnotations(); + HashMap updatedRecordFieldNames = new HashMap<>(); + HashSet updatedFields = new HashSet<>(); + HashSet updatedValues = new HashSet<>(); + + for (BString annotationsKey : annotations.getKeys()) { + String key = annotationsKey.getValue(); + if (key.contains(Constants.FIELD)) { + BMap annotMap = ((BMap) annotations.get(annotationsKey)); + for (BString mapKey : annotMap.getKeys()) { + if (mapKey.getValue().endsWith(Constants.NAME)) { + String name = ((Map) annotMap.get(mapKey)).get(Constants.VALUE).toString(); + String originalName = key.substring(Constants.FIELD.length()); + if (updatedValues.contains(name)) { + throw DiagnosticLog.error(DiagnosticErrorCode.DUPLICATE_FIELD, name); + } + if (updatedFields.contains(originalName)) { + throw DiagnosticLog.error(DiagnosticErrorCode.DUPLICATE_FIELD, originalName); + } + updatedFields.add(originalName); + updatedValues.add(name); + updatedRecordFieldNames.put(name, originalName); + break; + } + } + } + } + for (String field : fieldHierarchy.keySet()) { + if (updatedFields.contains(field)) { + continue; + } + if (updatedValues.contains(field) || updatedRecordFieldNames.containsKey(field)) { + throw DiagnosticLog.error(DiagnosticErrorCode.DUPLICATE_FIELD, field); + } + updatedRecordFieldNames.put(field, field); + } + + return updatedRecordFieldNames; + } + + public static String getUpdatedHeaders(Map updatedRecords, String key, boolean isKeyContains) { + String fieldName = updatedRecords.get(key); + if (fieldName != null) { + return fieldName; + } + if (isKeyContains) { + throw DiagnosticLog.error(DiagnosticErrorCode.DUPLICATE_FIELD, key); + } + return key; + } + + public static long[] getSkipLinesFromStringConfigValue(String configValue) { + String[] parts = configValue.split(SKIP_LINE_RANGE_SEP); + if (parts.length != 2) { + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_RANGE_FOR_SKIPLINES); + } + try { + int start = Integer.parseInt(parts[0]); + int end = Integer.parseInt(parts[1]); + int size = end - start + 1; + if (size <= 0) { + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_RANGE_FOR_SKIPLINES); + } + long[] result = new long[size]; + for (int i = 0; i < size; i++) { + result[i] = start + i; + } + return result; + } catch (NumberFormatException e) { + throw DiagnosticLog.error(DiagnosticErrorCode.INVALID_VALUE_FOR_SKIPLINES); + } + } + + public static long[] getSkipDataRows(Object skipLines) { + if (skipLines == null) { + return EMPTY_LONG_ARRAY; + } + + if (skipLines instanceof BArray skipLinesArray) { + if (skipLinesArray.getLength() == 0) { + return EMPTY_LONG_ARRAY; + } + return skipLinesArray.getIntArray(); + } + + return getSkipLinesFromStringConfigValue(StringUtils.getStringValue(skipLines)); + } + + public static boolean isNullValue(Object nullValue, Object value) { + if (value == null) { + return true; + } + if (value instanceof BString bString) { + value = StringUtils.getStringValue(bString); + } + if (value instanceof String v) { + return handleStringNullValue(nullValue, v, value); + } + return false; + } + + private static boolean handleStringNullValue(Object nullValue, String v, Object value) { + if ((nullValue == null) && (Constants.Values.NULL.equalsIgnoreCase(v) + || Constants.Values.BALLERINA_NULL.equalsIgnoreCase(v))) { + return true; + } + return nullValue != null && value.equals(StringUtils.getStringValue(nullValue)); + } + + public static boolean isCharContainsInLineTerminatorUserConfig(char c, + Object lineTerminatorObj, boolean isCarriageTokenPresent) { + if (lineTerminatorObj instanceof BArray array) { + Object[] lineTerminators = array.getValues(); + for (Object lineTerminator: lineTerminators) { + Optional value = handleLineTerminator(lineTerminator, c, isCarriageTokenPresent); + if (value.isEmpty()) { + continue; + } + return value.get(); + } + return false; + } + + Optional value = handleLineTerminator(lineTerminatorObj, c, isCarriageTokenPresent); + if (value.isEmpty()) { + return false; + } + return value.get(); + } + + private static Optional handleLineTerminator(Object lineTerminator, + char c, boolean isCarriageTokenPresent) { + if (lineTerminator == null || c != Constants.LineTerminator.LF) { + return Optional.empty(); + } + if (lineTerminator.equals(Constants.LineTerminator.CRLF)) { + return Optional.of(isCarriageTokenPresent); + } + return Optional.of(true); + } + + public static class UnMappedValue { + public static final UnMappedValue VALUE = new UnMappedValue(); + } + + public static class SkipMappedValue { + public static final SkipMappedValue VALUE = new SkipMappedValue(); + } + + public static Locale createLocaleFromString(String localeString) { + // Split the string into language, country, and variant + String[] parts = localeString.split(Constants.UNDERSCORE); + int length = parts.length; + if (length == 3) { + return new Locale(parts[0], parts[1], parts[2]); + } else if (length == 2) { + return new Locale(parts[0], parts[1]); + } else { + return new Locale(parts[0]); // Only language + } + } + + public static void validateConfigs(CsvConfig config) { + if (config.headerRows > 1 && config.customHeaders == null) { + throw DiagnosticLog.error(DiagnosticErrorCode.NO_CUSTOM_HEADER_PROVIDED); + } + } +} diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/utils/DataUtils.java b/native/src/main/java/io/ballerina/lib/data/csvdata/utils/DataUtils.java new file mode 100644 index 0000000..b509a24 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/utils/DataUtils.java @@ -0,0 +1,36 @@ +package io.ballerina.lib.data.csvdata.utils; + +import io.ballerina.runtime.api.values.BError; +import io.ballerina.runtime.api.values.BTypedesc; +import io.ballerina.stdlib.constraint.Constraints; + +/* + * Utility functions used in the CSV data module. + * + * @since 0.1.0 + */ +public final class DataUtils { + private DataUtils() { + } + + public static Object validateConstraints(Object convertedValue, BTypedesc typed, boolean requireValidation) { + if (!requireValidation) { + return convertedValue; + } + + Object result = Constraints.validate(convertedValue, typed); + if (result instanceof BError bError) { + return DiagnosticLog.getCsvError(getPrintableErrorMsg(bError)); + } + return convertedValue; + } + + private static String getPrintableErrorMsg(BError err) { + StringBuilder errorBuilder = new StringBuilder(err.getMessage() != null ? err.getMessage() : ""); + Object details = err.getDetails(); + if (details != null && !details.toString().equals("{}")) { + errorBuilder.append(!errorBuilder.isEmpty() ? ", " : "" + details); + } + return errorBuilder.toString(); + } +} diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/utils/DiagnosticErrorCode.java b/native/src/main/java/io/ballerina/lib/data/csvdata/utils/DiagnosticErrorCode.java new file mode 100644 index 0000000..82df3a5 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/utils/DiagnosticErrorCode.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.data.csvdata.utils; + +/** + * Represents a diagnostic error code. + * + * @since 0.1.0 + */ +public enum DiagnosticErrorCode { + + INVALID_TYPE("BDE_0001", "invalid.type"), + UNION_TYPES_NOT_ALLOWED("BDE_0002", "union.types.not.allowed.as.expected.type"), + INVALID_ARRAY_MEMBER("BDE_0003", "invalid.array.member.in.expected.type"), + INVALID_FIELD_IN_CSV("BDE_0004", "cannot.found.field.in.csv"), + INVALID_CAST("BDE_0005", "csv.value.cannot.cast.into.expected.type"), + INVALID_EXPECTED_TYPE("BDE_0006", "invalid.expected.type"), + INVALID_TOKEN("BDE_0007", "invalid.token.while.reading.the.csv.data"), + INVALID_CSV_DATA_FORMAT("BDE_0008", "invalid.csv.data.format"), + INVALID_EXPECTED_ARRAY_SIZE("BDE_0009", "invalid.expected.array.size"), + INVALID_EXPECTED_TUPLE_SIZE("BDE_0010", "invalid.expected.tuple.size"), + INVALID_SKIP_COLUMN_QUERY("BDE_0011", "invalid.skip.column.query"), + INVALID_TYPE_FOR_FIELD("BDE_0012", "invalid.type.for.field"), + INVALID_TYPE_FOR_ARRAY("BDE_0013", "invalid.type.for.array"), + INVALID_CONVERSION_FOR_ARRAY_TO_MAP("BDE_0014", "invalid.conversion.for.array.to.map"), + INVALID_CONFIGURATIONS("BDE_0015", "invalid.configurations"), + EXPECTED_TYPE_CAN_ONLY_CONTAIN_BASIC_TYPES("BDE_0016", "expected.type.can.only.contains.basic.types"), + INVALID_FORMAT_FOR_SKIPLINES("BDE_0017", "invalid.format.for.skiplines"), + INVALID_RANGE_FOR_SKIPLINES("BDE_0018", "invalid.range.for.skiplines"), + INVALID_VALUE_FOR_SKIPLINES("BDE_0019", "invalid.value.for.skiplines"), + INCONSISTENT_HEADER("BDE_0020", "inconsistent.header"), + INVALID_CUSTOM_HEADER_LENGTH("BDE_0021", "invalid.custom.header.length"), + INVALID_HEADER_NAMES_LENGTH("BDE_0022", "invalid.header.names.length"), + HEADER_CANNOT_BE_EMPTY("BDE_0023", "header.cannot.be.empty"), + NO_FIELD_FOR_HEADER("BDE_0024", "no.field.for.header"), + DUPLICATE_FIELD("BDE_0025", "duplicate.field"), + SOURCE_CANNOT_CONVERT_INTO_EXP_TYPE("BDE_0026", "cannot.convert.into.exptype"), + NO_CUSTOM_HEADER_PROVIDED("BDE_0027", "no.custom.header.provided"), + HEADERS_WITH_VARYING_LENGTH_NOT_SUPPORTED("BDE_0027", + "headers.with.varying.length.not.supported"), + HEADER_VALUE_CANNOT_BE_EMPTY("BDE_0028", "header.value.cannot.be.empty"), + DUPLICATE_HEADER("BDE_0029", "duplicate.header"); + + String diagnosticId; + String messageKey; + + DiagnosticErrorCode(String diagnosticId, String messageKey) { + this.diagnosticId = diagnosticId; + this.messageKey = messageKey; + } + + public String messageKey() { + return messageKey; + } +} diff --git a/native/src/main/java/io/ballerina/lib/data/csvdata/utils/DiagnosticLog.java b/native/src/main/java/io/ballerina/lib/data/csvdata/utils/DiagnosticLog.java new file mode 100644 index 0000000..6de4ee3 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/utils/DiagnosticLog.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.data.csvdata.utils; + +import io.ballerina.runtime.api.creators.ErrorCreator; +import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.values.BError; + +import java.text.MessageFormat; +import java.util.Locale; +import java.util.ResourceBundle; + +/** + * Diagnostic log for data module. + * + * @since 0.1.0 + */ +public final class DiagnosticLog { + private static final String ERROR_PREFIX = "error"; + private static final String CSV_CONVERSION_ERROR = "Error"; + private static final String UNSUPPORTED_OPERATION_ERROR = "Error"; + private static final ResourceBundle MESSAGES = ResourceBundle.getBundle("csv_error", Locale.getDefault()); + + private DiagnosticLog() { + } + + public static BError error(DiagnosticErrorCode code, Object... args) { + String msg = formatMessage(code, args); + return getCsvError(msg); + } + + private static String formatMessage(DiagnosticErrorCode code, Object[] args) { + String msgKey = MESSAGES.getString(ERROR_PREFIX + "." + code.messageKey()); + return MessageFormat.format(msgKey, args); + } + + public static BError getCsvError(String message, String errorType) { + return ErrorCreator.createError(ModuleUtils.getModule(), + errorType, StringUtils.fromString(message), null, null); + } + + public static BError getCsvError(String message) { + return getCsvError(message, CSV_CONVERSION_ERROR); + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/data/csvdata/utils/ModuleUtils.java b/native/src/main/java/io/ballerina/lib/data/csvdata/utils/ModuleUtils.java similarity index 96% rename from native/src/main/java/io/ballerina/stdlib/data/csvdata/utils/ModuleUtils.java rename to native/src/main/java/io/ballerina/lib/data/csvdata/utils/ModuleUtils.java index d9de709..486c9f3 100644 --- a/native/src/main/java/io/ballerina/stdlib/data/csvdata/utils/ModuleUtils.java +++ b/native/src/main/java/io/ballerina/lib/data/csvdata/utils/ModuleUtils.java @@ -16,7 +16,7 @@ * under the License. */ -package io.ballerina.stdlib.data.csvdata.utils; +package io.ballerina.lib.data.csvdata.utils; import io.ballerina.runtime.api.Environment; import io.ballerina.runtime.api.Module; diff --git a/native/src/main/java/io/ballerina/stdlib/data/csvdata/csv/Native.java b/native/src/main/java/io/ballerina/stdlib/data/csvdata/csv/Native.java deleted file mode 100644 index 1f5d3d6..0000000 --- a/native/src/main/java/io/ballerina/stdlib/data/csvdata/csv/Native.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com). - * - * WSO2 LLC. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.ballerina.stdlib.data.csvdata.csv; - -import io.ballerina.runtime.api.Environment; -import io.ballerina.runtime.api.values.BArray; -import io.ballerina.runtime.api.values.BMap; -import io.ballerina.runtime.api.values.BStream; -import io.ballerina.runtime.api.values.BString; -import io.ballerina.runtime.api.values.BTypedesc; - -/** - * Csv conversion. - * - * @since 0.1.0 - */ -public class Native { - - public static Object parseStringToRecord(BString csv, BMap options, BTypedesc type) { - return null; - } - - public static Object parseBytesToRecord(BArray csv, BMap options, BTypedesc type) { - return null; - } - - public static Object parseStreamToRecord(Environment env, BStream csv, - BMap options, BTypedesc type) { - return null; - } - - public static Object parseStringToList(BString csv, BMap options, BTypedesc type) { - return null; - } - - public static Object parseBytesToList(BArray csv, BMap options, BTypedesc type) { - return null; - } - - public static Object parseStreamToList(Environment env, BStream csv, - BMap options, BTypedesc type) { - return null; - } - - public static Object parseRecordAsRecordType(BArray csv, BMap options, BTypedesc type) { - return null; - } - - public static Object parseRecordAsListType(BArray csv, BArray headers, - BMap options, BTypedesc type) { - return null; - } - - public static Object parseListAsRecordType(BArray csv, Object customHeaders, - BMap options, BTypedesc type) { - return null; - } - - public static Object parseListAsListType(BArray csv, BMap options, BTypedesc type) { - return null; - } -} diff --git a/native/src/main/java/module-info.java b/native/src/main/java/module-info.java index 1f625c2..27c1c99 100644 --- a/native/src/main/java/module-info.java +++ b/native/src/main/java/module-info.java @@ -16,6 +16,12 @@ * under the License. */ -module io.ballerina.stdlib.data { +module io.ballerina.lib.data { requires io.ballerina.runtime; + requires io.ballerina.lang.value; + requires junit; + requires org.apache.commons.lang3; + requires io.ballerina.stdlib.constraint; + exports io.ballerina.lib.data.csvdata.csv; + exports io.ballerina.lib.data.csvdata.utils; } diff --git a/native/src/main/resources/META-INF/native-image/io.ballerina.stdlib/data/csvdata-native/resource-config.json b/native/src/main/resources/META-INF/native-image/io.ballerina.stdlib/data/csvdata-native/resource-config.json new file mode 100644 index 0000000..fea4f2b --- /dev/null +++ b/native/src/main/resources/META-INF/native-image/io.ballerina.stdlib/data/csvdata-native/resource-config.json @@ -0,0 +1,6 @@ +{ + "bundles":[{ + "name":"csv_error", + "locales":[""] + }] +} diff --git a/native/src/main/resources/csv_error.properties b/native/src/main/resources/csv_error.properties new file mode 100644 index 0000000..96cc250 --- /dev/null +++ b/native/src/main/resources/csv_error.properties @@ -0,0 +1,112 @@ +# +# Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# ------------------------- +# Data module error messages +# ------------------------- + +error.invalid.type=\ + invalid expected type ''{0}'', expected a subtype of (map|anydata[])[] + +error.union.types.not.allowed.as.expected.type=\ + union types are not allowed in the expected type, found ''{0}'' + +error.invalid.array.member.in.expected.type=\ + invalid type ''{0}'' in the expected array type + +error.cannot.found.field.in.csv=\ + no matching header value is found for the required field ''{0}'' + +error.csv.value.cannot.cast.into.expected.type=\ + value ''{0}'' cannot be cast into ''{1}'' + +error.invalid.expected.type=\ + expected a array type, found ''{0}'' + +error.invalid.token.while.reading.the.csv.data=\ + error while reading the csv data ''{0}'' in line: ''{1}'', column: ''{2}'' + +error.invalid.csv.data.format=\ + invalid csv data format + +error.invalid.expected.array.size=\ + invalid array size for expected array type, cannot be greater than ''{0}'' + +error.invalid.expected.tuple.size=\ + invalid array size for expected tuple type, cannot be greater than ''{0}'' + +error.invalid.skip.column.query=\ + invalid query found for skip column field, ''{0}'' + +error.invalid.type.for.field=\ + no mapping type found for value ''{0}'' in key ''{1}'' + +error.invalid.type.for.array=\ + value ''{0}'' in index ''{1}'' is not compatible with array type ''{2}'' + +error.expected.type.can.only.contains.basic.types=\ + expected type cannot contains types other than basic types, ''{0}'' not allowed + +error.invalid.conversion.for.array.to.map=\ + value ''{0}'' cannot be cast into ''{1}'', because fields in ''{1}'' or the provided \ + expected headers are not matching with the ''{0}'' + +error.invalid.configurations=\ + invalid configurations: ''{0}'' + +error.invalid.format.for.skiplines=\ + invalid format for the skipLines field. Expected format: 'start-end' + +error.invalid.range.for.skiplines=\ + invalid range for the skipLines field. Start value must be less than or equal to end value. + +error.invalid.value.for.skiplines=\ + invalid input for the skipLines field. Both start and end values must be integers. + +error.inconsistent.header=\ + header ''{0}'' cannot be found in data rows + +error.invalid.custom.header.length=\ + invalid number of headers + +error.invalid.header.names.length=\ + invalid number of headers + +error.header.cannot.be.empty=\ + provided header row is empty + +error.no.field.for.header=\ + no mapping field in the expected type for header ''{0}'' + +error.duplicate.field=\ + duplicate field found in record fields: ''{0}'' + +error.cannot.convert.into.exptype=\ + source value cannot converted in to the ''{0}'' + +error.no.custom.header.provided=\ + custom headers should be provided + +error.headers.with.varying.length.not.supported=\ + CSV data rows with varying headers are not yet supported + +error.header.value.cannot.be.empty=\ + header cannot be empty + +error.duplicate.header=\ + duplicate header found: ''{0}'' diff --git a/native/src/main/resources/error.properties b/native/src/main/resources/error.properties deleted file mode 100644 index 84e78d6..0000000 --- a/native/src/main/resources/error.properties +++ /dev/null @@ -1,21 +0,0 @@ -# -# Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). -# -# WSO2 LLC. licenses this file to you under the Apache License, -# Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# ------------------------- -# Data module error messages -# ------------------------- diff --git a/settings.gradle b/settings.gradle index 72b6bfd..ab6d512 100644 --- a/settings.gradle +++ b/settings.gradle @@ -46,12 +46,14 @@ include(':data.csv-native') include(':data.csv-ballerina') include(':data.csv-compiler-plugin') include(':data.csv-compiler-plugin-tests') +include(':data.csv-ballerina-tests') project(':checkstyle').projectDir = file("build-config${File.separator}checkstyle") project(':data.csv-native').projectDir = file('native') project(':data.csv-ballerina').projectDir = file('ballerina') project(':data.csv-compiler-plugin').projectDir = file('compiler-plugin') project(':data.csv-compiler-plugin-tests').projectDir = file('compiler-plugin-test') +project(':data.csv-ballerina-tests').projectDir = file('ballerina-tests') gradleEnterprise { buildScan {