Skip to content

Commit

Permalink
e2e_CloudSqlPostgreSql incorporated review comments
Browse files Browse the repository at this point in the history
  • Loading branch information
itsmekumari committed Jun 5, 2023
1 parent 8cdbadd commit 285d30e
Show file tree
Hide file tree
Showing 22 changed files with 45 additions and 37 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
# the License.
#

@Cloudsqlpostgresql_Sink @Cloudsqlpostgresql_Sink_Required
@Regression @Sink_Required
Feature: CloudSQL-PostgreSQL sink - Verify CloudSQL-PostgreSQL sink plugin design time scenarios

Scenario: To verify CloudSQLPostgreSQL sink plugin validation with connection and basic details for connectivity
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
# the License.
#

@Cloudsqlpostgresql_Sink @Cloudsqlpostgresql_Sink_Required
@Regression @Sink_Required
Feature: CloudSQL-PostgreSQL sink- Verify CloudSQL-PostgreSQL sink plugin design time macro scenarios

Scenario: To verify CloudSQLPostgreSQL sink plugin validation with macro enabled fields for connection section
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright © 2022 Cask Data, Inc.
# Copyright © 2023 Cask Data, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
Expand All @@ -14,7 +14,7 @@
# the License.
#

@Cloudsqlpostgresql_Sink @Cloudsqlpostgresql_Sink_Required
@Regression @Sink_Required
Feature: CloudSQL-PostgreSQL Sink - Verify CloudSQL-postgreSQL Sink Plugin Error scenarios

Scenario:Verify CloudSQLPostgreSQL sink plugin validation errors for mandatory fields
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
# the License.
#

@Cloudsqlpostgresql_Sink @Cloudsqlpostgresql_Sink_Required
@Regression @Sink_Required
Feature: CloudSQL-PostgreSQL sink - Verify data transfer from BigQuery source to CloudSQL-PostgreSQL sink

@BQ_SOURCE_TEST @CLOUDSQLPOSTGRESQL_TEST_TABLE @PLUGIN-1526
Expand Down Expand Up @@ -50,7 +50,7 @@ Feature: CloudSQL-PostgreSQL sink - Verify data transfer from BigQuery source to
Then Save the pipeline
Then Preview and run the pipeline
Then Verify the preview of pipeline is "success"
Then Click on preview data for CloudSQLPostgreSQL sink
Then Click on the Preview Data link on the Sink plugin node: "CloudSQLPostgreSQL"
Then Close the preview data
Then Deploy the pipeline
Then Run the Pipeline in Runtime
Expand Down Expand Up @@ -93,7 +93,7 @@ Feature: CloudSQL-PostgreSQL sink - Verify data transfer from BigQuery source to
Then Save the pipeline
Then Preview and run the pipeline
Then Verify the preview of pipeline is "success"
Then Click on preview data for CloudSQLPostgreSQL sink
Then Click on the Preview Data link on the Sink plugin node: "CloudSQLPostgreSQL"
Then Close the preview data
Then Deploy the pipeline
Then Run the Pipeline in Runtime
Expand Down Expand Up @@ -136,7 +136,7 @@ Feature: CloudSQL-PostgreSQL sink - Verify data transfer from BigQuery source to
Then Save the pipeline
Then Preview and run the pipeline
Then Verify the preview of pipeline is "success"
Then Click on preview data for CloudSQLPostgreSQL sink
Then Click on the Preview Data link on the Sink plugin node: "CloudSQLPostgreSQL"
Then Close the preview data
Then Deploy the pipeline
Then Run the Pipeline in Runtime
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
# the License.
#

@Cloudsqlpostgresql_Sink @Cloudsqlpostgresql_Sink_Required
@Regression @Sink_Required
Feature: CloudSQL-PostgreSQL sink - Verify data transfer to PostgreSQL sink with macro arguments

@BQ_SOURCE_TEST @CLOUDSQLPOSTGRESQL_TEST_TABLE @PLUGIN-1629 @PLUGIN-1526
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
# the License.
#

@Cloudsqlpostgresql_Source @Cloudsqlpostgresql_Source_Required
@Regression @Source_Required
Feature: CloudSQL-PostgreSQL source - Verify CloudSQLPostgreSQL source plugin design time scenarios

@CLOUDSQLPOSTGRESQL_SOURCE_TEST @CLOUDSQLPOSTGRESQL_SINK_TEST
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
# the License.
#

@Cloudsqlpostgresql_Source @Cloudsqlpostgresql_Source_Required
@Regression @Source_Required
Feature: CloudSQL-PostgreSQL source - Verify CloudSQL-PostgreSQL source plugin design time macros scenarios

Scenario: To verify CloudSQLPostgreSQL source plugin validation with macro enabled fields for connection section
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
# the License.
#

@Cloudsqlpostgresql_Source @Cloudsqlpostgresql_Source_Required
@Regression @Source_Required
Feature: CloudSQL-postgreSQL source - Verify CloudSQL-postgreSQL source plugin design time validation scenarios

Scenario:Verify CloudSQLPostgreSQL source plugin validation errors for mandatory fields
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
# the License.
#

@Cloudsqlpostgresql_Source @Cloudsqlpostgresql_Source_Required
@Regression @Source_Required
Feature: CloudSQL-PostGreSQL Source - Run Time scenarios

@CLOUDSQLPOSTGRESQL_SOURCE_TEST @BQ_SINK_TEST @CLOUDSQLPOSTGRESQL_SINK_TEST @PLUGIN-1526
Expand Down Expand Up @@ -227,7 +227,7 @@ Feature: CloudSQL-PostGreSQL Source - Run Time scenarios
Then Save the pipeline
Then Preview and run the pipeline
Then Verify the preview of pipeline is "success"
Then Click on preview data for CloudSQLPostgreSQL sink
Then Click on the Preview Data link on the Sink plugin node: "CloudSQLPostgreSQL"
Then Close the preview data
Then Deploy the pipeline
Then Run the Pipeline in Runtime
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
# the License.
#

@Cloudsqlpostgresql_Source @Cloudsqlpostgresql_Source_Required
@Regression @Source_Required
Feature: CloudSQL-PostGreSQL source - Verify CloudSQL-PostGreSQL plugin data transfer with macro arguments

@CLOUDSQLPOSTGRESQL_SOURCE_TEST @CLOUDSQLPOSTGRESQL_SINK_TEST
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ public static int countRecord(String table, String schema) throws SQLException,
}

/**
* Extracts entire data from source and target tables.
* Extracts Result set of source and target table.
* @param sourceTable table at the source side
* @param targetTable table at the sink side
* @return true if the values in source and target side are equal
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,4 @@
/**
* Package contains BQ validation..
*/
package io.cdap.plugin.cloudsqlpostgresql;
package io.cdap.plugin.cloudsqlpostgresql;
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright © 2022 Cask Data, Inc.
* Copyright © 2023 Cask Data, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
Expand All @@ -26,7 +26,7 @@
@CucumberOptions(
features = {"src/e2e-test/features"},
glue = {"io.cdap.plugin.cloudsqlpostgresql.stepsdesign", "stepsdesign", "io.cdap.plugin.common.stepsdesign"},
tags = {"@Cloudsqlpostgresql_Sink and not @PLUGIN-1629 and not @PLUGIN-1526"},
tags = {"@Regression and not @PLUGIN-1629 and not @PLUGIN-1526"},
/* TODO :Enable tests once issue fixed https://cdap.atlassian.net/browse/PLUGIN-1629,
https://cdap.atlassian.net/browse/PLUGIN-1526
*/
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright © 2022 Cask Data, Inc.
* Copyright © 2023 Cask Data, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
Expand All @@ -20,13 +20,16 @@
import org.junit.runner.RunWith;

/**
* Test Runner to execute only required CloudSQLPostgreSQL Source plugin testcases.
* Test Runner to execute only required CloudSQLPostgreSQL Sink plugin testcases.
*/
@RunWith(Cucumber.class)
@CucumberOptions(
features = {"src/e2e-test/features"},
glue = {"io.cdap.plugin.cloudsqlpostgresql.stepsdesign", "stepsdesign", "io.cdap.plugin.common.stepsdesign"},
tags = {"@Cloudsqlpostgresql_Sink_Required"},
tags = {"@Sink_Required and not @PLUGIN-1629 and not @PLUGIN-1526"},
/* TODO :Enable tests once issue fixed https://cdap.atlassian.net/browse/PLUGIN-1629,
https://cdap.atlassian.net/browse/PLUGIN-1526
*/
monochrome = true,
plugin = {"pretty", "html:target/cucumber-html-report/cloudsqlpostgresql-sink-required",
"json:target/cucumber-reports/cucumber-cloudsqlpostgresql-sink-required.json",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright © 2022 Cask Data, Inc.
* Copyright © 2023 Cask Data, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
Expand All @@ -13,6 +13,7 @@
* License for the specific language governing permissions and limitations under
* the License.
*/

/**
* Package contains the runner for the CloudSQLPostgreSQL sink plugin.
*/
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright © 2022 Cask Data, Inc.
* Copyright © 2023 Cask Data, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
Expand All @@ -26,7 +26,7 @@
@CucumberOptions(
features = {"src/e2e-test/features"},
glue = {"io.cdap.plugin.cloudsqlpostgresql.stepsdesign", "stepsdesign", "io.cdap.plugin.common.stepsdesign"},
tags = {"@Cloudsqlpostgresql_Source and not @PLUGIN-1526"},
tags = {"@Regression and not @PLUGIN-1526"},
/* TODO :Enable tests once issue fixed https://cdap.atlassian.net/browse/PLUGIN-1526
*/
monochrome = true,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright © 2022 Cask Data, Inc.
* Copyright © 2023 Cask Data, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
Expand All @@ -26,7 +26,9 @@
@CucumberOptions(
features = {"src/e2e-test/features"},
glue = {"io.cdap.plugin.cloudsqlpostgresql.stepsdesign", "stepsdesign", "io.cdap.plugin.common.stepsdesign"},
tags = {"@Cloudsqlpostgresql_Source_Required"},
tags = {"@Source_Required and not @PLUGIN-1526"},
/* TODO :Enable tests once issue fixed https://cdap.atlassian.net/browse/PLUGIN-1526
*/
monochrome = true,
plugin = {"pretty", "html:target/cucumber-html-report/cloudsqlpostgresql-source-required",
"json:target/cucumber-reports/cucumber-cloudsqlpostgresql-source-required.json",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright © 2022 Cask Data, Inc.
* Copyright © 2023 Cask Data, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
Expand All @@ -13,6 +13,7 @@
* License for the specific language governing permissions and limitations under
* the License.
*/

/**
* Package contains the runner for the CloudSQLPostgreSQL source plugin.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,6 @@
*/
public class CloudSqlPostgreSql implements CdfHelper {

@Then("Click on preview data for CloudSQLPostgreSQL sink")
public void clickOnPreviewDataForCloudSQLPostgreSQLSink() {
openSinkPluginPreviewData("CloudSQLPostgreSQL");
}

@Then("Validate the values of records transferred to target table is equal to the values from source table")
public void validateTheValuesOfRecordsTransferredToTargetTableIsEqualToTheValuesFromSourceTable() throws
SQLException, ClassNotFoundException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,7 @@
* the License.
*/


/**
* Package contains stepDesign for CloudSqlPostgresql features.
*/
package io.cdap.plugin.cloudsqlpostgresql.stepsdesign;
package io.cdap.plugin.cloudsqlpostgresql.stepsdesign;
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,4 @@
/**
* Package contains CloudSqlPostgresql Client.
*/
package io.cdap.plugin;
package io.cdap.plugin;
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
driverName=cloudsql-postgresql
username=CLOUDSQL_POSTGRESQL_USERNAME
password=CLOUDSQL_POSTGRESQL_PASSWORD
connectionName=CLOUDSQL_POSTGRESQL_CONNECTION_NAME
databaseName=test_automation_db
schema=public
URL=jdbc:postgresql://google/%s?cloudSqlInstance=%s&socketFactory=com.google.cloud.sql.postgres.SocketFactory&user=%s&password=%s

datatypesColumns=( id varchar(100) primary key, col1 bpchar, col2 bpchar(10), col3 varchar, col4 varchar(3), \
col5 bytea, col6 int2, col7 int4, col8 int8, col10 numeric(10, 4), col11 numeric(10), col12 float4, col13 float8, \
col14 money, col15 text, col16 name, col17 float8, col18 numeric(38), col22 timestamp, col23 timestamptz, \
Expand Down Expand Up @@ -55,6 +57,7 @@ datatypesSchema=[{"key":"id","value":"string"},{"key":"col1","value":"string"},{
{"key":"col61","value":"float"},{"key":"col62","value":"int"},{"key":"col63","value":"int"},\
{"key":"col64","value":"int"},{"key":"col65","value":"double"},{"key":"col66","value":"long"},\
{"key":"col67","value":"long"},{"key":"col68","value":"boolean"}]

#CLOUDSQLPOSTGRESQL Invalid Properties
invalidRef=invalidRef&^*&&*
invalidDatabaseName=invalidDB
Expand All @@ -67,27 +70,31 @@ invalidBoundingQuery=SELECT MIN(id),MAX(id) FROM table
invalidBoundingQueryValue=select;
invalidTable=table
invalidConnectionName=abd3

#CLOUDSQLPOSTGRESQL Valid Properties
connectionArgumentsList=[{"key":"queryTimeout","value":"-1"}]
connectionTimeout=150
numberOfSplits=2
zeroValue=0
splitByColumn=ID
importQuery=where $CONDITIONS
connectionName=CLOUDSQL_POSTGRESQL_CONNECTION_NAME

#bq properties
projectId=cdf-athena
dataset=test_automation
bqOutputMultipleDatatypesSchema=[{"key":"col1","value":"bytes"},{"key":"col2","value":"string"},\
{"key":"col3","value":"date"},{"key":"col4","value":"double"},{"key":"col5","value":"decimal"},\
{"key":"col6","value":"timestamp"},{"key":"col7","value":"decimal"},{"key":"col8","value":"boolean"},\
{"key":"col9","value":"long"},{"key":"col10","value":"time"}]

#bq macro properties
bqTruncateTable=true
bqUpdateTableSchema=true

#bq queries file path
CreateBQTableQueryFile=testdata/BigQuery/BigQueryCreateTableQuery.txt
InsertBQDataQueryFile=testdata/BigQuery/BigQueryInsertDataQuery.txt

#BIGQUERY Datatypes
bigQueryDatatypesColumns=(col1 bytea, col2 varchar(100), col3 date, col4 double precision, col5 numeric(5, 2), \
col6 timestamp, col7 numeric(5, 2), col8 boolean, col9 int8, col10 time)
Expand Down

0 comments on commit 285d30e

Please sign in to comment.