From 688f9670c69de916e92c7be1dc9328311f2e870b Mon Sep 17 00:00:00 2001 From: Javier <10879637+javiertuya@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:50:22 +0200 Subject: [PATCH 1/6] ISSUE-680 # Add database SQL executor --- core/pom.xml | 4 + .../zerocode/core/db/DbSqlExecutor.java | 83 ++++++++++++++++ .../jsmart/zerocode/core/db/DbSqlRequest.java | 36 +++++++ .../jsmart/zerocode/core/db/DbSqlRunner.java | 42 ++++++++ .../core/db/DbSqlExecutorScenarioTest.java | 17 ++++ .../zerocode/core/db/DbSqlRunnerTest.java | 95 +++++++++++++++++++ core/src/test/resources/db_test.properties | 15 +++ .../db/db_sql_execute.json | 49 ++++++++++ pom.xml | 6 ++ 9 files changed, 347 insertions(+) create mode 100644 core/src/main/java/org/jsmart/zerocode/core/db/DbSqlExecutor.java create mode 100644 core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRequest.java create mode 100644 core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRunner.java create mode 100644 core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java create mode 100644 core/src/test/java/org/jsmart/zerocode/core/db/DbSqlRunnerTest.java create mode 100644 core/src/test/resources/db_test.properties create mode 100644 core/src/test/resources/integration_test_files/db/db_sql_execute.json diff --git a/core/pom.xml b/core/pom.xml index f59e00aa..5878b99a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -175,6 +175,10 @@ micro-simulator test + + commons-dbutils + commons-dbutils + com.h2database h2 diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlExecutor.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlExecutor.java new file mode 100644 index 00000000..dbdb6a41 --- /dev/null +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlExecutor.java @@ -0,0 +1,83 @@ +package org.jsmart.zerocode.core.db; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import com.google.inject.name.Named; + +import org.apache.commons.dbutils.DbUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Interaction with a database using SQL to read/write + * Requires the appropriated connection data in the target environment + * properties, see src/test/resources/db_test.properties + */ +public class DbSqlExecutor { + private static final Logger LOGGER = LoggerFactory.getLogger(DbSqlExecutor.class); + public static final String SQL_RESULTS_KEY = "rows"; + + @Inject + @Named("db.driver.url") private String url; + + @Inject(optional = true) + @Named("db.driver.user") private String user; + + @Inject(optional = true) + @Named("db.driver.password") private String password; + + /** + * The EXECUTE operation returns the records retrieved by the SQL specified in the request + * under the key "rows" (select) or an empty object (insert, update) + */ + public Map EXECUTE(DbSqlRequest request) { + return execute(request); + } + + public Map execute(DbSqlRequest request) { + Connection conn = createAndGetConnection(); + try { + LOGGER.info("Execute SQL, request -> {} ", request); + DbSqlRunner runner = new DbSqlRunner(conn); + List> results = runner.execute(request.getSql(), request.getSqlParams()); + Map response = new HashMap<>(); + if (results == null) { // will return empty node, use "verify":{} + response.put(SQL_RESULTS_KEY, new ObjectMapper().createObjectNode()); + } else { + response.put(SQL_RESULTS_KEY, results); + } + return response; + } catch (SQLException e) { + LOGGER.error("Failed to execute SQL", e); + throw new RuntimeException(e); + } finally { + closeConnection(conn); + } + } + + /** + * Returns a new JDBC connection using DriverManager. + * Override this method in case you get the connections using another approach + * (e.g. DataSource) + */ + protected Connection createAndGetConnection() { + LOGGER.info("Create and get connection, url: {}, user: {}", url, user); + try { + return DriverManager.getConnection(url, user, password); + } catch (SQLException e) { + LOGGER.error("Failed to create connection", e); + throw new RuntimeException(e); + } + } + + protected void closeConnection(Connection conn) { + DbUtils.closeQuietly(conn); + } + +} \ No newline at end of file diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRequest.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRequest.java new file mode 100644 index 00000000..a1537bf2 --- /dev/null +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRequest.java @@ -0,0 +1,36 @@ +package org.jsmart.zerocode.core.db; + +import java.util.Arrays; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class DbSqlRequest { + private final String sql; + private final Object[] sqlParams; + + @JsonCreator + public DbSqlRequest(@JsonProperty("sqlStatement") String sql, + @JsonProperty("sqlParams") Object[] sqlParams) { + this.sql = sql; + this.sqlParams = sqlParams; + } + + public String getSql() { + return sql; + } + + public Object[] getSqlParams() { + return sqlParams; + } + + @Override + public String toString() { + return "Request{" + + "sql=" + sql + + ", sqlParams=" + (sqlParams == null ? "[]" : Arrays.asList(sqlParams).toString()) + + '}'; + } +} diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRunner.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRunner.java new file mode 100644 index 00000000..ff8ad488 --- /dev/null +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRunner.java @@ -0,0 +1,42 @@ +package org.jsmart.zerocode.core.db; + +import org.apache.commons.dbutils.QueryRunner; +import org.apache.commons.dbutils.handlers.MapListHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; + +class DbSqlRunner { + private static final Logger LOGGER = LoggerFactory.getLogger(DbSqlRunner.class); + private Connection conn; + + public DbSqlRunner(Connection conn) { + this.conn = conn; + } + + /** + * Execute an sql with parameters (optional) and returns a list of maps + * with the ResultSet content (select) or null (insert, update) + */ + List> execute(String sql, Object[] params) throws SQLException { + // As there is only one execute operation instead of separate update and query, + // the DbUtils execute method returns a list containing each ResultSet (each is a list of maps): + // - Empty (insert and update) + // - With one or more ResultSets (select). + // - Note that some drivers never return more than one ResultSet (e.g. H2) + QueryRunner runner = new QueryRunner(); + List>> result = runner.execute(conn, sql, new MapListHandler(), params); + if (result.isEmpty()) { + return null; + } else { + if (result.size() > 1) + LOGGER.warn("The SQL query returned more than one ResultSet, keeping only the first one"); + return result.get(0); + } + } + +} \ No newline at end of file diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java new file mode 100644 index 00000000..3856b4b0 --- /dev/null +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java @@ -0,0 +1,17 @@ +package org.jsmart.zerocode.core.db; +import org.jsmart.zerocode.core.domain.Scenario; +import org.jsmart.zerocode.core.domain.TargetEnv; +import org.jsmart.zerocode.core.runner.ZeroCodeUnitRunner; +import org.junit.Test; +import org.junit.runner.RunWith; + +@TargetEnv("db_test.properties") +@RunWith(ZeroCodeUnitRunner.class) +public class DbSqlExecutorScenarioTest { + + @Test + @Scenario("integration_test_files/db/db_sql_execute.json") + public void testDbSqlExecute() throws Exception { + } + +} diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlRunnerTest.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlRunnerTest.java new file mode 100644 index 00000000..5d267754 --- /dev/null +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlRunnerTest.java @@ -0,0 +1,95 @@ +package org.jsmart.zerocode.core.db; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.apache.commons.dbutils.DbUtils; +import org.apache.commons.dbutils.QueryRunner; +import org.jsmart.zerocode.core.utils.PropertiesProviderUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class DbSqlRunnerTest { + + private static final String DB_PROPERTIES_RESOURCE = "db_test.properties"; + private Connection conn; + + @BeforeClass + public static void classSetUp() throws FileNotFoundException, SQLException, IOException { + Connection createConn = connect(); + new QueryRunner().update(createConn, "DROP TABLE IF EXISTS SQLTABLE; " + + "CREATE TABLE SQLTABLE (ID INTEGER, NAME VARCHAR(20)); "); + DbUtils.closeQuietly(createConn); + } + + @Before + public void setUp() throws ClassNotFoundException, SQLException, FileNotFoundException, IOException { + conn = connect(); + new QueryRunner().update(conn, "DELETE FROM SQLTABLE; " + + "INSERT INTO SQLTABLE VALUES (1, 'string 1'); " + + "INSERT INTO SQLTABLE VALUES (2, 'string 2');"); + } + + @After + public void tearDown() throws Exception { + DbUtils.closeQuietly(conn); + } + + private static Connection connect() throws SQLException, FileNotFoundException, IOException { + Properties prop = PropertiesProviderUtils.getProperties(DB_PROPERTIES_RESOURCE); + return DriverManager.getConnection( + prop.getProperty("db.driver.url"), prop.getProperty("db.driver.user"), prop.getProperty("db.driver.password") ); + } + + private List> execute(String sql, Object[] params) throws SQLException { + DbSqlRunner runner = new DbSqlRunner(conn); + return runner.execute(sql, params); + } + + @Test + public void sqlSelectQueryShouldReturnListOfMap() throws ClassNotFoundException, SQLException { + List> rows = execute("SELECT ID, NAME FROM SQLTABLE ORDER BY ID DESC", null); + assertThat(rows.toString(), equalTo("[{ID=2, NAME=string 2}, {ID=1, NAME=string 1}]")); + } + + @Test + public void sqlSelectWithoutResultsShouldReturnEmptyList() throws ClassNotFoundException, SQLException { + List> rows = execute("SELECT ID, NAME FROM SQLTABLE where ID<0", null); + assertThat(rows.toString(), equalTo("[]")); + } + + @Test + public void multipleSqlSelectShouldReturnTheFirstResultSet() throws ClassNotFoundException, SQLException { + List> rows = execute("SELECT ID, NAME FROM SQLTABLE where ID=2; SELECT ID, NAME FROM SQLTABLE where ID=1;", null); + assertThat(rows.toString(), equalTo("[{ID=2, NAME=string 2}]")); + } + + @Test + public void sqlInsertShouldReturnNull() throws ClassNotFoundException, SQLException { + Object nullRows = execute("INSERT INTO SQLTABLE VALUES (3, 'string 3')", null); + assertThat(nullRows, nullValue()); + // check rows are inserted + List> rows = execute("SELECT ID, NAME FROM SQLTABLE ORDER BY ID", new Object[] {}); + assertThat(rows.toString(), equalTo("[{ID=1, NAME=string 1}, {ID=2, NAME=string 2}, {ID=3, NAME=string 3}]")); + } + + @Test + public void executeWithParametersShouldAllowNulls() throws SQLException { + execute("INSERT INTO SQLTABLE VALUES (?, ?)", new Object[] { 4, null }); + List> rows = execute("SELECT ID, NAME FROM SQLTABLE where ID = ?", new Object[] { 4 }); + assertThat(rows.toString(), equalTo("[{ID=4, NAME=null}]")); + } + +} diff --git a/core/src/test/resources/db_test.properties b/core/src/test/resources/db_test.properties new file mode 100644 index 00000000..0f741ab7 --- /dev/null +++ b/core/src/test/resources/db_test.properties @@ -0,0 +1,15 @@ +# Connection info used by the DbSqlExecutor + +# JDBC connection string to the test database (H2) +db.driver.url=jdbc:h2:./target/test_db_sql_executor +# If connection requires authentication, specify user and password: +# db.driver.user= +# db.driver.password= + +# To run the tests with postgres: +# - run container: docker run --name zerocode-postgres -p:5432:5432 -e POSTGRES_PASSWORD=mypassword -d postgres +# - add the driver dependency to the pom.xml: https://central.sonatype.com/artifact/org.postgresql/postgresql +# - and uncomment these properties +# db.driver.url=jdbc:postgresql://localhost:5432/postgres +# db.driver.user=postgres +# db.driver.password=mypassword diff --git a/core/src/test/resources/integration_test_files/db/db_sql_execute.json b/core/src/test/resources/integration_test_files/db/db_sql_execute.json new file mode 100644 index 00000000..1ec9ffe2 --- /dev/null +++ b/core/src/test/resources/integration_test_files/db/db_sql_execute.json @@ -0,0 +1,49 @@ +{ + "scenarioName": "DbSqlExecutor: Read and write data using SQL", + "steps": [ + { + "name": "Test database setup", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "DROP TABLE IF EXISTS PEOPLE; CREATE TABLE PEOPLE (ID INTEGER, NAME VARCHAR(20), START DATE, ACTIVE BOOLEAN);" + }, + "verify": { } + }, + { + "name": "Insert rows using SQL", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "INSERT INTO PEOPLE VALUES (1, 'Jeff Bejo', '2024-09-01', true); INSERT INTO PEOPLE VALUES (2, 'John Bajo', '2024-09-02', false);" + }, + "verify": { } + }, + { + "name": "Insert with parameters and nulls", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "execute", //<-- Uppercase for consistency, but also allows lowercase + "request": { + "sql": "INSERT INTO PEOPLE (ID, NAME, START, ACTIVE) VALUES (?, ?, ?, ?);", + "sqlParams": [3, null, null, true] + }, + "verify": { } + }, + { + "name": "Retrieve rows using SQL", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "SELECT ID, NAME, to_char(START,'yyyy-MM-dd') AS START, ACTIVE FROM PEOPLE WHERE ACTIVE=?", + "sqlParams": [true] + }, + "verify": { + "rows.SIZE": 2, + "rows": [ + { "ID": 1, "NAME": "Jeff Bejo", "START": "2024-09-01", "ACTIVE": true }, + { "ID": 3, "NAME": null, "START": null, "ACTIVE": true } + ] + } + } + ] +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index c9658480..b34a4d4a 100644 --- a/pom.xml +++ b/pom.xml @@ -84,6 +84,7 @@ 4.4 3.14.0 1.11.0 + 1.8.1 1.1.10 4.5.13 4.5.12 @@ -287,6 +288,11 @@ ${micro-simulator.version} test + + commons-dbutils + commons-dbutils + ${commons-dbutils.version} + com.h2database h2 From 1d2edb37f90c73efd3617a206e2a5d90455960b2 Mon Sep 17 00:00:00 2001 From: Javier <10879637+javiertuya@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:59:08 +0200 Subject: [PATCH 2/6] ISSUE-680 # Add value converter and refactor tests --- core/pom.xml | 8 + .../zerocode/core/db/DbValueConverter.java | 155 ++++++++++++++++++ .../zerocode/core/db/DbSqlRunnerTest.java | 49 +----- .../jsmart/zerocode/core/db/DbTestBase.java | 53 ++++++ .../core/db/DbValueConverterTest.java | 150 +++++++++++++++++ .../db/db_sql_execute.json | 2 +- 6 files changed, 375 insertions(+), 42 deletions(-) create mode 100644 core/src/main/java/org/jsmart/zerocode/core/db/DbValueConverter.java create mode 100644 core/src/test/java/org/jsmart/zerocode/core/db/DbTestBase.java create mode 100644 core/src/test/java/org/jsmart/zerocode/core/db/DbValueConverterTest.java diff --git a/core/pom.xml b/core/pom.xml index 5878b99a..8e5593e5 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -184,6 +184,14 @@ h2 test + com.aventstack extentreports diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbValueConverter.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbValueConverter.java new file mode 100644 index 00000000..149ae885 --- /dev/null +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbValueConverter.java @@ -0,0 +1,155 @@ +package org.jsmart.zerocode.core.db; + +import static org.apache.commons.lang3.time.DateUtils.parseDate; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.text.ParseException; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Conversion of string values to be inserted in the database + * into objects compatible with the java.sql type of the target columns. + */ +public class DbValueConverter { + private static final Logger LOGGER = LoggerFactory.getLogger(DbSqlExecutor.class); + + private Connection conn; + private String table; + private DatabaseMetaData databaseMetaData; + public Map columnTypes; // java.sql.Types + + public DbValueConverter(Connection conn, String table) { + this.conn = conn; + this.table = table; + try { + initializeMetadata(); + } catch (Exception e) { + logInitializeError(); + } + } + + private void initializeMetadata() throws SQLException { + LOGGER.info("Metadata initialization for table: {}", table); + columnTypes = new LinkedHashMap<>(); // must keep column order + databaseMetaData = conn.getMetaData(); + + table = convertToStoredCase(table); // to locate table name in metadata + LOGGER.info("Database storesLowerCaseIdentifiers={}, storesUpperCaseIdentifiers={}", + databaseMetaData.storesLowerCaseIdentifiers(), databaseMetaData.storesUpperCaseIdentifiers()); + + try (ResultSet rs = databaseMetaData.getColumns(null, null, table, "%")) { + while (rs.next()) { + String storedName = rs.getString("COLUMN_NAME"); + int typeValue = rs.getInt("DATA_TYPE"); + // internally, key is lowercase to allow case insensitive lookups + columnTypes.put(storedName.toLowerCase(), typeValue); + } + } + LOGGER.info("Mapping from java columns to sql types: {}", columnTypes.toString()); + if (columnTypes.isEmpty()) + logInitializeError(); + } + + private String convertToStoredCase(String identifier) throws SQLException { + if (databaseMetaData.storesLowerCaseIdentifiers()) + identifier = identifier.toLowerCase(); + else if (databaseMetaData.storesUpperCaseIdentifiers()) + identifier = identifier.toUpperCase(); + return identifier; + } + + private void logInitializeError() { + LOGGER.error("Initialization of metadata for table {} failed. " + + "Errors may appear when matching query parameters to their data types", table); + } + + /** + * Given an array of column names and their corresponding values (as strings) + * transforms each value to the compatible data type that allow to be inserted in the database. + * If the column names are missing, uses all columns in the current table as fallback. + */ + Object[] convertColumnValues(String[] columns, String[] values) { + if (columns == null || columns.length == 0) // if no specified, use all columns in the table + columns = columnTypes.keySet().toArray(new String[0]); + + Object[] converted = new Object[values.length]; + for (int i = 0; i < values.length; i++) { + converted[i] = i < columns.length && i < values.length + ? convertColumnValue(columns[i], values[i]) + : values[i]; + } + return converted; + } + + private Object convertColumnValue(String column, String value) { + try { + return convertColumnValueWithThrow(column, value); + } catch (ParseException e) { + LOGGER.error("Can't convert the data type of value {} at column {}", value, column); + return value; + } + } + + /** + * Converts the string representation of a data type value into the appropriate simple sql data type. + * If a data type is not handled by this method, returns the input value as fallback. + * + * See table B-1 in JDBC 4.2 Specification + */ + private Object convertColumnValueWithThrow(String column, String value) throws ParseException { + if (value == null) + return null; + if (!columnTypes.containsKey(column.toLowerCase())) // fallback if no metadata + return value; + + int sqlType = columnTypes.get(column.toLowerCase()); + return convertColumnValueFromJavaSqlType(sqlType, value); + } + + private Object convertColumnValueFromJavaSqlType(int sqlType, String value) throws ParseException { + switch (sqlType) { + case java.sql.Types.NUMERIC: + case java.sql.Types.DECIMAL: return java.math.BigDecimal.valueOf(Double.parseDouble(value)); + + case java.sql.Types.BIT: //accepts "1" as true (e.g. SqlServer) + case java.sql.Types.BOOLEAN: return Boolean.valueOf("1".equals(value) ? "true" : value); + + case java.sql.Types.TINYINT: return Byte.valueOf(value); + case java.sql.Types.SMALLINT: return Short.valueOf(value); + case java.sql.Types.INTEGER: return Integer.valueOf(value); + case java.sql.Types.BIGINT: return Long.valueOf(value); + + case java.sql.Types.REAL: return Float.valueOf(value); + case java.sql.Types.FLOAT: return Double.valueOf(value); + case java.sql.Types.DOUBLE: return Double.valueOf(value); + + case java.sql.Types.DATE: return new java.sql.Date(parseDate(value, getDateFormats()).getTime()); + case java.sql.Types.TIME: return new java.sql.Time(parseDate(value, getTimeFormats()).getTime()); + case java.sql.Types.TIMESTAMP: return new java.sql.Timestamp(parseDate(value, getTimestampFormats()).getTime()); + default: + return value; + } + } + + // Currently, supported date time formats are a few common ISO-8601 formats + // (other common format strings in org.apache.commons.lang3.time.DateFormatUtils) + // This may be made user configurable later, via properties and/or embedded in the payload + + private String[] getDateFormats() { + return new String[] {"yyyy-MM-dd"}; + } + private String[] getTimeFormats() { + return new String[] {"HH:mm:ssZ", "HH:mm:ss.SSSZ"}; + } + private String[] getTimestampFormats() { + return new String[] {"yyyy-MM-dd'T'HH:mm:ssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ"}; + } + +} diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlRunnerTest.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlRunnerTest.java index 5d267754..59fc19e6 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlRunnerTest.java +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlRunnerTest.java @@ -6,62 +6,29 @@ import java.io.FileNotFoundException; import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.SQLException; import java.util.List; import java.util.Map; -import java.util.Properties; -import org.apache.commons.dbutils.DbUtils; import org.apache.commons.dbutils.QueryRunner; -import org.jsmart.zerocode.core.utils.PropertiesProviderUtils; -import org.junit.After; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; -public class DbSqlRunnerTest { - - private static final String DB_PROPERTIES_RESOURCE = "db_test.properties"; - private Connection conn; - - @BeforeClass - public static void classSetUp() throws FileNotFoundException, SQLException, IOException { - Connection createConn = connect(); - new QueryRunner().update(createConn, "DROP TABLE IF EXISTS SQLTABLE; " - + "CREATE TABLE SQLTABLE (ID INTEGER, NAME VARCHAR(20)); "); - DbUtils.closeQuietly(createConn); - } +public class DbSqlRunnerTest extends DbTestBase { @Before public void setUp() throws ClassNotFoundException, SQLException, FileNotFoundException, IOException { - conn = connect(); - new QueryRunner().update(conn, "DELETE FROM SQLTABLE; " + super.setUp(); + new QueryRunner().update(conn, "DROP TABLE IF EXISTS SQLTABLE; " + + "CREATE TABLE SQLTABLE (ID INTEGER, NAME VARCHAR(20)); " + "INSERT INTO SQLTABLE VALUES (1, 'string 1'); " + "INSERT INTO SQLTABLE VALUES (2, 'string 2');"); } - @After - public void tearDown() throws Exception { - DbUtils.closeQuietly(conn); - } - - private static Connection connect() throws SQLException, FileNotFoundException, IOException { - Properties prop = PropertiesProviderUtils.getProperties(DB_PROPERTIES_RESOURCE); - return DriverManager.getConnection( - prop.getProperty("db.driver.url"), prop.getProperty("db.driver.user"), prop.getProperty("db.driver.password") ); - } - - private List> execute(String sql, Object[] params) throws SQLException { - DbSqlRunner runner = new DbSqlRunner(conn); - return runner.execute(sql, params); - } - @Test public void sqlSelectQueryShouldReturnListOfMap() throws ClassNotFoundException, SQLException { List> rows = execute("SELECT ID, NAME FROM SQLTABLE ORDER BY ID DESC", null); - assertThat(rows.toString(), equalTo("[{ID=2, NAME=string 2}, {ID=1, NAME=string 1}]")); + assertThat(rows.toString(), equalTo(convertDbCase("[{ID=2, NAME=string 2}, {ID=1, NAME=string 1}]"))); } @Test @@ -73,7 +40,7 @@ public void sqlSelectWithoutResultsShouldReturnEmptyList() throws ClassNotFoundE @Test public void multipleSqlSelectShouldReturnTheFirstResultSet() throws ClassNotFoundException, SQLException { List> rows = execute("SELECT ID, NAME FROM SQLTABLE where ID=2; SELECT ID, NAME FROM SQLTABLE where ID=1;", null); - assertThat(rows.toString(), equalTo("[{ID=2, NAME=string 2}]")); + assertThat(rows.toString(), equalTo(convertDbCase("[{ID=2, NAME=string 2}]"))); } @Test @@ -82,14 +49,14 @@ public void sqlInsertShouldReturnNull() throws ClassNotFoundException, SQLExcept assertThat(nullRows, nullValue()); // check rows are inserted List> rows = execute("SELECT ID, NAME FROM SQLTABLE ORDER BY ID", new Object[] {}); - assertThat(rows.toString(), equalTo("[{ID=1, NAME=string 1}, {ID=2, NAME=string 2}, {ID=3, NAME=string 3}]")); + assertThat(rows.toString(), equalTo(convertDbCase("[{ID=1, NAME=string 1}, {ID=2, NAME=string 2}, {ID=3, NAME=string 3}]"))); } @Test public void executeWithParametersShouldAllowNulls() throws SQLException { execute("INSERT INTO SQLTABLE VALUES (?, ?)", new Object[] { 4, null }); List> rows = execute("SELECT ID, NAME FROM SQLTABLE where ID = ?", new Object[] { 4 }); - assertThat(rows.toString(), equalTo("[{ID=4, NAME=null}]")); + assertThat(rows.toString(), equalTo(convertDbCase("[{ID=4, NAME=null}]"))); } } diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbTestBase.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbTestBase.java new file mode 100644 index 00000000..6fcb2a70 --- /dev/null +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbTestBase.java @@ -0,0 +1,53 @@ +package org.jsmart.zerocode.core.db; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.apache.commons.dbutils.DbUtils; +import org.jsmart.zerocode.core.utils.PropertiesProviderUtils; +import org.junit.After; +import org.junit.Before; + +/** + * Base class for the unit DB test classes: manages connections, + * execution of queries and DBMS specific features + */ +public class DbTestBase { + + private static final String DB_PROPERTIES_RESOURCE = "db_test.properties"; + protected Connection conn; // managed connection for each test + protected boolean isPostgres = false; // set by each connection, to allow portable assertions (postgres is lowercase) + + @Before + public void setUp() throws ClassNotFoundException, SQLException, FileNotFoundException, IOException { + conn = connect(); + } + + @After + public void tearDown() throws Exception { + DbUtils.closeQuietly(conn); + } + + protected Connection connect() throws SQLException { + Properties prop = PropertiesProviderUtils.getProperties(DB_PROPERTIES_RESOURCE); + isPostgres = prop.getProperty("db.driver.url").startsWith("jdbc:postgresql:"); + return DriverManager.getConnection( + prop.getProperty("db.driver.url"), prop.getProperty("db.driver.user"), prop.getProperty("db.driver.password") ); + } + + protected List> execute(String sql, Object[] params) throws SQLException { + DbSqlRunner runner = new DbSqlRunner(conn); + return runner.execute(sql, params); + } + + protected String convertDbCase(String value) { + return isPostgres ? value.toLowerCase() : value; + } + +} \ No newline at end of file diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbValueConverterTest.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbValueConverterTest.java new file mode 100644 index 00000000..fc9665f8 --- /dev/null +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbValueConverterTest.java @@ -0,0 +1,150 @@ +package org.jsmart.zerocode.core.db; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertThrows; + +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; + +import org.junit.Test; + +public class DbValueConverterTest extends DbTestBase { + + @Test + public void convertBasicDataTypeValues() throws SQLException { + doTestConversion("", "Btable", "VINT INTEGER, VCHAR VARCHAR(20), VBOOL BOOLEAN, EXTRA CHAR(1)", + new String[] { "Vint", "Vchar", "Vbool" }, + new String[] { "101", "astring", "true" }, + "[{VINT=101, VCHAR=astring, VBOOL=true, EXTRA=null}]"); + } + + @Test + public void convertNullAndBitValues() throws SQLException { + doTestConversion("", "NTABLE", "VINT INT, VCHAR VARCHAR(20), VBOOL BOOLEAN", + new String[] { "VINT", "VCHAR", "VBOOL" }, + new String[] { null, null, "1" }, // incl. alternate boolean + "[{VINT=null, VCHAR=null, VBOOL=true}]"); + } + + @Test + public void convertDecimalAndFloatValues() throws SQLException { + doTestConversion("", "FTABLE", "VEXACT NUMERIC(10,0), VDEC DECIMAL(10,2), VFLOAT FLOAT, VREAL REAL", + new String[] { "VEXACT", "VDEC", "VFLOAT", "VREAL" }, + new String[] { "102", "123.45", "234.56", "3.4561E+2" }, + "[{VEXACT=102, VDEC=123.45, VFLOAT=234.56, VREAL=345.61}]"); + } + + @Test + public void convertDateAndTimeValues() throws SQLException { + List> rows = doTestConversion("", "DTABLE", "VTS1 TIMESTAMP, VTS2 TIMESTAMP, VTIME TIME, VDATE DATE", + new String[] { "VTS1", "VTS2", "VTIME", "VDATE" }, + new String[] { "2024-09-04T08:01:02.456+0300", "2024-09-04T08:01:02+0300", "08:01:02+0300", "2024-09-04" }, + null); + // assert individually to allow compare with GMT time (not local) + assertThat(gmtTimestamp((Date) rows.get(0).get("VTS1")), equalTo("2024-09-04T05:01:02.456")); + assertThat(gmtTimestamp((Date) rows.get(0).get("VTS2")), equalTo("2024-09-04T05:01:02.000")); + assertThat(gmtTimestamp((Date) rows.get(0).get("VTIME")), equalTo("1970-01-01T05:01:02.000")); + assertThat(rows.get(0).get("VDATE").toString(), "2024-09-04", equalTo(rows.get(0).get("VDATE").toString())); + } + + @Test + public void convertWithLowerAndUppercase() throws SQLException { + // Test date types to ensure that is the converter who makes the conversion, not the driver + List> rows = doTestConversion("", "ITable", "VDATE DATE", + new String[] { "VDate" }, + new String[] { "2024-09-04" }, + "[{VDATE=2024-09-04}]"); + assertThat(rows.get(0).get("VDATE").toString(), equalTo("2024-09-04")); + } + + @Test + public void whenNoColumnsSpecified_ThenAllTableColumns_AreIncluded() throws SQLException { + doTestConversion("", "OTABLE", "VINT SMALLINT, VCHAR VARCHAR(20)", + null, + new String[] { "101", "astring" }, + "[{VINT=101, VCHAR=astring}]"); + } + + @Test + public void whenColumnNotFound_ThenConversionFails() throws SQLException { + assertThrows(SQLException.class, () -> { + doTestConversion("", "FCTABLE", "VINT INTEGER, VCHAR VARCHAR(20)", + new String[] { "VINT", "NOTEXISTS" }, + new String[] { "101", "astring" }, + "[{VINT=101, VCHAR=notexists}]"); + }); + } + + // Failures due to problems with metadata: + // - These tests will pass because the H2 driver converts numeric values + // - but fail if driver is changed to Postgres (does not convert numeric), skipped + + @Test + public void whenMetadataNotFound_ThenConversions_AreUpToTheDriver_WithColumns() throws SQLException { + if (isPostgres) + return; + doTestConversion("table", "F1TABLE", "VINT INTEGER, VCHAR VARCHAR(20)", + new String[] { "VINT", "VCHAR" }, + new String[] { "101", "astring" }, + "[{VINT=101, VCHAR=astring}]"); + } + + @Test + public void whenMetadataNotFound_ThenConversions_AreUpToTheDriver_WithoutColumns() throws SQLException { + if (isPostgres) + return; + doTestConversion("table", "F2CTABLE", "VINT INTEGER, VCHAR VARCHAR(20)", + null, + new String[] { "101", "astring" }, + "[{VINT=101, VCHAR=astring}]"); + } + + @Test + public void whenMetadataFails_ThenConversions_AreUpToTheDriver() throws SQLException { + if (isPostgres) + return; + doTestConversion("conn", "F3TABLE", "VINT INTEGER, VCHAR VARCHAR(20)", + new String[] { "VINT", "VCHAR" }, + new String[] { "101", "astring" }, + "[{VINT=101, VCHAR=astring}]"); + } + + private List> doTestConversion(String failureToSimulate, String table, + String ddlTypes, String[] columns, String[] params, String expected) throws SQLException { + execute("DROP TABLE IF EXISTS " + table + ";" + + " CREATE TABLE " + table + " (" + ddlTypes + ");", null); + String sql = "INSERT INTO " + table + + (columns != null ? " (" + String.join(",", columns) + ")" : "") + + " VALUES (" + placeholders(params.length) + ")"; + + DbValueConverter converter = new DbValueConverter( + "conn".equals(failureToSimulate) ? null : conn, + "table".equals(failureToSimulate) ? "notexists" : table); + Object[] converted = converter.convertColumnValues(columns, params); + execute(sql, converted); + + List> rows = execute("SELECT * FROM " + table, null); + if (expected != null) // null to check without specified columns + assertThat(rows.toString(), equalTo(convertDbCase(expected))); + return rows; + } + + private String placeholders(int columnCount) { + String[] placeholders = new String[columnCount]; + Arrays.fill(placeholders, "?"); + return String.join(",", placeholders); + } + + private String gmtTimestamp(Date dt) { + java.text.DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS"); + df.setTimeZone(TimeZone.getTimeZone("GMT")); + return df.format(dt); + } + +} diff --git a/core/src/test/resources/integration_test_files/db/db_sql_execute.json b/core/src/test/resources/integration_test_files/db/db_sql_execute.json index 1ec9ffe2..6e0d0daf 100644 --- a/core/src/test/resources/integration_test_files/db/db_sql_execute.json +++ b/core/src/test/resources/integration_test_files/db/db_sql_execute.json @@ -39,7 +39,7 @@ }, "verify": { "rows.SIZE": 2, - "rows": [ + "rows": [ //<-- to make this pass in postgres, set the keys to lowercase { "ID": 1, "NAME": "Jeff Bejo", "START": "2024-09-01", "ACTIVE": true }, { "ID": 3, "NAME": null, "START": null, "ACTIVE": true } ] From 5972efb3dcac927d747c318531f39504d3f90858 Mon Sep 17 00:00:00 2001 From: Javier <10879637+javiertuya@users.noreply.github.com> Date: Sat, 5 Oct 2024 18:26:44 +0200 Subject: [PATCH 3/6] ISSUE-680 # Add database CSV loader --- .../jsmart/zerocode/core/db/DbCsvLoader.java | 132 ++++++++++++++++++ .../jsmart/zerocode/core/db/DbCsvRequest.java | 101 ++++++++++++++ .../zerocode/core/db/DbSqlExecutor.java | 31 ++++ .../zerocode/core/db/DbCsvLoaderTest.java | 120 ++++++++++++++++ .../core/db/DbSqlExecutorScenarioTest.java | 10 ++ .../zerocode/core/db/DbSqlRunnerTest.java | 7 +- .../jsmart/zerocode/core/db/DbTestBase.java | 37 +++-- .../core/db/DbValueConverterTest.java | 3 + .../db/db_csv_load_with_headers.json | 43 ++++++ .../db/db_csv_load_without_headers.json | 43 ++++++ .../db/players_with_headers.csv | 4 + .../db/players_without_headers.csv | 3 + 12 files changed, 520 insertions(+), 14 deletions(-) create mode 100644 core/src/main/java/org/jsmart/zerocode/core/db/DbCsvLoader.java create mode 100644 core/src/main/java/org/jsmart/zerocode/core/db/DbCsvRequest.java create mode 100644 core/src/test/java/org/jsmart/zerocode/core/db/DbCsvLoaderTest.java create mode 100644 core/src/test/resources/integration_test_files/db/db_csv_load_with_headers.json create mode 100644 core/src/test/resources/integration_test_files/db/db_csv_load_without_headers.json create mode 100644 core/src/test/resources/integration_test_files/db/players_with_headers.csv create mode 100644 core/src/test/resources/integration_test_files/db/players_without_headers.csv diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbCsvLoader.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbCsvLoader.java new file mode 100644 index 00000000..72896878 --- /dev/null +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbCsvLoader.java @@ -0,0 +1,132 @@ +package org.jsmart.zerocode.core.db; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import org.apache.commons.dbutils.QueryRunner; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.univocity.parsers.csv.CsvParser; + +class DbCsvLoader { + private static final Logger LOGGER = LoggerFactory.getLogger(DbCsvLoader.class); + private Connection conn; + private CsvParser csvParser; + + public DbCsvLoader(Connection conn, CsvParser csvParser) { + this.conn = conn; + this.csvParser = csvParser; + } + + /** + * Loads rows in csv format (csvLines) into a table in the database + * and returns the total number of rows + */ + int loadCsv(String table, List csvLines, boolean withHeaders, String nullString) throws SQLException { + if (csvLines == null || csvLines.isEmpty()) + return 0; + + List lines = parseLines(table, csvLines); + + String[] headers = buildHeaders(lines.get(0), withHeaders); + List paramset = buildParameters(table, headers, lines, withHeaders, nullString); + if (paramset.isEmpty()) // can have headers, but no rows + return 0; + + String sql = buildSql(table, headers, paramset.get(0).length); + LOGGER.info("Loading CSV using this sql: {}", sql); + + QueryRunner runner = new QueryRunner(); + int insertCount = 0; + for (int i = 0 ; i < paramset.size(); i++) { + insertRow(runner, i, sql, paramset.get(i)); + insertCount++; + } + LOGGER.info("Total of rows inserted: {}", insertCount); + return insertCount; + } + + private List parseLines(String table, List lines) { + int numCol = 0; // will check that every row has same columns than the first + List parsedLines = new ArrayList<>(); + for (int i = 0; i buildParameters(String table, String[] headers, List lines, boolean withHeaders, String nullString) { + DbValueConverter converter = new DbValueConverter(conn, table); + List paramset = new ArrayList<>(); + for (int i = withHeaders ? 1 : 0; i < lines.size(); i++) { + String[] parsedLine = lines.get(i); + parsedLine = processNulls(parsedLine, nullString); + Object[] params; + try { + params = converter.convertColumnValues(headers, parsedLine); + LOGGER.info(" row [{}] params: {}", i + 1, Arrays.asList(params).toString()); + } catch (Exception e) { // Not only SQLException as converter also does parsing + String message = String.format("Error matching data type of parameters and table columns at CSV row %d", i + 1); + LOGGER.error(message); // do not log the exception because it will be logged by the parent executor (DbCsvLoader) + LOGGER.error("Exception message: {}", e.getMessage()); + throw new RuntimeException(message, e); + } + paramset.add(params); + } + return paramset; + } + + private String[] processNulls(String[] line, String nullString) { + for (int i = 0; i < line.length; i++) { + if (StringUtils.isBlank(nullString) && StringUtils.isBlank(line[i])) { + line[i] = null; + } else if (!StringUtils.isBlank(nullString)) { + if (StringUtils.isBlank(line[i])) // null must be empty string + line[i] = ""; + else if (nullString.trim().equalsIgnoreCase(line[i].trim())) + line[i] = null; + } + } + return line; + } + + private String buildSql(String table, String[] headers, int columnCount) { + String placeholders = IntStream.range(0, columnCount) + .mapToObj(i -> "?").collect(Collectors.joining(",")); + return "INSERT INTO " + table + + (headers.length > 0 ? " (" + String.join(",", headers) + ")" : "") + + " VALUES (" + placeholders + ");"; + } + + private void insertRow(QueryRunner runner, int rowId, String sql, Object[] params) { + try { + runner.update(conn, sql, params); + } catch (SQLException e) { + String message = String.format("Error inserting data at CSV row %d", rowId + 1); + LOGGER.error(message); // do not log the exception because it will be logged by the parent executor (DbCsvLoader) + LOGGER.error("Exception message: {}", e.getMessage()); + throw new RuntimeException(message, e); + } + } + +} \ No newline at end of file diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbCsvRequest.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbCsvRequest.java new file mode 100644 index 00000000..62eeb1a5 --- /dev/null +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbCsvRequest.java @@ -0,0 +1,101 @@ +package org.jsmart.zerocode.core.db; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import org.apache.commons.lang3.StringUtils; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +public class DbCsvRequest { + private final String tableName; + private final List csvSource; + private final Boolean withHeaders; + private final String nullString; + + public DbCsvRequest( + @JsonProperty(value="tableName", required=true) String tableName, + @JsonProperty("csvSource") JsonNode csvSourceJsonNode, + @JsonProperty("withHeaders") Boolean withHeaders, + @JsonProperty("nullString") String nullString) { + this.tableName = tableName; + this.withHeaders = Optional.ofNullable(withHeaders).orElse(false); + this.nullString = Optional.ofNullable(nullString).orElse(""); + this.csvSource = Optional.ofNullable(csvSourceJsonNode).map(this::getCsvSourceFrom).orElse(Collections.emptyList()); + } + + public String getTableName() { + return tableName; + } + + public List getCsvSource() { + return csvSource; + } + + public boolean getWithHeaders() { + return withHeaders; + } + + public String getNullString() { + return nullString; + } + + // Code below is duplicated from org.jsmart.zerocode.core.domain.Parametrized.java and not included in tests. + // TODO Consider some refactoring later (to SmartUtils?) and review error message when file not found + + private List getCsvSourceFrom(JsonNode csvSourceJsonNode) { + try { + if (csvSourceJsonNode.isArray()) { + return readCsvSourceFromJson(csvSourceJsonNode); + + } else { + return readCsvSourceFromExternalCsvFile(csvSourceJsonNode); + } + } catch (IOException e) { + throw new RuntimeException("Error deserializing csvSource", e); + } + } + + private List readCsvSourceFromJson(JsonNode csvSourceJsonNode) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + ObjectReader reader = mapper.readerFor(new TypeReference>() { + }); + return reader.readValue(csvSourceJsonNode); + } + + private List readCsvSourceFromExternalCsvFile(JsonNode csvSourceJsonNode) throws IOException { + String csvSourceFilePath = csvSourceJsonNode.textValue(); + if (StringUtils.isNotBlank(csvSourceFilePath)) { + Path path = Paths.get("./src/test/resources/",csvSourceFilePath); + List csvSourceFileLines = Files.lines(path) + .filter(StringUtils::isNotBlank) + .collect(Collectors.toList()); + //if (this.ignoreHeader) { + // return csvSourceFileLines.stream() + // .skip(1) + // .collect(Collectors.toList()); + //} + return csvSourceFileLines; + } + return Collections.emptyList(); + } + + @Override + public String toString() { + return "Parameterized{" + + "tableName=" + tableName + + ", csvSource=" + csvSource + + ", withHeaders=" + withHeaders + + ", nullString=" + nullString + + '}'; + } +} diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlExecutor.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlExecutor.java index dbdb6a41..373dc0e4 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlExecutor.java +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlExecutor.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; import com.google.inject.name.Named; +import com.univocity.parsers.csv.CsvParser; import org.apache.commons.dbutils.DbUtils; import org.slf4j.Logger; @@ -22,6 +23,7 @@ public class DbSqlExecutor { private static final Logger LOGGER = LoggerFactory.getLogger(DbSqlExecutor.class); public static final String SQL_RESULTS_KEY = "rows"; + public static final String CSV_RESULTS_KEY = "size"; @Inject @Named("db.driver.url") private String url; @@ -32,6 +34,35 @@ public class DbSqlExecutor { @Inject(optional = true) @Named("db.driver.password") private String password; + @Inject + private CsvParser csvParser; + + /** + * The LOADCSV operation inserts the content of a CSV file into a table, + * and returns the number of records inserted under the key "size" + */ + public Map LOADCSV(DbCsvRequest request) { // uppercase for consistency with http api operations + return loadcsv(request); + } + + public Map loadcsv(DbCsvRequest request) { + Connection conn = createAndGetConnection(); + try { + LOGGER.info("Load CSV, request -> {} ", request); + DbCsvLoader runner = new DbCsvLoader(conn, csvParser); + long result = runner.loadCsv(request.getTableName(), request.getCsvSource(), + request.getWithHeaders(), request.getNullString()); + Map response = new HashMap<>(); + response.put(CSV_RESULTS_KEY, result); + return response; + } catch (Exception e) { + LOGGER.error("Failed to load CSV", e); + throw new RuntimeException(e); + } finally { + closeConnection(conn); + } + } + /** * The EXECUTE operation returns the records retrieved by the SQL specified in the request * under the key "rows" (select) or an empty object (insert, update) diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbCsvLoaderTest.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbCsvLoaderTest.java new file mode 100644 index 00000000..80729ad3 --- /dev/null +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbCsvLoaderTest.java @@ -0,0 +1,120 @@ +package org.jsmart.zerocode.core.db; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertThrows; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import org.jukito.JukitoRunner; +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.google.inject.Inject; +import com.univocity.parsers.csv.CsvParser; + +@RunWith(JukitoRunner.class) +public class DbCsvLoaderTest extends DbTestBase{ + + private DbCsvLoader loader; + + @Inject + CsvParser csvParser; + + @Override + public void setUp() throws SQLException { + super.setUp(); + loader = new DbCsvLoader(conn, csvParser); + execute("DROP TABLE IF EXISTS CSVTABLE; " + + "CREATE TABLE CSVTABLE (ID INTEGER, NAME VARCHAR(20), STATUS BOOLEAN)", null); + } + + private int loadCsv(String table, String[] lines, boolean withHeaders, String nullString) throws SQLException { + List linesList = Arrays.asList(lines); + return loader.loadCsv(table, linesList, withHeaders, nullString); + } + + private void assertLoaded(int count, String expected) throws SQLException { + List> rows = execute("SELECT ID,NAME,STATUS FROM CSVTABLE ORDER BY ID NULLS FIRST", null); + assertThat(rows.toString(), equalTo(convertDbCase(expected))); + assertThat(rows.size(), equalTo(count)); + } + + @Test + public void testLoadSimpleCsvWithoutHeaders() throws SQLException { + int count = loadCsv("CSVTABLE", new String[] { "101,me,false", "102,you,true" }, false, ""); + assertLoaded(count, "[{ID=101, NAME=me, STATUS=false}, {ID=102, NAME=you, STATUS=true}]"); + } + + @Test + public void testLoadSimpleCsvWithHeaders() throws SQLException { + int count = loadCsv("CSVTABLE", new String[] { "ID,NAME,STATUS", "101,me,false", "102,you,true" }, true, ""); + assertLoaded(count, "[{ID=101, NAME=me, STATUS=false}, {ID=102, NAME=you, STATUS=true}]"); + } + + @Test + public void testLoadCsvIsNotCleanInsert() throws SQLException { + loadCsv("CSVTABLE", new String[] { "101,me,false" }, false, ""); + loadCsv("CSVTABLE", new String[] { "103,other,true" }, false, ""); + assertLoaded(2, "[{ID=101, NAME=me, STATUS=false}, {ID=103, NAME=other, STATUS=true}]"); + } + + @Test + public void whenNoDataRows_thenReturnZero() throws SQLException { + int count = loader.loadCsv("CSVTABLE", null, false, ""); + assertLoaded(count, "[]"); + count = loadCsv("CSVTABLE", new String[] { }, false, ""); //noheaders norows + assertLoaded(count, "[]"); + count = loadCsv("CSVTABLE", new String[] {"ID,NAME,STATUS" }, true, ""); //headers norows + assertLoaded(count, "[]"); + count = loadCsv("CSVTABLE", new String[] { }, true, ""); //headers missing + assertLoaded(count, "[]"); + } + + @Test + public void whenCsvValuesContainSpaces_thenValuesAreTrimmed() throws SQLException { + loadCsv("CSVTABLE", new String[] { " ID , \t NAME \r , STATUS ", " 101 ,\tmy\t name\r, false " }, true, ""); + assertLoaded(1, "[{ID=101, NAME=my\t name, STATUS=false}]"); + } + + @Test + public void whenNullStringUnset_thenEmptyIsNull() throws SQLException { + loadCsv("CSVTABLE", new String[] { " \t , me , \t ", "102,,true" }, false, ""); + assertLoaded(2, "[{ID=null, NAME=me, STATUS=null}, {ID=102, NAME=null, STATUS=true}]"); + } + + @Test + public void whenNullStringSet_thenEmptyIsNotNull_AndCaseInsensitive() throws SQLException { + loadCsv("CSVTABLE", new String[] { " null ,me, NULL ", "102, ,true" }, false, "null"); + assertLoaded(2, "[{ID=null, NAME=me, STATUS=null}, {ID=102, NAME=, STATUS=true}]"); + } + + @Test + public void whenRowsHaveDistinctSizes_thenRaiseExceptionWithMessage() throws SQLException { + RuntimeException e = assertThrows(RuntimeException.class, () -> { + loadCsv("CSVTABLE", new String[] { "ID,NAME,STATUS", "101,me,true,additional" , "102,you,true" }, true, ""); + }); + assertThat(e.getMessage(), equalTo( + "Error parsing CSV content to load into table CSVTABLE: Row 2 has 4 columns and should have 3")); + } + + @Test + public void whenParameterHasWrongType_thenRaiseExceptionWithMessage() throws SQLException { + RuntimeException e = assertThrows(RuntimeException.class, () -> { + loadCsv("CSVTABLE", new String[] { "ID,NAME,STATUS", "101,me,true" , "XXXX,you,true" }, true, ""); + }); + assertThat(e.getMessage(), equalTo("Error matching data type of parameters and table columns at CSV row 3")); + } + + @Test + public void whenInsertFails_thenRaiseExceptionWithMessage() throws SQLException { + RuntimeException e = assertThrows(RuntimeException.class, () -> { + loadCsv("CSVTABLE", new String[] { "101,me,true,extra1" , "102,you,true,extra2" }, false, ""); + }); + assertThat(e.getMessage(), equalTo("Error inserting data at CSV row 1")); + } + +} \ No newline at end of file diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java index 3856b4b0..8a9bcd14 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java @@ -9,6 +9,16 @@ @RunWith(ZeroCodeUnitRunner.class) public class DbSqlExecutorScenarioTest { + @Test + @Scenario("integration_test_files/db/db_csv_load_with_headers.json") + public void testDbCsvLoadWithHeaders() throws Exception { + } + + @Test // same scenario and test database + @Scenario("integration_test_files/db/db_csv_load_without_headers.json") + public void testDbCsvLoadWithoutHeaders() throws Exception { + } + @Test @Scenario("integration_test_files/db/db_sql_execute.json") public void testDbSqlExecute() throws Exception { diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlRunnerTest.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlRunnerTest.java index 59fc19e6..79d433f5 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlRunnerTest.java +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlRunnerTest.java @@ -4,20 +4,21 @@ import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; -import java.io.FileNotFoundException; -import java.io.IOException; import java.sql.SQLException; import java.util.List; import java.util.Map; import org.apache.commons.dbutils.QueryRunner; +import org.jukito.JukitoRunner; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +@RunWith(JukitoRunner.class) public class DbSqlRunnerTest extends DbTestBase { @Before - public void setUp() throws ClassNotFoundException, SQLException, FileNotFoundException, IOException { + public void setUp() throws SQLException { super.setUp(); new QueryRunner().update(conn, "DROP TABLE IF EXISTS SQLTABLE; " + "CREATE TABLE SQLTABLE (ID INTEGER, NAME VARCHAR(20)); " diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbTestBase.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbTestBase.java index 6fcb2a70..432cf7bd 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/db/DbTestBase.java +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbTestBase.java @@ -1,31 +1,48 @@ package org.jsmart.zerocode.core.db; -import java.io.FileNotFoundException; -import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.util.List; import java.util.Map; -import java.util.Properties; import org.apache.commons.dbutils.DbUtils; -import org.jsmart.zerocode.core.utils.PropertiesProviderUtils; +import org.jsmart.zerocode.core.di.main.ApplicationMainModule; +import org.jukito.TestModule; import org.junit.After; import org.junit.Before; +import com.google.inject.Inject; +import com.google.inject.name.Named; + /** * Base class for the unit DB test classes: manages connections, * execution of queries and DBMS specific features */ -public class DbTestBase { +public abstract class DbTestBase { + // Subclasses must use JukitoRunner + public static class JukitoModule extends TestModule { + @Override + protected void configureTest() { + ApplicationMainModule applicationMainModule = new ApplicationMainModule("db_test.properties"); + install(applicationMainModule); + } + } + + @Inject + @Named("db.driver.url") protected String url; + + @Inject(optional = true) + @Named("db.driver.user") protected String user; - private static final String DB_PROPERTIES_RESOURCE = "db_test.properties"; + @Inject(optional = true) + @Named("db.driver.password") protected String password; + protected Connection conn; // managed connection for each test protected boolean isPostgres = false; // set by each connection, to allow portable assertions (postgres is lowercase) @Before - public void setUp() throws ClassNotFoundException, SQLException, FileNotFoundException, IOException { + public void setUp() throws SQLException { conn = connect(); } @@ -35,10 +52,8 @@ public void tearDown() throws Exception { } protected Connection connect() throws SQLException { - Properties prop = PropertiesProviderUtils.getProperties(DB_PROPERTIES_RESOURCE); - isPostgres = prop.getProperty("db.driver.url").startsWith("jdbc:postgresql:"); - return DriverManager.getConnection( - prop.getProperty("db.driver.url"), prop.getProperty("db.driver.user"), prop.getProperty("db.driver.password") ); + isPostgres = url.startsWith("jdbc:postgresql:"); + return DriverManager.getConnection(url, user, password); } protected List> execute(String sql, Object[] params) throws SQLException { diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbValueConverterTest.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbValueConverterTest.java index fc9665f8..796740fe 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/db/DbValueConverterTest.java +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbValueConverterTest.java @@ -12,8 +12,11 @@ import java.util.Map; import java.util.TimeZone; +import org.jukito.JukitoRunner; import org.junit.Test; +import org.junit.runner.RunWith; +@RunWith(JukitoRunner.class) public class DbValueConverterTest extends DbTestBase { @Test diff --git a/core/src/test/resources/integration_test_files/db/db_csv_load_with_headers.json b/core/src/test/resources/integration_test_files/db/db_csv_load_with_headers.json new file mode 100644 index 00000000..0519377d --- /dev/null +++ b/core/src/test/resources/integration_test_files/db/db_csv_load_with_headers.json @@ -0,0 +1,43 @@ +{ + "scenarioName": "DbSqlExecutor: Load a CSV file with headers", + "steps": [ + { + "name": "Test database setup", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "DROP TABLE IF EXISTS PLAYERS; CREATE TABLE PLAYERS (ID INTEGER, NAME VARCHAR(20), AGE INTEGER);" + }, + "verify": { } + }, + { + "name": "Insert rows from a CSV file with headers", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "LOADCSV", + "request": { + "tableName": "players", + "csvSource": "integration_test_files/db/players_with_headers.csv", + "withHeaders" : true + }, + "verify": { + "size" : 3 + } + }, + { + "name": "Check the content of inserted rows", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "SELECT ID, NAME, AGE FROM PLAYERS ORDER BY ID" + }, + "verify": { + "rows.SIZE": 3, + "rows": [ //<-- to make this pass in postgres, set the keys to lowercase + { "ID": 1001, "NAME": "Ronaldo", "AGE": 23 }, + { "ID": 1002, "NAME": "Devaldo", "AGE": null }, + { "ID": 1003, "NAME": "Trevaldo", "AGE": 35 } + ] + } + } + ] +} \ No newline at end of file diff --git a/core/src/test/resources/integration_test_files/db/db_csv_load_without_headers.json b/core/src/test/resources/integration_test_files/db/db_csv_load_without_headers.json new file mode 100644 index 00000000..f8f093d4 --- /dev/null +++ b/core/src/test/resources/integration_test_files/db/db_csv_load_without_headers.json @@ -0,0 +1,43 @@ +{ + "scenarioName": "DbSqlExecutor: Load a CSV file without headers", + "steps": [ + { + "name": "Test database setup", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "DROP TABLE IF EXISTS PLAYERS; CREATE TABLE PLAYERS (ID INTEGER, NAME VARCHAR(20), AGE INTEGER);" + }, + "verify": { } + }, + { + "name": "Insert rows from a CSV file without headers", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "LOADCSV", + "request": { + "tableName": "players", + "csvSource": "integration_test_files/db/players_without_headers.csv", + "nullString": "NULL!!" + }, + "verify": { + "size" : 3 + } + }, + { + "name": "Check the content of inserted rows", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "SELECT ID, NAME, AGE FROM PLAYERS ORDER BY ID" + }, + "verify": { + "rows.SIZE": 3, + "rows": [ //<-- to make this pass in postgres, set the keys to lowercase + { "ID": 1001, "NAME": "Ronaldo", "AGE": 23 }, + { "ID": 1002, "NAME": null, "AGE": 24 }, + { "ID": 1003, "NAME": "Trevaldo", "AGE": 35 } + ] + } + } + ] +} \ No newline at end of file diff --git a/core/src/test/resources/integration_test_files/db/players_with_headers.csv b/core/src/test/resources/integration_test_files/db/players_with_headers.csv new file mode 100644 index 00000000..22f8c751 --- /dev/null +++ b/core/src/test/resources/integration_test_files/db/players_with_headers.csv @@ -0,0 +1,4 @@ +ID, AGE, NAME +1001, 23, Ronaldo +1002, , Devaldo +1003, 35, Trevaldo \ No newline at end of file diff --git a/core/src/test/resources/integration_test_files/db/players_without_headers.csv b/core/src/test/resources/integration_test_files/db/players_without_headers.csv new file mode 100644 index 00000000..d6f049e4 --- /dev/null +++ b/core/src/test/resources/integration_test_files/db/players_without_headers.csv @@ -0,0 +1,3 @@ +1001, Ronaldo, 23 +1002, null!!, 24 +1003, Trevaldo, 35 \ No newline at end of file From 5a6a1340174deae29c49f323d622d4190ffe895c Mon Sep 17 00:00:00 2001 From: Javier <10879637+javiertuya@users.noreply.github.com> Date: Tue, 8 Oct 2024 18:30:14 +0200 Subject: [PATCH 4/6] ISSUE-680 # Developer review and coverage check --- .../jsmart/zerocode/core/db/DbCsvLoader.java | 13 +++++--- .../jsmart/zerocode/core/db/DbCsvRequest.java | 2 +- .../zerocode/core/db/DbSqlExecutor.java | 21 +++++++----- .../jsmart/zerocode/core/db/DbSqlRequest.java | 3 +- .../jsmart/zerocode/core/db/DbSqlRunner.java | 13 +++++--- .../zerocode/core/db/DbValueConverter.java | 32 +++++++++++-------- .../core/db/DbSqlExecutorScenarioTest.java | 16 ++++++++++ .../jsmart/zerocode/core/db/DbTestBase.java | 4 ++- .../core/db/DbValueConverterTest.java | 31 +++++++++++++++--- core/src/test/resources/db_test.properties | 7 ++-- .../db/db_csv_load_with_headers.json | 8 ++--- .../db/db_csv_load_without_headers.json | 8 ++--- 12 files changed, 110 insertions(+), 48 deletions(-) diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbCsvLoader.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbCsvLoader.java index 72896878..877f4469 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/db/DbCsvLoader.java +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbCsvLoader.java @@ -15,6 +15,9 @@ import com.univocity.parsers.csv.CsvParser; +/** + * Data loading in the database from a CSV external source + */ class DbCsvLoader { private static final Logger LOGGER = LoggerFactory.getLogger(DbCsvLoader.class); private Connection conn; @@ -26,10 +29,10 @@ public DbCsvLoader(Connection conn, CsvParser csvParser) { } /** - * Loads rows in csv format (csvLines) into a table in the database - * and returns the total number of rows + * Loads rows in CSV format (csvLines) into a table in the database + * and returns the total number of rows. */ - int loadCsv(String table, List csvLines, boolean withHeaders, String nullString) throws SQLException { + public int loadCsv(String table, List csvLines, boolean withHeaders, String nullString) throws SQLException { if (csvLines == null || csvLines.isEmpty()) return 0; @@ -87,7 +90,7 @@ private List buildParameters(String table, String[] headers, List getCsvSourceFrom(JsonNode csvSourceJsonNode) { try { diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlExecutor.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlExecutor.java index 373dc0e4..22392d24 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlExecutor.java +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlExecutor.java @@ -25,7 +25,8 @@ public class DbSqlExecutor { public static final String SQL_RESULTS_KEY = "rows"; public static final String CSV_RESULTS_KEY = "size"; - @Inject + // Optional to log the explanatory error message if the env variables are no defined + @Inject(optional = true) @Named("db.driver.url") private String url; @Inject(optional = true) @@ -56,8 +57,9 @@ public Map loadcsv(DbCsvRequest request) { response.put(CSV_RESULTS_KEY, result); return response; } catch (Exception e) { - LOGGER.error("Failed to load CSV", e); - throw new RuntimeException(e); + String message = "Failed to load CSV"; + LOGGER.error(message, e); + throw new RuntimeException(message, e); } finally { closeConnection(conn); } @@ -65,7 +67,7 @@ public Map loadcsv(DbCsvRequest request) { /** * The EXECUTE operation returns the records retrieved by the SQL specified in the request - * under the key "rows" (select) or an empty object (insert, update) + * under the key "rows" (select), or an empty object (insert, update) */ public Map EXECUTE(DbSqlRequest request) { return execute(request); @@ -85,8 +87,9 @@ public Map execute(DbSqlRequest request) { } return response; } catch (SQLException e) { - LOGGER.error("Failed to execute SQL", e); - throw new RuntimeException(e); + String message = "Failed to execute SQL"; + LOGGER.error(message, e); + throw new RuntimeException(message, e); } finally { closeConnection(conn); } @@ -102,8 +105,10 @@ protected Connection createAndGetConnection() { try { return DriverManager.getConnection(url, user, password); } catch (SQLException e) { - LOGGER.error("Failed to create connection", e); - throw new RuntimeException(e); + String message = "Failed to create connection, Please check the target environment properties " + + "to connect the database (db.driver.url, db.driver.user and db.driver.password)"; + LOGGER.error(message, e); + throw new RuntimeException(message, e); } } diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRequest.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRequest.java index a1537bf2..c89b84c1 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRequest.java +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRequest.java @@ -12,7 +12,8 @@ public class DbSqlRequest { private final Object[] sqlParams; @JsonCreator - public DbSqlRequest(@JsonProperty("sqlStatement") String sql, + public DbSqlRequest( + @JsonProperty("sql") String sql, @JsonProperty("sqlParams") Object[] sqlParams) { this.sql = sql; this.sqlParams = sqlParams; diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRunner.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRunner.java index ff8ad488..81c58c5d 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRunner.java +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRunner.java @@ -10,6 +10,9 @@ import java.util.List; import java.util.Map; +/** + * Execution of SQL statements against a database + */ class DbSqlRunner { private static final Logger LOGGER = LoggerFactory.getLogger(DbSqlRunner.class); private Connection conn; @@ -19,14 +22,14 @@ public DbSqlRunner(Connection conn) { } /** - * Execute an sql with parameters (optional) and returns a list of maps + * Executes a SQL statement with parameters (optional) and returns a list of maps * with the ResultSet content (select) or null (insert, update) */ - List> execute(String sql, Object[] params) throws SQLException { - // As there is only one execute operation instead of separate update and query, - // the DbUtils execute method returns a list containing each ResultSet (each is a list of maps): + public List> execute(String sql, Object[] params) throws SQLException { + // There is only one execute operation instead of separate update and query. + // The DbUtils execute method returns a list containing each ResultSet (each is a list of maps): // - Empty (insert and update) - // - With one or more ResultSets (select). + // - With one or more ResultSets (select): use the first one // - Note that some drivers never return more than one ResultSet (e.g. H2) QueryRunner runner = new QueryRunner(); List>> result = runner.execute(conn, sql, new MapListHandler(), params); diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbValueConverter.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbValueConverter.java index 149ae885..36432536 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/db/DbValueConverter.java +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbValueConverter.java @@ -10,12 +10,13 @@ import java.util.LinkedHashMap; import java.util.Map; +import org.apache.commons.lang3.ArrayUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Conversion of string values to be inserted in the database - * into objects compatible with the java.sql type of the target columns. + * into objects compatible with the java sql type of the target columns. */ public class DbValueConverter { private static final Logger LOGGER = LoggerFactory.getLogger(DbSqlExecutor.class); @@ -58,9 +59,9 @@ private void initializeMetadata() throws SQLException { } private String convertToStoredCase(String identifier) throws SQLException { - if (databaseMetaData.storesLowerCaseIdentifiers()) + if (databaseMetaData.storesLowerCaseIdentifiers()) // e.g. Postgres identifier = identifier.toLowerCase(); - else if (databaseMetaData.storesUpperCaseIdentifiers()) + else if (databaseMetaData.storesUpperCaseIdentifiers()) // e.g. H2 identifier = identifier.toUpperCase(); return identifier; } @@ -71,12 +72,12 @@ private void logInitializeError() { } /** - * Given an array of column names and their corresponding values (as strings) + * Given an array of column names and other array with their corresponding values (as strings), * transforms each value to the compatible data type that allow to be inserted in the database. - * If the column names are missing, uses all columns in the current table as fallback. + * If the column names are missing, uses all columns in the current table as the column names. */ Object[] convertColumnValues(String[] columns, String[] values) { - if (columns == null || columns.length == 0) // if no specified, use all columns in the table + if (ArrayUtils.isEmpty(columns)) // if no specified, use all columns in the table columns = columnTypes.keySet().toArray(new String[0]); Object[] converted = new Object[values.length]; @@ -98,8 +99,8 @@ private Object convertColumnValue(String column, String value) { } /** - * Converts the string representation of a data type value into the appropriate simple sql data type. - * If a data type is not handled by this method, returns the input value as fallback. + * Converts the string representation of a data type value into the appropriate simple SQL data type. + * If a data type is not handled by this method (or is string), returns the input string value as fallback. * * See table B-1 in JDBC 4.2 Specification */ @@ -135,21 +136,26 @@ private Object convertColumnValueFromJavaSqlType(int sqlType, String value) thro case java.sql.Types.TIMESTAMP: return new java.sql.Timestamp(parseDate(value, getTimestampFormats()).getTime()); default: return value; + // Not including: binary and advanced datatypes (e.g. blob, array...) } } - // Currently, supported date time formats are a few common ISO-8601 formats - // (other common format strings in org.apache.commons.lang3.time.DateFormatUtils) + // Supported date time formats are the common ISO-8601 formats + // defined in org.apache.commons.lang3.time.DateFormatUtils, + // as well as their variants that specify milliseconds. // This may be made user configurable later, via properties and/or embedded in the payload private String[] getDateFormats() { - return new String[] {"yyyy-MM-dd"}; + return new String[] { "yyyy-MM-dd" }; } + private String[] getTimeFormats() { - return new String[] {"HH:mm:ssZ", "HH:mm:ss.SSSZ"}; + return new String[] { "HH:mm:ssZ", "HH:mm:ss.SSSZ", "HH:mm:ss", "HH:mm:ss.SSS" }; } + private String[] getTimestampFormats() { - return new String[] {"yyyy-MM-dd'T'HH:mm:ssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ"}; + return new String[] { "yyyy-MM-dd'T'HH:mm:ssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ", + "yyyy-MM-dd'T'HH:mm:ss", "yyyy-MM-dd'T'HH:mm:ss.SSS" }; } } diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java index 8a9bcd14..f3893f0d 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java @@ -9,6 +9,9 @@ @RunWith(ZeroCodeUnitRunner.class) public class DbSqlExecutorScenarioTest { + // NOTE: Below tests will fail when run in postgres because this database stores identifiers in lowercase. + // to make tests pass, change the keys in the response.rows to lowercase + @Test @Scenario("integration_test_files/db/db_csv_load_with_headers.json") public void testDbCsvLoadWithHeaders() throws Exception { @@ -24,4 +27,17 @@ public void testDbCsvLoadWithoutHeaders() throws Exception { public void testDbSqlExecute() throws Exception { } + // Manual test: error handling. + // To facilitate the location of the source of possible errors (e.g. a wrong SQL statement), + // exceptions that occur in the DbSqlExecutor should show: + // - A log entry with the error message + // - The stacktrace of the exception to facilitate locating the source + // - The usual chain of errors and stacktraces produced by zerocode when an step fails + // + // Recommended situations for manual test: + // - Target environment variables are no defined + // - A syntactically wrong SQL statement in a step + // - A header that does not correspond with any column when loading data from CSV + // - A value with the wrong data type (e.g. string in a numeric column) when loading data from CSV + } diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbTestBase.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbTestBase.java index 432cf7bd..86fe5a77 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/db/DbTestBase.java +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbTestBase.java @@ -39,7 +39,7 @@ protected void configureTest() { @Named("db.driver.password") protected String password; protected Connection conn; // managed connection for each test - protected boolean isPostgres = false; // set by each connection, to allow portable assertions (postgres is lowercase) + protected boolean isPostgres = false; // set by each connection, to allow portable assertions @Before public void setUp() throws SQLException { @@ -61,6 +61,8 @@ protected List> execute(String sql, Object[] params) throws return runner.execute(sql, params); } + // Table and columns in all tests are uppercase because H2 stores uppercase by default. + // But postgres stores lowercase, so some expected strings need case conversion protected String convertDbCase(String value) { return isPostgres ? value.toLowerCase() : value; } diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbValueConverterTest.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbValueConverterTest.java index 796740fe..46ebe82a 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/db/DbValueConverterTest.java +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbValueConverterTest.java @@ -57,8 +57,9 @@ public void convertDateAndTimeValues() throws SQLException { } @Test - public void convertWithLowerAndUppercase() throws SQLException { - // Test date types to ensure that is the converter who makes the conversion, not the driver + public void convertWithMixedCaseColumnName() throws SQLException { + // Uses a date type to ensure that is the converter who tries making the conversion, not the driver + // (neither H2 nor Postgres drivers do conversion of dates) List> rows = doTestConversion("", "ITable", "VDATE DATE", new String[] { "VDate" }, new String[] { "2024-09-04" }, @@ -74,6 +75,16 @@ public void whenNoColumnsSpecified_ThenAllTableColumns_AreIncluded() throws SQLE "[{VINT=101, VCHAR=astring}]"); } + @Test + public void whenNoColumnsSpecified_AndColumnCountMismatch_ThenConversionFails() throws SQLException { + assertThrows(SQLException.class, () -> { + doTestConversion("", "FMTABLE", "VINT BIGINT", + null, + new String[] { "101", "999" }, + "[{VINT=101}]"); + }); + } + @Test public void whenColumnNotFound_ThenConversionFails() throws SQLException { assertThrows(SQLException.class, () -> { @@ -84,8 +95,20 @@ public void whenColumnNotFound_ThenConversionFails() throws SQLException { }); } - // Failures due to problems with metadata: - // - These tests will pass because the H2 driver converts numeric values + @Test + public void whenValueHasWrongFormat_ThenConversionFails() throws SQLException { + assertThrows(SQLException.class, () -> { + doTestConversion("", "FVTABLE", "VTS TIMESTAMP", + new String[] { "VTS" }, + new String[] { "notadate" }, + "[{VTS=notadate}]"); + }); + } + + // Failures due to problems with metadata. + // Simulates failures getting metadata so that the conversion is left to + // be done by the driver. + // - Below tests will pass because the H2 driver converts numeric values // - but fail if driver is changed to Postgres (does not convert numeric), skipped @Test diff --git a/core/src/test/resources/db_test.properties b/core/src/test/resources/db_test.properties index 0f741ab7..2df82c79 100644 --- a/core/src/test/resources/db_test.properties +++ b/core/src/test/resources/db_test.properties @@ -7,9 +7,12 @@ db.driver.url=jdbc:h2:./target/test_db_sql_executor # db.driver.password= # To run the tests with postgres: -# - run container: docker run --name zerocode-postgres -p:5432:5432 -e POSTGRES_PASSWORD=mypassword -d postgres +# - Spin up the DB container (this was tested with the below versions #680, latest version is the recommended) +# docker run --name zerocode-postgres -p:5432:5432 -e POSTGRES_PASSWORD=mypassword -d postgres:17.0 +# docker run --name zerocode-postgres -p:5432:5432 -e POSTGRES_PASSWORD=mypassword -d postgres:12.0 +# docker run --name zerocode-postgres -p:5432:5432 -e POSTGRES_PASSWORD=mypassword -d postgres:9.0 # - add the driver dependency to the pom.xml: https://central.sonatype.com/artifact/org.postgresql/postgresql -# - and uncomment these properties +# - and uncomment the properties below: # db.driver.url=jdbc:postgresql://localhost:5432/postgres # db.driver.user=postgres # db.driver.password=mypassword diff --git a/core/src/test/resources/integration_test_files/db/db_csv_load_with_headers.json b/core/src/test/resources/integration_test_files/db/db_csv_load_with_headers.json index 0519377d..f2f0f258 100644 --- a/core/src/test/resources/integration_test_files/db/db_csv_load_with_headers.json +++ b/core/src/test/resources/integration_test_files/db/db_csv_load_with_headers.json @@ -18,10 +18,10 @@ "tableName": "players", "csvSource": "integration_test_files/db/players_with_headers.csv", "withHeaders" : true - }, + }, "verify": { - "size" : 3 - } + "size" : 3 + } }, { "name": "Check the content of inserted rows", @@ -31,7 +31,7 @@ "sql": "SELECT ID, NAME, AGE FROM PLAYERS ORDER BY ID" }, "verify": { - "rows.SIZE": 3, + "rows.SIZE": 3, "rows": [ //<-- to make this pass in postgres, set the keys to lowercase { "ID": 1001, "NAME": "Ronaldo", "AGE": 23 }, { "ID": 1002, "NAME": "Devaldo", "AGE": null }, diff --git a/core/src/test/resources/integration_test_files/db/db_csv_load_without_headers.json b/core/src/test/resources/integration_test_files/db/db_csv_load_without_headers.json index f8f093d4..16212042 100644 --- a/core/src/test/resources/integration_test_files/db/db_csv_load_without_headers.json +++ b/core/src/test/resources/integration_test_files/db/db_csv_load_without_headers.json @@ -18,10 +18,10 @@ "tableName": "players", "csvSource": "integration_test_files/db/players_without_headers.csv", "nullString": "NULL!!" - }, + }, "verify": { - "size" : 3 - } + "size" : 3 + } }, { "name": "Check the content of inserted rows", @@ -31,7 +31,7 @@ "sql": "SELECT ID, NAME, AGE FROM PLAYERS ORDER BY ID" }, "verify": { - "rows.SIZE": 3, + "rows.SIZE": 3, "rows": [ //<-- to make this pass in postgres, set the keys to lowercase { "ID": 1001, "NAME": "Ronaldo", "AGE": 23 }, { "ID": 1002, "NAME": null, "AGE": 24 }, From 3ca2575ddac828dd36fbf88fb930d2d53955394b Mon Sep 17 00:00:00 2001 From: Javier <10879637+javiertuya@users.noreply.github.com> Date: Sat, 23 Nov 2024 07:52:29 +0100 Subject: [PATCH 5/6] ISSUE-680 # Run Postgres tests in CI --- .github/workflows/main.yml | 3 ++ core/pom.xml | 3 -- .../db/DbSqlExecutorScenarioPostgresTest.java | 24 +++++++++ .../core/db/DbSqlExecutorScenarioTest.java | 3 -- core/src/test/resources/db_test.properties | 11 ----- .../resources/db_test_postgres.properties | 7 +++ .../db/db_csv_load_with_headers_postgres.json | 43 ++++++++++++++++ .../db/db_sql_execute_postgres.json | 49 +++++++++++++++++++ pom.xml | 7 +++ 9 files changed, 133 insertions(+), 17 deletions(-) create mode 100644 core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioPostgresTest.java create mode 100644 core/src/test/resources/db_test_postgres.properties create mode 100644 core/src/test/resources/integration_test_files/db/db_csv_load_with_headers_postgres.json create mode 100644 core/src/test/resources/integration_test_files/db/db_sql_execute_postgres.json diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5289f034..e8e82f35 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,5 +28,8 @@ jobs: - name: Running Kafka run: docker-compose -f docker/compose/kafka-schema-registry.yml up -d && sleep 10 + - name: Running PostgreSQL (to test DB SQL Executor) + run: docker-compose -f docker/compose/pg_compose.yml up -d + - name: Building and testing the changes run: mvn clean test diff --git a/core/pom.xml b/core/pom.xml index 8e5593e5..b20e6aad 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -184,14 +184,11 @@ h2 test - com.aventstack extentreports diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioPostgresTest.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioPostgresTest.java new file mode 100644 index 00000000..7dd0c31a --- /dev/null +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioPostgresTest.java @@ -0,0 +1,24 @@ +package org.jsmart.zerocode.core.db; +import org.jsmart.zerocode.core.domain.Scenario; +import org.jsmart.zerocode.core.domain.TargetEnv; +import org.jsmart.zerocode.core.runner.ZeroCodeUnitRunner; +import org.junit.Test; +import org.junit.runner.RunWith; + +@TargetEnv("db_test_postgres.properties") +@RunWith(ZeroCodeUnitRunner.class) +public class DbSqlExecutorScenarioPostgresTest { + + // Note: Spin up the DB container before running this test: docker/compose/pg_compose.yml + + @Test + @Scenario("integration_test_files/db/db_csv_load_with_headers_postgres.json") + public void testDbCsvLoadWithHeaders() throws Exception { + } + + @Test + @Scenario("integration_test_files/db/db_sql_execute_postgres.json") + public void testDbSqlExecute() throws Exception { + } + +} diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java index f3893f0d..e294777c 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java @@ -9,9 +9,6 @@ @RunWith(ZeroCodeUnitRunner.class) public class DbSqlExecutorScenarioTest { - // NOTE: Below tests will fail when run in postgres because this database stores identifiers in lowercase. - // to make tests pass, change the keys in the response.rows to lowercase - @Test @Scenario("integration_test_files/db/db_csv_load_with_headers.json") public void testDbCsvLoadWithHeaders() throws Exception { diff --git a/core/src/test/resources/db_test.properties b/core/src/test/resources/db_test.properties index 2df82c79..fe99f714 100644 --- a/core/src/test/resources/db_test.properties +++ b/core/src/test/resources/db_test.properties @@ -5,14 +5,3 @@ db.driver.url=jdbc:h2:./target/test_db_sql_executor # If connection requires authentication, specify user and password: # db.driver.user= # db.driver.password= - -# To run the tests with postgres: -# - Spin up the DB container (this was tested with the below versions #680, latest version is the recommended) -# docker run --name zerocode-postgres -p:5432:5432 -e POSTGRES_PASSWORD=mypassword -d postgres:17.0 -# docker run --name zerocode-postgres -p:5432:5432 -e POSTGRES_PASSWORD=mypassword -d postgres:12.0 -# docker run --name zerocode-postgres -p:5432:5432 -e POSTGRES_PASSWORD=mypassword -d postgres:9.0 -# - add the driver dependency to the pom.xml: https://central.sonatype.com/artifact/org.postgresql/postgresql -# - and uncomment the properties below: -# db.driver.url=jdbc:postgresql://localhost:5432/postgres -# db.driver.user=postgres -# db.driver.password=mypassword diff --git a/core/src/test/resources/db_test_postgres.properties b/core/src/test/resources/db_test_postgres.properties new file mode 100644 index 00000000..ed699717 --- /dev/null +++ b/core/src/test/resources/db_test_postgres.properties @@ -0,0 +1,7 @@ +# Connection info used by the DbSqlExecutor + +# JDBC connection string to a PostgreSQL test database +# Spin up the DB container before running the postgres tests: docker/compose/pg_compose.yml +db.driver.url=jdbc:postgresql://localhost:35432/postgres +db.driver.user=postgres +db.driver.password=example diff --git a/core/src/test/resources/integration_test_files/db/db_csv_load_with_headers_postgres.json b/core/src/test/resources/integration_test_files/db/db_csv_load_with_headers_postgres.json new file mode 100644 index 00000000..544e32c0 --- /dev/null +++ b/core/src/test/resources/integration_test_files/db/db_csv_load_with_headers_postgres.json @@ -0,0 +1,43 @@ +{ + "scenarioName": "DbSqlExecutor: Load a CSV file with headers - PostgreSQL", + "steps": [ + { + "name": "Test database setup", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "DROP TABLE IF EXISTS PLAYERS; CREATE TABLE PLAYERS (ID INTEGER, NAME VARCHAR(20), AGE INTEGER);" + }, + "verify": { } + }, + { + "name": "Insert rows from a CSV file with headers", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "LOADCSV", + "request": { + "tableName": "players", + "csvSource": "integration_test_files/db/players_with_headers.csv", + "withHeaders" : true + }, + "verify": { + "size" : 3 + } + }, + { + "name": "Check the content of inserted rows", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "SELECT ID, NAME, AGE FROM PLAYERS ORDER BY ID" + }, + "verify": { + "rows.SIZE": 3, + "rows": [ //<-- same than db_csv_load_with_headers.json, but keys in lowercase (postgres converts to lower) + { "id": 1001, "name": "Ronaldo", "age": 23 }, + { "id": 1002, "name": "Devaldo", "age": null }, + { "id": 1003, "name": "Trevaldo", "age": 35 } + ] + } + } + ] +} \ No newline at end of file diff --git a/core/src/test/resources/integration_test_files/db/db_sql_execute_postgres.json b/core/src/test/resources/integration_test_files/db/db_sql_execute_postgres.json new file mode 100644 index 00000000..11d283ef --- /dev/null +++ b/core/src/test/resources/integration_test_files/db/db_sql_execute_postgres.json @@ -0,0 +1,49 @@ +{ + "scenarioName": "DbSqlExecutor: Read and write data using SQL - PostgreSQL", + "steps": [ + { + "name": "Test database setup", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "DROP TABLE IF EXISTS PEOPLE; CREATE TABLE PEOPLE (ID INTEGER, NAME VARCHAR(20), START DATE, ACTIVE BOOLEAN);" + }, + "verify": { } + }, + { + "name": "Insert rows using SQL", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "INSERT INTO PEOPLE VALUES (1, 'Jeff Bejo', '2024-09-01', true); INSERT INTO PEOPLE VALUES (2, 'John Bajo', '2024-09-02', false);" + }, + "verify": { } + }, + { + "name": "Insert with parameters and nulls", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "execute", //<-- Uppercase for consistency, but also allows lowercase + "request": { + "sql": "INSERT INTO PEOPLE (ID, NAME, START, ACTIVE) VALUES (?, ?, ?, ?);", + "sqlParams": [3, null, null, true] + }, + "verify": { } + }, + { + "name": "Retrieve rows using SQL", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "SELECT ID, NAME, to_char(START,'yyyy-MM-dd') AS START, ACTIVE FROM PEOPLE WHERE ACTIVE=?", + "sqlParams": [true] + }, + "verify": { + "rows.SIZE": 2, + "rows": [ //<-- same than db_sql_execute.json, but keys in lowercase (postgres converts to lower) + { "id": 1, "name": "Jeff Bejo", "start": "2024-09-01", "active": true }, + { "id": 3, "name": null, "start": null, "active": true } + ] + } + } + ] +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index b34a4d4a..ad9f4b87 100644 --- a/pom.xml +++ b/pom.xml @@ -89,6 +89,7 @@ 4.5.13 4.5.12 2.2.220 + 42.7.4 5.0.9 3.7.0 2.10.1 @@ -299,6 +300,12 @@ ${h2.db.version} test + + org.postgresql + postgresql + ${postgres.db.version} + test + com.aventstack extentreports From ce8f2aaa751e255c9c893ea35b8d33da0c7bf8de Mon Sep 17 00:00:00 2001 From: Author Japps Date: Sun, 22 Dec 2024 12:57:32 +0000 Subject: [PATCH 6/6] PG Driver in compile scope. Also removed unused adminer from compose --- core/pom.xml | 2 +- docker/compose/pg_compose.yml | 6 ------ pom.xml | 1 - 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index b20e6aad..7fb25c0b 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -187,7 +187,7 @@ org.postgresql postgresql - test + com.aventstack diff --git a/docker/compose/pg_compose.yml b/docker/compose/pg_compose.yml index 100398f6..3ad83a2e 100644 --- a/docker/compose/pg_compose.yml +++ b/docker/compose/pg_compose.yml @@ -12,9 +12,3 @@ services: POSTGRES_PASSWORD: example ports: - 35432:5432 - - adminer: - image: adminer:4.7.0 - restart: always - ports: - - 8080:8080 \ No newline at end of file diff --git a/pom.xml b/pom.xml index ad9f4b87..9d354afa 100644 --- a/pom.xml +++ b/pom.xml @@ -304,7 +304,6 @@ org.postgresql postgresql ${postgres.db.version} - test com.aventstack