diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5289f034..e8e82f35 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,5 +28,8 @@ jobs: - name: Running Kafka run: docker-compose -f docker/compose/kafka-schema-registry.yml up -d && sleep 10 + - name: Running PostgreSQL (to test DB SQL Executor) + run: docker-compose -f docker/compose/pg_compose.yml up -d + - name: Building and testing the changes run: mvn clean test diff --git a/core/pom.xml b/core/pom.xml index f59e00aa..7fb25c0b 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -175,11 +175,20 @@ micro-simulator test + + commons-dbutils + commons-dbutils + com.h2database h2 test + + org.postgresql + postgresql + + com.aventstack extentreports diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbCsvLoader.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbCsvLoader.java new file mode 100644 index 00000000..877f4469 --- /dev/null +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbCsvLoader.java @@ -0,0 +1,135 @@ +package org.jsmart.zerocode.core.db; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import org.apache.commons.dbutils.QueryRunner; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.univocity.parsers.csv.CsvParser; + +/** + * Data loading in the database from a CSV external source + */ +class DbCsvLoader { + private static final Logger LOGGER = LoggerFactory.getLogger(DbCsvLoader.class); + private Connection conn; + private CsvParser csvParser; + + public DbCsvLoader(Connection conn, CsvParser csvParser) { + this.conn = conn; + this.csvParser = csvParser; + } + + /** + * Loads rows in CSV format (csvLines) into a table in the database + * and returns the total number of rows. + */ + public int loadCsv(String table, List csvLines, boolean withHeaders, String nullString) throws SQLException { + if (csvLines == null || csvLines.isEmpty()) + return 0; + + List lines = parseLines(table, csvLines); + + String[] headers = buildHeaders(lines.get(0), withHeaders); + List paramset = buildParameters(table, headers, lines, withHeaders, nullString); + if (paramset.isEmpty()) // can have headers, but no rows + return 0; + + String sql = buildSql(table, headers, paramset.get(0).length); + LOGGER.info("Loading CSV using this sql: {}", sql); + + QueryRunner runner = new QueryRunner(); + int insertCount = 0; + for (int i = 0 ; i < paramset.size(); i++) { + insertRow(runner, i, sql, paramset.get(i)); + insertCount++; + } + LOGGER.info("Total of rows inserted: {}", insertCount); + return insertCount; + } + + private List parseLines(String table, List lines) { + int numCol = 0; // will check that every row has same columns than the first + List parsedLines = new ArrayList<>(); + for (int i = 0; i buildParameters(String table, String[] headers, List lines, boolean withHeaders, String nullString) { + DbValueConverter converter = new DbValueConverter(conn, table); + List paramset = new ArrayList<>(); + for (int i = withHeaders ? 1 : 0; i < lines.size(); i++) { + String[] parsedLine = lines.get(i); + parsedLine = processNulls(parsedLine, nullString); + Object[] params; + try { + params = converter.convertColumnValues(headers, parsedLine); + LOGGER.info(" row [{}] params: {}", i + 1, Arrays.asList(params).toString()); + } catch (Exception e) { // Not only SQLException as converter also does parsing + String message = String.format("Error matching data type of parameters and table columns at CSV row %d", i + 1); + LOGGER.error(message); + LOGGER.error("Exception message: {}", e.getMessage()); + throw new RuntimeException(message, e); + } + paramset.add(params); + } + return paramset; + } + + private String[] processNulls(String[] line, String nullString) { + for (int i = 0; i < line.length; i++) { + if (StringUtils.isBlank(nullString) && StringUtils.isBlank(line[i])) { + line[i] = null; + } else if (!StringUtils.isBlank(nullString)) { + if (StringUtils.isBlank(line[i])) // null must be empty string + line[i] = ""; + else if (nullString.trim().equalsIgnoreCase(line[i].trim())) + line[i] = null; + } + } + return line; + } + + private String buildSql(String table, String[] headers, int columnCount) { + String placeholders = IntStream.range(0, columnCount) + .mapToObj(i -> "?").collect(Collectors.joining(",")); + return "INSERT INTO " + table + + (headers.length > 0 ? " (" + String.join(",", headers) + ")" : "") + + " VALUES (" + placeholders + ");"; + } + + private void insertRow(QueryRunner runner, int rowId, String sql, Object[] params) { + try { + runner.update(conn, sql, params); + } catch (SQLException e) { + String message = String.format("Error inserting data at CSV row %d", rowId + 1); + LOGGER.error(message); + LOGGER.error("Exception message: {}", e.getMessage()); + throw new RuntimeException(message, e); + } + } + +} \ No newline at end of file diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbCsvRequest.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbCsvRequest.java new file mode 100644 index 00000000..cebffe1c --- /dev/null +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbCsvRequest.java @@ -0,0 +1,101 @@ +package org.jsmart.zerocode.core.db; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import org.apache.commons.lang3.StringUtils; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +public class DbCsvRequest { + private final String tableName; + private final List csvSource; + private final Boolean withHeaders; + private final String nullString; + + public DbCsvRequest( + @JsonProperty(value="tableName", required=true) String tableName, + @JsonProperty("csvSource") JsonNode csvSourceJsonNode, + @JsonProperty("withHeaders") Boolean withHeaders, + @JsonProperty("nullString") String nullString) { + this.tableName = tableName; + this.withHeaders = Optional.ofNullable(withHeaders).orElse(false); + this.nullString = Optional.ofNullable(nullString).orElse(""); + this.csvSource = Optional.ofNullable(csvSourceJsonNode).map(this::getCsvSourceFrom).orElse(Collections.emptyList()); + } + + public String getTableName() { + return tableName; + } + + public List getCsvSource() { + return csvSource; + } + + public boolean getWithHeaders() { + return withHeaders; + } + + public String getNullString() { + return nullString; + } + + // Code below is duplicated from org.jsmart.zerocode.core.domain.Parametrized.java and not included in tests. + // TODO Consider some refactoring later and review error message when file not found + + private List getCsvSourceFrom(JsonNode csvSourceJsonNode) { + try { + if (csvSourceJsonNode.isArray()) { + return readCsvSourceFromJson(csvSourceJsonNode); + + } else { + return readCsvSourceFromExternalCsvFile(csvSourceJsonNode); + } + } catch (IOException e) { + throw new RuntimeException("Error deserializing csvSource", e); + } + } + + private List readCsvSourceFromJson(JsonNode csvSourceJsonNode) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + ObjectReader reader = mapper.readerFor(new TypeReference>() { + }); + return reader.readValue(csvSourceJsonNode); + } + + private List readCsvSourceFromExternalCsvFile(JsonNode csvSourceJsonNode) throws IOException { + String csvSourceFilePath = csvSourceJsonNode.textValue(); + if (StringUtils.isNotBlank(csvSourceFilePath)) { + Path path = Paths.get("./src/test/resources/",csvSourceFilePath); + List csvSourceFileLines = Files.lines(path) + .filter(StringUtils::isNotBlank) + .collect(Collectors.toList()); + //if (this.ignoreHeader) { + // return csvSourceFileLines.stream() + // .skip(1) + // .collect(Collectors.toList()); + //} + return csvSourceFileLines; + } + return Collections.emptyList(); + } + + @Override + public String toString() { + return "Parameterized{" + + "tableName=" + tableName + + ", csvSource=" + csvSource + + ", withHeaders=" + withHeaders + + ", nullString=" + nullString + + '}'; + } +} diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlExecutor.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlExecutor.java new file mode 100644 index 00000000..22392d24 --- /dev/null +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlExecutor.java @@ -0,0 +1,119 @@ +package org.jsmart.zerocode.core.db; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import com.google.inject.name.Named; +import com.univocity.parsers.csv.CsvParser; + +import org.apache.commons.dbutils.DbUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Interaction with a database using SQL to read/write + * Requires the appropriated connection data in the target environment + * properties, see src/test/resources/db_test.properties + */ +public class DbSqlExecutor { + private static final Logger LOGGER = LoggerFactory.getLogger(DbSqlExecutor.class); + public static final String SQL_RESULTS_KEY = "rows"; + public static final String CSV_RESULTS_KEY = "size"; + + // Optional to log the explanatory error message if the env variables are no defined + @Inject(optional = true) + @Named("db.driver.url") private String url; + + @Inject(optional = true) + @Named("db.driver.user") private String user; + + @Inject(optional = true) + @Named("db.driver.password") private String password; + + @Inject + private CsvParser csvParser; + + /** + * The LOADCSV operation inserts the content of a CSV file into a table, + * and returns the number of records inserted under the key "size" + */ + public Map LOADCSV(DbCsvRequest request) { // uppercase for consistency with http api operations + return loadcsv(request); + } + + public Map loadcsv(DbCsvRequest request) { + Connection conn = createAndGetConnection(); + try { + LOGGER.info("Load CSV, request -> {} ", request); + DbCsvLoader runner = new DbCsvLoader(conn, csvParser); + long result = runner.loadCsv(request.getTableName(), request.getCsvSource(), + request.getWithHeaders(), request.getNullString()); + Map response = new HashMap<>(); + response.put(CSV_RESULTS_KEY, result); + return response; + } catch (Exception e) { + String message = "Failed to load CSV"; + LOGGER.error(message, e); + throw new RuntimeException(message, e); + } finally { + closeConnection(conn); + } + } + + /** + * The EXECUTE operation returns the records retrieved by the SQL specified in the request + * under the key "rows" (select), or an empty object (insert, update) + */ + public Map EXECUTE(DbSqlRequest request) { + return execute(request); + } + + public Map execute(DbSqlRequest request) { + Connection conn = createAndGetConnection(); + try { + LOGGER.info("Execute SQL, request -> {} ", request); + DbSqlRunner runner = new DbSqlRunner(conn); + List> results = runner.execute(request.getSql(), request.getSqlParams()); + Map response = new HashMap<>(); + if (results == null) { // will return empty node, use "verify":{} + response.put(SQL_RESULTS_KEY, new ObjectMapper().createObjectNode()); + } else { + response.put(SQL_RESULTS_KEY, results); + } + return response; + } catch (SQLException e) { + String message = "Failed to execute SQL"; + LOGGER.error(message, e); + throw new RuntimeException(message, e); + } finally { + closeConnection(conn); + } + } + + /** + * Returns a new JDBC connection using DriverManager. + * Override this method in case you get the connections using another approach + * (e.g. DataSource) + */ + protected Connection createAndGetConnection() { + LOGGER.info("Create and get connection, url: {}, user: {}", url, user); + try { + return DriverManager.getConnection(url, user, password); + } catch (SQLException e) { + String message = "Failed to create connection, Please check the target environment properties " + + "to connect the database (db.driver.url, db.driver.user and db.driver.password)"; + LOGGER.error(message, e); + throw new RuntimeException(message, e); + } + } + + protected void closeConnection(Connection conn) { + DbUtils.closeQuietly(conn); + } + +} \ No newline at end of file diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRequest.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRequest.java new file mode 100644 index 00000000..c89b84c1 --- /dev/null +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRequest.java @@ -0,0 +1,37 @@ +package org.jsmart.zerocode.core.db; + +import java.util.Arrays; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class DbSqlRequest { + private final String sql; + private final Object[] sqlParams; + + @JsonCreator + public DbSqlRequest( + @JsonProperty("sql") String sql, + @JsonProperty("sqlParams") Object[] sqlParams) { + this.sql = sql; + this.sqlParams = sqlParams; + } + + public String getSql() { + return sql; + } + + public Object[] getSqlParams() { + return sqlParams; + } + + @Override + public String toString() { + return "Request{" + + "sql=" + sql + + ", sqlParams=" + (sqlParams == null ? "[]" : Arrays.asList(sqlParams).toString()) + + '}'; + } +} diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRunner.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRunner.java new file mode 100644 index 00000000..81c58c5d --- /dev/null +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbSqlRunner.java @@ -0,0 +1,45 @@ +package org.jsmart.zerocode.core.db; + +import org.apache.commons.dbutils.QueryRunner; +import org.apache.commons.dbutils.handlers.MapListHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; + +/** + * Execution of SQL statements against a database + */ +class DbSqlRunner { + private static final Logger LOGGER = LoggerFactory.getLogger(DbSqlRunner.class); + private Connection conn; + + public DbSqlRunner(Connection conn) { + this.conn = conn; + } + + /** + * Executes a SQL statement with parameters (optional) and returns a list of maps + * with the ResultSet content (select) or null (insert, update) + */ + public List> execute(String sql, Object[] params) throws SQLException { + // There is only one execute operation instead of separate update and query. + // The DbUtils execute method returns a list containing each ResultSet (each is a list of maps): + // - Empty (insert and update) + // - With one or more ResultSets (select): use the first one + // - Note that some drivers never return more than one ResultSet (e.g. H2) + QueryRunner runner = new QueryRunner(); + List>> result = runner.execute(conn, sql, new MapListHandler(), params); + if (result.isEmpty()) { + return null; + } else { + if (result.size() > 1) + LOGGER.warn("The SQL query returned more than one ResultSet, keeping only the first one"); + return result.get(0); + } + } + +} \ No newline at end of file diff --git a/core/src/main/java/org/jsmart/zerocode/core/db/DbValueConverter.java b/core/src/main/java/org/jsmart/zerocode/core/db/DbValueConverter.java new file mode 100644 index 00000000..36432536 --- /dev/null +++ b/core/src/main/java/org/jsmart/zerocode/core/db/DbValueConverter.java @@ -0,0 +1,161 @@ +package org.jsmart.zerocode.core.db; + +import static org.apache.commons.lang3.time.DateUtils.parseDate; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.text.ParseException; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.commons.lang3.ArrayUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Conversion of string values to be inserted in the database + * into objects compatible with the java sql type of the target columns. + */ +public class DbValueConverter { + private static final Logger LOGGER = LoggerFactory.getLogger(DbSqlExecutor.class); + + private Connection conn; + private String table; + private DatabaseMetaData databaseMetaData; + public Map columnTypes; // java.sql.Types + + public DbValueConverter(Connection conn, String table) { + this.conn = conn; + this.table = table; + try { + initializeMetadata(); + } catch (Exception e) { + logInitializeError(); + } + } + + private void initializeMetadata() throws SQLException { + LOGGER.info("Metadata initialization for table: {}", table); + columnTypes = new LinkedHashMap<>(); // must keep column order + databaseMetaData = conn.getMetaData(); + + table = convertToStoredCase(table); // to locate table name in metadata + LOGGER.info("Database storesLowerCaseIdentifiers={}, storesUpperCaseIdentifiers={}", + databaseMetaData.storesLowerCaseIdentifiers(), databaseMetaData.storesUpperCaseIdentifiers()); + + try (ResultSet rs = databaseMetaData.getColumns(null, null, table, "%")) { + while (rs.next()) { + String storedName = rs.getString("COLUMN_NAME"); + int typeValue = rs.getInt("DATA_TYPE"); + // internally, key is lowercase to allow case insensitive lookups + columnTypes.put(storedName.toLowerCase(), typeValue); + } + } + LOGGER.info("Mapping from java columns to sql types: {}", columnTypes.toString()); + if (columnTypes.isEmpty()) + logInitializeError(); + } + + private String convertToStoredCase(String identifier) throws SQLException { + if (databaseMetaData.storesLowerCaseIdentifiers()) // e.g. Postgres + identifier = identifier.toLowerCase(); + else if (databaseMetaData.storesUpperCaseIdentifiers()) // e.g. H2 + identifier = identifier.toUpperCase(); + return identifier; + } + + private void logInitializeError() { + LOGGER.error("Initialization of metadata for table {} failed. " + + "Errors may appear when matching query parameters to their data types", table); + } + + /** + * Given an array of column names and other array with their corresponding values (as strings), + * transforms each value to the compatible data type that allow to be inserted in the database. + * If the column names are missing, uses all columns in the current table as the column names. + */ + Object[] convertColumnValues(String[] columns, String[] values) { + if (ArrayUtils.isEmpty(columns)) // if no specified, use all columns in the table + columns = columnTypes.keySet().toArray(new String[0]); + + Object[] converted = new Object[values.length]; + for (int i = 0; i < values.length; i++) { + converted[i] = i < columns.length && i < values.length + ? convertColumnValue(columns[i], values[i]) + : values[i]; + } + return converted; + } + + private Object convertColumnValue(String column, String value) { + try { + return convertColumnValueWithThrow(column, value); + } catch (ParseException e) { + LOGGER.error("Can't convert the data type of value {} at column {}", value, column); + return value; + } + } + + /** + * Converts the string representation of a data type value into the appropriate simple SQL data type. + * If a data type is not handled by this method (or is string), returns the input string value as fallback. + * + * See table B-1 in JDBC 4.2 Specification + */ + private Object convertColumnValueWithThrow(String column, String value) throws ParseException { + if (value == null) + return null; + if (!columnTypes.containsKey(column.toLowerCase())) // fallback if no metadata + return value; + + int sqlType = columnTypes.get(column.toLowerCase()); + return convertColumnValueFromJavaSqlType(sqlType, value); + } + + private Object convertColumnValueFromJavaSqlType(int sqlType, String value) throws ParseException { + switch (sqlType) { + case java.sql.Types.NUMERIC: + case java.sql.Types.DECIMAL: return java.math.BigDecimal.valueOf(Double.parseDouble(value)); + + case java.sql.Types.BIT: //accepts "1" as true (e.g. SqlServer) + case java.sql.Types.BOOLEAN: return Boolean.valueOf("1".equals(value) ? "true" : value); + + case java.sql.Types.TINYINT: return Byte.valueOf(value); + case java.sql.Types.SMALLINT: return Short.valueOf(value); + case java.sql.Types.INTEGER: return Integer.valueOf(value); + case java.sql.Types.BIGINT: return Long.valueOf(value); + + case java.sql.Types.REAL: return Float.valueOf(value); + case java.sql.Types.FLOAT: return Double.valueOf(value); + case java.sql.Types.DOUBLE: return Double.valueOf(value); + + case java.sql.Types.DATE: return new java.sql.Date(parseDate(value, getDateFormats()).getTime()); + case java.sql.Types.TIME: return new java.sql.Time(parseDate(value, getTimeFormats()).getTime()); + case java.sql.Types.TIMESTAMP: return new java.sql.Timestamp(parseDate(value, getTimestampFormats()).getTime()); + default: + return value; + // Not including: binary and advanced datatypes (e.g. blob, array...) + } + } + + // Supported date time formats are the common ISO-8601 formats + // defined in org.apache.commons.lang3.time.DateFormatUtils, + // as well as their variants that specify milliseconds. + // This may be made user configurable later, via properties and/or embedded in the payload + + private String[] getDateFormats() { + return new String[] { "yyyy-MM-dd" }; + } + + private String[] getTimeFormats() { + return new String[] { "HH:mm:ssZ", "HH:mm:ss.SSSZ", "HH:mm:ss", "HH:mm:ss.SSS" }; + } + + private String[] getTimestampFormats() { + return new String[] { "yyyy-MM-dd'T'HH:mm:ssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ", + "yyyy-MM-dd'T'HH:mm:ss", "yyyy-MM-dd'T'HH:mm:ss.SSS" }; + } + +} diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbCsvLoaderTest.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbCsvLoaderTest.java new file mode 100644 index 00000000..80729ad3 --- /dev/null +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbCsvLoaderTest.java @@ -0,0 +1,120 @@ +package org.jsmart.zerocode.core.db; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertThrows; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import org.jukito.JukitoRunner; +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.google.inject.Inject; +import com.univocity.parsers.csv.CsvParser; + +@RunWith(JukitoRunner.class) +public class DbCsvLoaderTest extends DbTestBase{ + + private DbCsvLoader loader; + + @Inject + CsvParser csvParser; + + @Override + public void setUp() throws SQLException { + super.setUp(); + loader = new DbCsvLoader(conn, csvParser); + execute("DROP TABLE IF EXISTS CSVTABLE; " + + "CREATE TABLE CSVTABLE (ID INTEGER, NAME VARCHAR(20), STATUS BOOLEAN)", null); + } + + private int loadCsv(String table, String[] lines, boolean withHeaders, String nullString) throws SQLException { + List linesList = Arrays.asList(lines); + return loader.loadCsv(table, linesList, withHeaders, nullString); + } + + private void assertLoaded(int count, String expected) throws SQLException { + List> rows = execute("SELECT ID,NAME,STATUS FROM CSVTABLE ORDER BY ID NULLS FIRST", null); + assertThat(rows.toString(), equalTo(convertDbCase(expected))); + assertThat(rows.size(), equalTo(count)); + } + + @Test + public void testLoadSimpleCsvWithoutHeaders() throws SQLException { + int count = loadCsv("CSVTABLE", new String[] { "101,me,false", "102,you,true" }, false, ""); + assertLoaded(count, "[{ID=101, NAME=me, STATUS=false}, {ID=102, NAME=you, STATUS=true}]"); + } + + @Test + public void testLoadSimpleCsvWithHeaders() throws SQLException { + int count = loadCsv("CSVTABLE", new String[] { "ID,NAME,STATUS", "101,me,false", "102,you,true" }, true, ""); + assertLoaded(count, "[{ID=101, NAME=me, STATUS=false}, {ID=102, NAME=you, STATUS=true}]"); + } + + @Test + public void testLoadCsvIsNotCleanInsert() throws SQLException { + loadCsv("CSVTABLE", new String[] { "101,me,false" }, false, ""); + loadCsv("CSVTABLE", new String[] { "103,other,true" }, false, ""); + assertLoaded(2, "[{ID=101, NAME=me, STATUS=false}, {ID=103, NAME=other, STATUS=true}]"); + } + + @Test + public void whenNoDataRows_thenReturnZero() throws SQLException { + int count = loader.loadCsv("CSVTABLE", null, false, ""); + assertLoaded(count, "[]"); + count = loadCsv("CSVTABLE", new String[] { }, false, ""); //noheaders norows + assertLoaded(count, "[]"); + count = loadCsv("CSVTABLE", new String[] {"ID,NAME,STATUS" }, true, ""); //headers norows + assertLoaded(count, "[]"); + count = loadCsv("CSVTABLE", new String[] { }, true, ""); //headers missing + assertLoaded(count, "[]"); + } + + @Test + public void whenCsvValuesContainSpaces_thenValuesAreTrimmed() throws SQLException { + loadCsv("CSVTABLE", new String[] { " ID , \t NAME \r , STATUS ", " 101 ,\tmy\t name\r, false " }, true, ""); + assertLoaded(1, "[{ID=101, NAME=my\t name, STATUS=false}]"); + } + + @Test + public void whenNullStringUnset_thenEmptyIsNull() throws SQLException { + loadCsv("CSVTABLE", new String[] { " \t , me , \t ", "102,,true" }, false, ""); + assertLoaded(2, "[{ID=null, NAME=me, STATUS=null}, {ID=102, NAME=null, STATUS=true}]"); + } + + @Test + public void whenNullStringSet_thenEmptyIsNotNull_AndCaseInsensitive() throws SQLException { + loadCsv("CSVTABLE", new String[] { " null ,me, NULL ", "102, ,true" }, false, "null"); + assertLoaded(2, "[{ID=null, NAME=me, STATUS=null}, {ID=102, NAME=, STATUS=true}]"); + } + + @Test + public void whenRowsHaveDistinctSizes_thenRaiseExceptionWithMessage() throws SQLException { + RuntimeException e = assertThrows(RuntimeException.class, () -> { + loadCsv("CSVTABLE", new String[] { "ID,NAME,STATUS", "101,me,true,additional" , "102,you,true" }, true, ""); + }); + assertThat(e.getMessage(), equalTo( + "Error parsing CSV content to load into table CSVTABLE: Row 2 has 4 columns and should have 3")); + } + + @Test + public void whenParameterHasWrongType_thenRaiseExceptionWithMessage() throws SQLException { + RuntimeException e = assertThrows(RuntimeException.class, () -> { + loadCsv("CSVTABLE", new String[] { "ID,NAME,STATUS", "101,me,true" , "XXXX,you,true" }, true, ""); + }); + assertThat(e.getMessage(), equalTo("Error matching data type of parameters and table columns at CSV row 3")); + } + + @Test + public void whenInsertFails_thenRaiseExceptionWithMessage() throws SQLException { + RuntimeException e = assertThrows(RuntimeException.class, () -> { + loadCsv("CSVTABLE", new String[] { "101,me,true,extra1" , "102,you,true,extra2" }, false, ""); + }); + assertThat(e.getMessage(), equalTo("Error inserting data at CSV row 1")); + } + +} \ No newline at end of file diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioPostgresTest.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioPostgresTest.java new file mode 100644 index 00000000..7dd0c31a --- /dev/null +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioPostgresTest.java @@ -0,0 +1,24 @@ +package org.jsmart.zerocode.core.db; +import org.jsmart.zerocode.core.domain.Scenario; +import org.jsmart.zerocode.core.domain.TargetEnv; +import org.jsmart.zerocode.core.runner.ZeroCodeUnitRunner; +import org.junit.Test; +import org.junit.runner.RunWith; + +@TargetEnv("db_test_postgres.properties") +@RunWith(ZeroCodeUnitRunner.class) +public class DbSqlExecutorScenarioPostgresTest { + + // Note: Spin up the DB container before running this test: docker/compose/pg_compose.yml + + @Test + @Scenario("integration_test_files/db/db_csv_load_with_headers_postgres.json") + public void testDbCsvLoadWithHeaders() throws Exception { + } + + @Test + @Scenario("integration_test_files/db/db_sql_execute_postgres.json") + public void testDbSqlExecute() throws Exception { + } + +} diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java new file mode 100644 index 00000000..e294777c --- /dev/null +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlExecutorScenarioTest.java @@ -0,0 +1,40 @@ +package org.jsmart.zerocode.core.db; +import org.jsmart.zerocode.core.domain.Scenario; +import org.jsmart.zerocode.core.domain.TargetEnv; +import org.jsmart.zerocode.core.runner.ZeroCodeUnitRunner; +import org.junit.Test; +import org.junit.runner.RunWith; + +@TargetEnv("db_test.properties") +@RunWith(ZeroCodeUnitRunner.class) +public class DbSqlExecutorScenarioTest { + + @Test + @Scenario("integration_test_files/db/db_csv_load_with_headers.json") + public void testDbCsvLoadWithHeaders() throws Exception { + } + + @Test // same scenario and test database + @Scenario("integration_test_files/db/db_csv_load_without_headers.json") + public void testDbCsvLoadWithoutHeaders() throws Exception { + } + + @Test + @Scenario("integration_test_files/db/db_sql_execute.json") + public void testDbSqlExecute() throws Exception { + } + + // Manual test: error handling. + // To facilitate the location of the source of possible errors (e.g. a wrong SQL statement), + // exceptions that occur in the DbSqlExecutor should show: + // - A log entry with the error message + // - The stacktrace of the exception to facilitate locating the source + // - The usual chain of errors and stacktraces produced by zerocode when an step fails + // + // Recommended situations for manual test: + // - Target environment variables are no defined + // - A syntactically wrong SQL statement in a step + // - A header that does not correspond with any column when loading data from CSV + // - A value with the wrong data type (e.g. string in a numeric column) when loading data from CSV + +} diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlRunnerTest.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlRunnerTest.java new file mode 100644 index 00000000..79d433f5 --- /dev/null +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbSqlRunnerTest.java @@ -0,0 +1,63 @@ +package org.jsmart.zerocode.core.db; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; + +import java.sql.SQLException; +import java.util.List; +import java.util.Map; + +import org.apache.commons.dbutils.QueryRunner; +import org.jukito.JukitoRunner; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(JukitoRunner.class) +public class DbSqlRunnerTest extends DbTestBase { + + @Before + public void setUp() throws SQLException { + super.setUp(); + new QueryRunner().update(conn, "DROP TABLE IF EXISTS SQLTABLE; " + + "CREATE TABLE SQLTABLE (ID INTEGER, NAME VARCHAR(20)); " + + "INSERT INTO SQLTABLE VALUES (1, 'string 1'); " + + "INSERT INTO SQLTABLE VALUES (2, 'string 2');"); + } + + @Test + public void sqlSelectQueryShouldReturnListOfMap() throws ClassNotFoundException, SQLException { + List> rows = execute("SELECT ID, NAME FROM SQLTABLE ORDER BY ID DESC", null); + assertThat(rows.toString(), equalTo(convertDbCase("[{ID=2, NAME=string 2}, {ID=1, NAME=string 1}]"))); + } + + @Test + public void sqlSelectWithoutResultsShouldReturnEmptyList() throws ClassNotFoundException, SQLException { + List> rows = execute("SELECT ID, NAME FROM SQLTABLE where ID<0", null); + assertThat(rows.toString(), equalTo("[]")); + } + + @Test + public void multipleSqlSelectShouldReturnTheFirstResultSet() throws ClassNotFoundException, SQLException { + List> rows = execute("SELECT ID, NAME FROM SQLTABLE where ID=2; SELECT ID, NAME FROM SQLTABLE where ID=1;", null); + assertThat(rows.toString(), equalTo(convertDbCase("[{ID=2, NAME=string 2}]"))); + } + + @Test + public void sqlInsertShouldReturnNull() throws ClassNotFoundException, SQLException { + Object nullRows = execute("INSERT INTO SQLTABLE VALUES (3, 'string 3')", null); + assertThat(nullRows, nullValue()); + // check rows are inserted + List> rows = execute("SELECT ID, NAME FROM SQLTABLE ORDER BY ID", new Object[] {}); + assertThat(rows.toString(), equalTo(convertDbCase("[{ID=1, NAME=string 1}, {ID=2, NAME=string 2}, {ID=3, NAME=string 3}]"))); + } + + @Test + public void executeWithParametersShouldAllowNulls() throws SQLException { + execute("INSERT INTO SQLTABLE VALUES (?, ?)", new Object[] { 4, null }); + List> rows = execute("SELECT ID, NAME FROM SQLTABLE where ID = ?", new Object[] { 4 }); + assertThat(rows.toString(), equalTo(convertDbCase("[{ID=4, NAME=null}]"))); + } + +} diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbTestBase.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbTestBase.java new file mode 100644 index 00000000..86fe5a77 --- /dev/null +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbTestBase.java @@ -0,0 +1,70 @@ +package org.jsmart.zerocode.core.db; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; + +import org.apache.commons.dbutils.DbUtils; +import org.jsmart.zerocode.core.di.main.ApplicationMainModule; +import org.jukito.TestModule; +import org.junit.After; +import org.junit.Before; + +import com.google.inject.Inject; +import com.google.inject.name.Named; + +/** + * Base class for the unit DB test classes: manages connections, + * execution of queries and DBMS specific features + */ +public abstract class DbTestBase { + // Subclasses must use JukitoRunner + public static class JukitoModule extends TestModule { + @Override + protected void configureTest() { + ApplicationMainModule applicationMainModule = new ApplicationMainModule("db_test.properties"); + install(applicationMainModule); + } + } + + @Inject + @Named("db.driver.url") protected String url; + + @Inject(optional = true) + @Named("db.driver.user") protected String user; + + @Inject(optional = true) + @Named("db.driver.password") protected String password; + + protected Connection conn; // managed connection for each test + protected boolean isPostgres = false; // set by each connection, to allow portable assertions + + @Before + public void setUp() throws SQLException { + conn = connect(); + } + + @After + public void tearDown() throws Exception { + DbUtils.closeQuietly(conn); + } + + protected Connection connect() throws SQLException { + isPostgres = url.startsWith("jdbc:postgresql:"); + return DriverManager.getConnection(url, user, password); + } + + protected List> execute(String sql, Object[] params) throws SQLException { + DbSqlRunner runner = new DbSqlRunner(conn); + return runner.execute(sql, params); + } + + // Table and columns in all tests are uppercase because H2 stores uppercase by default. + // But postgres stores lowercase, so some expected strings need case conversion + protected String convertDbCase(String value) { + return isPostgres ? value.toLowerCase() : value; + } + +} \ No newline at end of file diff --git a/core/src/test/java/org/jsmart/zerocode/core/db/DbValueConverterTest.java b/core/src/test/java/org/jsmart/zerocode/core/db/DbValueConverterTest.java new file mode 100644 index 00000000..46ebe82a --- /dev/null +++ b/core/src/test/java/org/jsmart/zerocode/core/db/DbValueConverterTest.java @@ -0,0 +1,176 @@ +package org.jsmart.zerocode.core.db; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertThrows; + +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; + +import org.jukito.JukitoRunner; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(JukitoRunner.class) +public class DbValueConverterTest extends DbTestBase { + + @Test + public void convertBasicDataTypeValues() throws SQLException { + doTestConversion("", "Btable", "VINT INTEGER, VCHAR VARCHAR(20), VBOOL BOOLEAN, EXTRA CHAR(1)", + new String[] { "Vint", "Vchar", "Vbool" }, + new String[] { "101", "astring", "true" }, + "[{VINT=101, VCHAR=astring, VBOOL=true, EXTRA=null}]"); + } + + @Test + public void convertNullAndBitValues() throws SQLException { + doTestConversion("", "NTABLE", "VINT INT, VCHAR VARCHAR(20), VBOOL BOOLEAN", + new String[] { "VINT", "VCHAR", "VBOOL" }, + new String[] { null, null, "1" }, // incl. alternate boolean + "[{VINT=null, VCHAR=null, VBOOL=true}]"); + } + + @Test + public void convertDecimalAndFloatValues() throws SQLException { + doTestConversion("", "FTABLE", "VEXACT NUMERIC(10,0), VDEC DECIMAL(10,2), VFLOAT FLOAT, VREAL REAL", + new String[] { "VEXACT", "VDEC", "VFLOAT", "VREAL" }, + new String[] { "102", "123.45", "234.56", "3.4561E+2" }, + "[{VEXACT=102, VDEC=123.45, VFLOAT=234.56, VREAL=345.61}]"); + } + + @Test + public void convertDateAndTimeValues() throws SQLException { + List> rows = doTestConversion("", "DTABLE", "VTS1 TIMESTAMP, VTS2 TIMESTAMP, VTIME TIME, VDATE DATE", + new String[] { "VTS1", "VTS2", "VTIME", "VDATE" }, + new String[] { "2024-09-04T08:01:02.456+0300", "2024-09-04T08:01:02+0300", "08:01:02+0300", "2024-09-04" }, + null); + // assert individually to allow compare with GMT time (not local) + assertThat(gmtTimestamp((Date) rows.get(0).get("VTS1")), equalTo("2024-09-04T05:01:02.456")); + assertThat(gmtTimestamp((Date) rows.get(0).get("VTS2")), equalTo("2024-09-04T05:01:02.000")); + assertThat(gmtTimestamp((Date) rows.get(0).get("VTIME")), equalTo("1970-01-01T05:01:02.000")); + assertThat(rows.get(0).get("VDATE").toString(), "2024-09-04", equalTo(rows.get(0).get("VDATE").toString())); + } + + @Test + public void convertWithMixedCaseColumnName() throws SQLException { + // Uses a date type to ensure that is the converter who tries making the conversion, not the driver + // (neither H2 nor Postgres drivers do conversion of dates) + List> rows = doTestConversion("", "ITable", "VDATE DATE", + new String[] { "VDate" }, + new String[] { "2024-09-04" }, + "[{VDATE=2024-09-04}]"); + assertThat(rows.get(0).get("VDATE").toString(), equalTo("2024-09-04")); + } + + @Test + public void whenNoColumnsSpecified_ThenAllTableColumns_AreIncluded() throws SQLException { + doTestConversion("", "OTABLE", "VINT SMALLINT, VCHAR VARCHAR(20)", + null, + new String[] { "101", "astring" }, + "[{VINT=101, VCHAR=astring}]"); + } + + @Test + public void whenNoColumnsSpecified_AndColumnCountMismatch_ThenConversionFails() throws SQLException { + assertThrows(SQLException.class, () -> { + doTestConversion("", "FMTABLE", "VINT BIGINT", + null, + new String[] { "101", "999" }, + "[{VINT=101}]"); + }); + } + + @Test + public void whenColumnNotFound_ThenConversionFails() throws SQLException { + assertThrows(SQLException.class, () -> { + doTestConversion("", "FCTABLE", "VINT INTEGER, VCHAR VARCHAR(20)", + new String[] { "VINT", "NOTEXISTS" }, + new String[] { "101", "astring" }, + "[{VINT=101, VCHAR=notexists}]"); + }); + } + + @Test + public void whenValueHasWrongFormat_ThenConversionFails() throws SQLException { + assertThrows(SQLException.class, () -> { + doTestConversion("", "FVTABLE", "VTS TIMESTAMP", + new String[] { "VTS" }, + new String[] { "notadate" }, + "[{VTS=notadate}]"); + }); + } + + // Failures due to problems with metadata. + // Simulates failures getting metadata so that the conversion is left to + // be done by the driver. + // - Below tests will pass because the H2 driver converts numeric values + // - but fail if driver is changed to Postgres (does not convert numeric), skipped + + @Test + public void whenMetadataNotFound_ThenConversions_AreUpToTheDriver_WithColumns() throws SQLException { + if (isPostgres) + return; + doTestConversion("table", "F1TABLE", "VINT INTEGER, VCHAR VARCHAR(20)", + new String[] { "VINT", "VCHAR" }, + new String[] { "101", "astring" }, + "[{VINT=101, VCHAR=astring}]"); + } + + @Test + public void whenMetadataNotFound_ThenConversions_AreUpToTheDriver_WithoutColumns() throws SQLException { + if (isPostgres) + return; + doTestConversion("table", "F2CTABLE", "VINT INTEGER, VCHAR VARCHAR(20)", + null, + new String[] { "101", "astring" }, + "[{VINT=101, VCHAR=astring}]"); + } + + @Test + public void whenMetadataFails_ThenConversions_AreUpToTheDriver() throws SQLException { + if (isPostgres) + return; + doTestConversion("conn", "F3TABLE", "VINT INTEGER, VCHAR VARCHAR(20)", + new String[] { "VINT", "VCHAR" }, + new String[] { "101", "astring" }, + "[{VINT=101, VCHAR=astring}]"); + } + + private List> doTestConversion(String failureToSimulate, String table, + String ddlTypes, String[] columns, String[] params, String expected) throws SQLException { + execute("DROP TABLE IF EXISTS " + table + ";" + + " CREATE TABLE " + table + " (" + ddlTypes + ");", null); + String sql = "INSERT INTO " + table + + (columns != null ? " (" + String.join(",", columns) + ")" : "") + + " VALUES (" + placeholders(params.length) + ")"; + + DbValueConverter converter = new DbValueConverter( + "conn".equals(failureToSimulate) ? null : conn, + "table".equals(failureToSimulate) ? "notexists" : table); + Object[] converted = converter.convertColumnValues(columns, params); + execute(sql, converted); + + List> rows = execute("SELECT * FROM " + table, null); + if (expected != null) // null to check without specified columns + assertThat(rows.toString(), equalTo(convertDbCase(expected))); + return rows; + } + + private String placeholders(int columnCount) { + String[] placeholders = new String[columnCount]; + Arrays.fill(placeholders, "?"); + return String.join(",", placeholders); + } + + private String gmtTimestamp(Date dt) { + java.text.DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS"); + df.setTimeZone(TimeZone.getTimeZone("GMT")); + return df.format(dt); + } + +} diff --git a/core/src/test/resources/db_test.properties b/core/src/test/resources/db_test.properties new file mode 100644 index 00000000..fe99f714 --- /dev/null +++ b/core/src/test/resources/db_test.properties @@ -0,0 +1,7 @@ +# Connection info used by the DbSqlExecutor + +# JDBC connection string to the test database (H2) +db.driver.url=jdbc:h2:./target/test_db_sql_executor +# If connection requires authentication, specify user and password: +# db.driver.user= +# db.driver.password= diff --git a/core/src/test/resources/db_test_postgres.properties b/core/src/test/resources/db_test_postgres.properties new file mode 100644 index 00000000..ed699717 --- /dev/null +++ b/core/src/test/resources/db_test_postgres.properties @@ -0,0 +1,7 @@ +# Connection info used by the DbSqlExecutor + +# JDBC connection string to a PostgreSQL test database +# Spin up the DB container before running the postgres tests: docker/compose/pg_compose.yml +db.driver.url=jdbc:postgresql://localhost:35432/postgres +db.driver.user=postgres +db.driver.password=example diff --git a/core/src/test/resources/integration_test_files/db/db_csv_load_with_headers.json b/core/src/test/resources/integration_test_files/db/db_csv_load_with_headers.json new file mode 100644 index 00000000..f2f0f258 --- /dev/null +++ b/core/src/test/resources/integration_test_files/db/db_csv_load_with_headers.json @@ -0,0 +1,43 @@ +{ + "scenarioName": "DbSqlExecutor: Load a CSV file with headers", + "steps": [ + { + "name": "Test database setup", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "DROP TABLE IF EXISTS PLAYERS; CREATE TABLE PLAYERS (ID INTEGER, NAME VARCHAR(20), AGE INTEGER);" + }, + "verify": { } + }, + { + "name": "Insert rows from a CSV file with headers", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "LOADCSV", + "request": { + "tableName": "players", + "csvSource": "integration_test_files/db/players_with_headers.csv", + "withHeaders" : true + }, + "verify": { + "size" : 3 + } + }, + { + "name": "Check the content of inserted rows", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "SELECT ID, NAME, AGE FROM PLAYERS ORDER BY ID" + }, + "verify": { + "rows.SIZE": 3, + "rows": [ //<-- to make this pass in postgres, set the keys to lowercase + { "ID": 1001, "NAME": "Ronaldo", "AGE": 23 }, + { "ID": 1002, "NAME": "Devaldo", "AGE": null }, + { "ID": 1003, "NAME": "Trevaldo", "AGE": 35 } + ] + } + } + ] +} \ No newline at end of file diff --git a/core/src/test/resources/integration_test_files/db/db_csv_load_with_headers_postgres.json b/core/src/test/resources/integration_test_files/db/db_csv_load_with_headers_postgres.json new file mode 100644 index 00000000..544e32c0 --- /dev/null +++ b/core/src/test/resources/integration_test_files/db/db_csv_load_with_headers_postgres.json @@ -0,0 +1,43 @@ +{ + "scenarioName": "DbSqlExecutor: Load a CSV file with headers - PostgreSQL", + "steps": [ + { + "name": "Test database setup", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "DROP TABLE IF EXISTS PLAYERS; CREATE TABLE PLAYERS (ID INTEGER, NAME VARCHAR(20), AGE INTEGER);" + }, + "verify": { } + }, + { + "name": "Insert rows from a CSV file with headers", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "LOADCSV", + "request": { + "tableName": "players", + "csvSource": "integration_test_files/db/players_with_headers.csv", + "withHeaders" : true + }, + "verify": { + "size" : 3 + } + }, + { + "name": "Check the content of inserted rows", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "SELECT ID, NAME, AGE FROM PLAYERS ORDER BY ID" + }, + "verify": { + "rows.SIZE": 3, + "rows": [ //<-- same than db_csv_load_with_headers.json, but keys in lowercase (postgres converts to lower) + { "id": 1001, "name": "Ronaldo", "age": 23 }, + { "id": 1002, "name": "Devaldo", "age": null }, + { "id": 1003, "name": "Trevaldo", "age": 35 } + ] + } + } + ] +} \ No newline at end of file diff --git a/core/src/test/resources/integration_test_files/db/db_csv_load_without_headers.json b/core/src/test/resources/integration_test_files/db/db_csv_load_without_headers.json new file mode 100644 index 00000000..16212042 --- /dev/null +++ b/core/src/test/resources/integration_test_files/db/db_csv_load_without_headers.json @@ -0,0 +1,43 @@ +{ + "scenarioName": "DbSqlExecutor: Load a CSV file without headers", + "steps": [ + { + "name": "Test database setup", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "DROP TABLE IF EXISTS PLAYERS; CREATE TABLE PLAYERS (ID INTEGER, NAME VARCHAR(20), AGE INTEGER);" + }, + "verify": { } + }, + { + "name": "Insert rows from a CSV file without headers", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "LOADCSV", + "request": { + "tableName": "players", + "csvSource": "integration_test_files/db/players_without_headers.csv", + "nullString": "NULL!!" + }, + "verify": { + "size" : 3 + } + }, + { + "name": "Check the content of inserted rows", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "SELECT ID, NAME, AGE FROM PLAYERS ORDER BY ID" + }, + "verify": { + "rows.SIZE": 3, + "rows": [ //<-- to make this pass in postgres, set the keys to lowercase + { "ID": 1001, "NAME": "Ronaldo", "AGE": 23 }, + { "ID": 1002, "NAME": null, "AGE": 24 }, + { "ID": 1003, "NAME": "Trevaldo", "AGE": 35 } + ] + } + } + ] +} \ No newline at end of file diff --git a/core/src/test/resources/integration_test_files/db/db_sql_execute.json b/core/src/test/resources/integration_test_files/db/db_sql_execute.json new file mode 100644 index 00000000..6e0d0daf --- /dev/null +++ b/core/src/test/resources/integration_test_files/db/db_sql_execute.json @@ -0,0 +1,49 @@ +{ + "scenarioName": "DbSqlExecutor: Read and write data using SQL", + "steps": [ + { + "name": "Test database setup", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "DROP TABLE IF EXISTS PEOPLE; CREATE TABLE PEOPLE (ID INTEGER, NAME VARCHAR(20), START DATE, ACTIVE BOOLEAN);" + }, + "verify": { } + }, + { + "name": "Insert rows using SQL", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "INSERT INTO PEOPLE VALUES (1, 'Jeff Bejo', '2024-09-01', true); INSERT INTO PEOPLE VALUES (2, 'John Bajo', '2024-09-02', false);" + }, + "verify": { } + }, + { + "name": "Insert with parameters and nulls", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "execute", //<-- Uppercase for consistency, but also allows lowercase + "request": { + "sql": "INSERT INTO PEOPLE (ID, NAME, START, ACTIVE) VALUES (?, ?, ?, ?);", + "sqlParams": [3, null, null, true] + }, + "verify": { } + }, + { + "name": "Retrieve rows using SQL", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "SELECT ID, NAME, to_char(START,'yyyy-MM-dd') AS START, ACTIVE FROM PEOPLE WHERE ACTIVE=?", + "sqlParams": [true] + }, + "verify": { + "rows.SIZE": 2, + "rows": [ //<-- to make this pass in postgres, set the keys to lowercase + { "ID": 1, "NAME": "Jeff Bejo", "START": "2024-09-01", "ACTIVE": true }, + { "ID": 3, "NAME": null, "START": null, "ACTIVE": true } + ] + } + } + ] +} \ No newline at end of file diff --git a/core/src/test/resources/integration_test_files/db/db_sql_execute_postgres.json b/core/src/test/resources/integration_test_files/db/db_sql_execute_postgres.json new file mode 100644 index 00000000..11d283ef --- /dev/null +++ b/core/src/test/resources/integration_test_files/db/db_sql_execute_postgres.json @@ -0,0 +1,49 @@ +{ + "scenarioName": "DbSqlExecutor: Read and write data using SQL - PostgreSQL", + "steps": [ + { + "name": "Test database setup", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "DROP TABLE IF EXISTS PEOPLE; CREATE TABLE PEOPLE (ID INTEGER, NAME VARCHAR(20), START DATE, ACTIVE BOOLEAN);" + }, + "verify": { } + }, + { + "name": "Insert rows using SQL", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "INSERT INTO PEOPLE VALUES (1, 'Jeff Bejo', '2024-09-01', true); INSERT INTO PEOPLE VALUES (2, 'John Bajo', '2024-09-02', false);" + }, + "verify": { } + }, + { + "name": "Insert with parameters and nulls", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "execute", //<-- Uppercase for consistency, but also allows lowercase + "request": { + "sql": "INSERT INTO PEOPLE (ID, NAME, START, ACTIVE) VALUES (?, ?, ?, ?);", + "sqlParams": [3, null, null, true] + }, + "verify": { } + }, + { + "name": "Retrieve rows using SQL", + "url": "org.jsmart.zerocode.core.db.DbSqlExecutor", + "operation": "EXECUTE", + "request": { + "sql": "SELECT ID, NAME, to_char(START,'yyyy-MM-dd') AS START, ACTIVE FROM PEOPLE WHERE ACTIVE=?", + "sqlParams": [true] + }, + "verify": { + "rows.SIZE": 2, + "rows": [ //<-- same than db_sql_execute.json, but keys in lowercase (postgres converts to lower) + { "id": 1, "name": "Jeff Bejo", "start": "2024-09-01", "active": true }, + { "id": 3, "name": null, "start": null, "active": true } + ] + } + } + ] +} \ No newline at end of file diff --git a/core/src/test/resources/integration_test_files/db/players_with_headers.csv b/core/src/test/resources/integration_test_files/db/players_with_headers.csv new file mode 100644 index 00000000..22f8c751 --- /dev/null +++ b/core/src/test/resources/integration_test_files/db/players_with_headers.csv @@ -0,0 +1,4 @@ +ID, AGE, NAME +1001, 23, Ronaldo +1002, , Devaldo +1003, 35, Trevaldo \ No newline at end of file diff --git a/core/src/test/resources/integration_test_files/db/players_without_headers.csv b/core/src/test/resources/integration_test_files/db/players_without_headers.csv new file mode 100644 index 00000000..d6f049e4 --- /dev/null +++ b/core/src/test/resources/integration_test_files/db/players_without_headers.csv @@ -0,0 +1,3 @@ +1001, Ronaldo, 23 +1002, null!!, 24 +1003, Trevaldo, 35 \ No newline at end of file diff --git a/docker/compose/pg_compose.yml b/docker/compose/pg_compose.yml index 100398f6..3ad83a2e 100644 --- a/docker/compose/pg_compose.yml +++ b/docker/compose/pg_compose.yml @@ -12,9 +12,3 @@ services: POSTGRES_PASSWORD: example ports: - 35432:5432 - - adminer: - image: adminer:4.7.0 - restart: always - ports: - - 8080:8080 \ No newline at end of file diff --git a/pom.xml b/pom.xml index c9658480..9d354afa 100644 --- a/pom.xml +++ b/pom.xml @@ -84,10 +84,12 @@ 4.4 3.14.0 1.11.0 + 1.8.1 1.1.10 4.5.13 4.5.12 2.2.220 + 42.7.4 5.0.9 3.7.0 2.10.1 @@ -287,12 +289,22 @@ ${micro-simulator.version} test + + commons-dbutils + commons-dbutils + ${commons-dbutils.version} + com.h2database h2 ${h2.db.version} test + + org.postgresql + postgresql + ${postgres.db.version} + com.aventstack extentreports