diff --git a/BUILDING.md b/BUILDING.md index 7866abd7e..e4f8654cb 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -25,9 +25,17 @@ or mvn -pl core clean test ``` +## Runing Only The Integration tests(core) +Right click and run the "integrationtests" package. It picks the tests ending with "*Test.java" +integration_tests_only_running_ + + ## With tests executed(kafka) Some tests require a running Kafka (and some related components like kafka-rest, and kafka-schema-registry). +Location: +https://github.com/authorjapps/zerocode/blob/master/docker/compose/kafka-schema-registry.yml + Download the file, and run(or `cd to the docker/compose` dir and run) ``` docker-compose -f kafka-schema-registry.yml up -d diff --git a/README.md b/README.md index afcd1df1d..04ea15ce4 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Documentation === For a quick introduction to Zerocode and its features, visit the + [Zerocode TDD Doc Site](https://zerocode-tdd-docs.pages.dev) ++ [Wan to contribute or Improve](https://github.com/authorjapps/zerocode/wiki/Documentation-How-To-Fix-Steps)? Steps and guidelines are [here](https://github.com/authorjapps/zerocode/wiki/Documentation-How-To-Fix-Steps) IDE Support By === diff --git a/core/src/main/java/org/jsmart/zerocode/core/constants/ZeroCodeReportConstants.java b/core/src/main/java/org/jsmart/zerocode/core/constants/ZeroCodeReportConstants.java index f812ff6ec..855e4a22b 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/constants/ZeroCodeReportConstants.java +++ b/core/src/main/java/org/jsmart/zerocode/core/constants/ZeroCodeReportConstants.java @@ -16,6 +16,7 @@ public interface ZeroCodeReportConstants { String REPORT_TITLE_DEFAULT = "Zerocode Test Report"; String REPORT_DISPLAY_NAME_DEFAULT = "Zerocode Interactive Report"; String DEFAULT_REGRESSION_CATEGORY = "Regression"; + String DEFAULT_REGRESSION_AUTHOR = "All"; String LINK_LABEL_NAME = "Spike Chart(Click here)"; String ZEROCODE_JUNIT = "zerocode.junit"; String CHARTS_AND_CSV = "gen-smart-charts-csv-reports"; diff --git a/core/src/main/java/org/jsmart/zerocode/core/domain/Parameterized.java b/core/src/main/java/org/jsmart/zerocode/core/domain/Parameterized.java index 14b119b4f..e37b50b04 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/domain/Parameterized.java +++ b/core/src/main/java/org/jsmart/zerocode/core/domain/Parameterized.java @@ -1,17 +1,33 @@ package org.jsmart.zerocode.core.domain; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import org.apache.commons.lang.StringUtils; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; public class Parameterized { private final List valueSource; private final List csvSource; + private final Boolean ignoreHeader; public Parameterized( @JsonProperty("valueSource") List valueSource, - @JsonProperty("csvSource") List csvSource) { + @JsonProperty("csvSource") JsonNode csvSourceJsonNode, + @JsonProperty("ignoreHeader") Boolean ignoreHeader) { this.valueSource = valueSource; - this.csvSource = csvSource; + this.ignoreHeader = Optional.ofNullable(ignoreHeader).orElse(false); + this.csvSource = getCsvSourceFrom(csvSourceJsonNode); } public List getValueSource() { @@ -22,6 +38,43 @@ public List getCsvSource() { return csvSource; } + private List getCsvSourceFrom(JsonNode csvSourceJsonNode) { + try { + if (csvSourceJsonNode.isArray()) { + return readCsvSourceFromJson(csvSourceJsonNode); + + } else { + return readCsvSourceFromExternalCsvFile(csvSourceJsonNode); + } + } catch (IOException e) { + throw new RuntimeException("Error deserializing csvSource", e); + } + } + + private List readCsvSourceFromJson(JsonNode csvSourceJsonNode) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + ObjectReader reader = mapper.readerFor(new TypeReference>() { + }); + return reader.readValue(csvSourceJsonNode); + } + + private List readCsvSourceFromExternalCsvFile(JsonNode csvSourceJsonNode) throws IOException { + String csvSourceFilePath = csvSourceJsonNode.textValue(); + if (StringUtils.isNotBlank(csvSourceFilePath)) { + Path path = Paths.get("./src/test/resources/",csvSourceFilePath); + List csvSourceFileLines = Files.lines(path) + .filter(StringUtils::isNotBlank) + .collect(Collectors.toList()); + if (this.ignoreHeader) { + return csvSourceFileLines.stream() + .skip(1) + .collect(Collectors.toList()); + } + return csvSourceFileLines; + } + return Collections.emptyList(); + } + @Override public String toString() { return "Parameterized{" + diff --git a/core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeAssertionsProcessor.java b/core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeAssertionsProcessor.java index 4b844fdd6..c74a8d9b8 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeAssertionsProcessor.java +++ b/core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeAssertionsProcessor.java @@ -1,5 +1,6 @@ package org.jsmart.zerocode.core.engine.preprocessor; +import org.jsmart.zerocode.core.domain.Step; import org.jsmart.zerocode.core.engine.assertion.FieldAssertionMatcher; import org.jsmart.zerocode.core.engine.assertion.JsonAsserter; @@ -18,4 +19,11 @@ public interface ZeroCodeAssertionsProcessor { List createJsonAsserters(String resolvedAssertionJson); List assertAllAndReturnFailed(List asserters, String executionResult); + + Step resolveJsonContent(Step thisStep, ScenarioExecutionState scenarioExecutionState); + + String fieldMasksRemoved(String resolvedRequestJson); + + String fieldMasksApplied(String resolvedRequestJson); + } diff --git a/core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeAssertionsProcessorImpl.java b/core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeAssertionsProcessorImpl.java index 6eacf0c76..e083918fc 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeAssertionsProcessorImpl.java +++ b/core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeAssertionsProcessorImpl.java @@ -1,5 +1,6 @@ package org.jsmart.zerocode.core.engine.preprocessor; +import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -7,6 +8,28 @@ import com.google.inject.Inject; import com.google.inject.name.Named; import com.jayway.jsonpath.JsonPath; +import net.minidev.json.JSONArray; +import org.apache.commons.lang.text.StrSubstitutor; +import org.jsmart.zerocode.core.domain.Step; +import org.jsmart.zerocode.core.engine.assertion.FieldAssertionMatcher; +import org.jsmart.zerocode.core.engine.assertion.JsonAsserter; +import org.jsmart.zerocode.core.engine.assertion.array.ArrayIsEmptyAsserterImpl; +import org.jsmart.zerocode.core.engine.assertion.array.ArraySizeAsserterImpl; +import org.jsmart.zerocode.core.engine.assertion.field.FieldContainsStringAsserter; +import org.jsmart.zerocode.core.engine.assertion.field.FieldContainsStringIgnoreCaseAsserter; +import org.jsmart.zerocode.core.engine.assertion.field.FieldHasDateAfterValueAsserter; +import org.jsmart.zerocode.core.engine.assertion.field.FieldHasDateBeforeValueAsserter; +import org.jsmart.zerocode.core.engine.assertion.field.FieldHasEqualNumberValueAsserter; +import org.jsmart.zerocode.core.engine.assertion.field.FieldHasExactValueAsserter; +import org.jsmart.zerocode.core.engine.assertion.field.FieldHasGreaterThanValueAsserter; +import org.jsmart.zerocode.core.engine.assertion.field.FieldHasInEqualNumberValueAsserter; +import org.jsmart.zerocode.core.engine.assertion.field.FieldHasLesserThanValueAsserter; +import org.jsmart.zerocode.core.engine.assertion.field.FieldIsNotNullAsserter; +import org.jsmart.zerocode.core.engine.assertion.field.FieldIsNullAsserter; +import org.jsmart.zerocode.core.engine.assertion.field.FieldIsOneOfValueAsserter; +import org.jsmart.zerocode.core.engine.assertion.field.FieldMatchesCustomAsserter; +import org.jsmart.zerocode.core.engine.assertion.field.FieldMatchesRegexPatternAsserter; + import java.io.IOException; import java.math.BigDecimal; import java.time.LocalDateTime; @@ -18,28 +41,47 @@ import java.util.List; import java.util.Map; import java.util.Properties; -import net.minidev.json.JSONArray; -import org.apache.commons.lang.text.StrSubstitutor; -import org.jsmart.zerocode.core.engine.assertion.FieldAssertionMatcher; -import org.jsmart.zerocode.core.engine.assertion.JsonAsserter; -import org.jsmart.zerocode.core.engine.assertion.array.ArrayIsEmptyAsserterImpl; -import org.jsmart.zerocode.core.engine.assertion.array.ArraySizeAsserterImpl; -import org.jsmart.zerocode.core.engine.assertion.field.*; import static java.lang.Integer.valueOf; import static java.lang.String.format; import static org.apache.commons.lang.StringEscapeUtils.escapeJava; import static org.apache.commons.lang.StringUtils.substringBetween; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.*; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_LOCAL_DATETIME_AFTER; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_LOCAL_DATETIME_BEFORE; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_PATH_SIZE; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_CONTAINS_STRING; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_CONTAINS_STRING_IGNORE_CASE; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_CUSTOM_ASSERT; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_EMPTY_ARRAY; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_EQUAL_TO_NUMBER; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_GREATER_THAN; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_IS_NOT_NULL; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_IS_NULL; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_IS_ONE_OF; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_LESSER_THAN; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_MATCHES_STRING; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_NOT_EQUAL_TO_NUMBER; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_NOT_NULL; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_NULL; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_ONE_OF; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.RAW_BODY; import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.$VALUE; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.JSON_CONTENT; import static org.jsmart.zerocode.core.utils.FieldTypeConversionUtils.deepTypeCast; import static org.jsmart.zerocode.core.utils.FieldTypeConversionUtils.fieldTypes; import static org.jsmart.zerocode.core.utils.PropertiesProviderUtils.loadAbsoluteProperties; +import static org.jsmart.zerocode.core.utils.SmartUtils.checkDigNeeded; +import static org.jsmart.zerocode.core.utils.SmartUtils.getJsonFilePhToken; import static org.jsmart.zerocode.core.utils.SmartUtils.isValidAbsolutePath; +import static org.jsmart.zerocode.core.utils.TokenUtils.getMasksRemoved; +import static org.jsmart.zerocode.core.utils.TokenUtils.getMasksReplaced; import static org.jsmart.zerocode.core.utils.TokenUtils.getTestCaseTokens; import static org.jsmart.zerocode.core.utils.TokenUtils.populateParamMap; import static org.slf4j.LoggerFactory.getLogger; +; +; + public class ZeroCodeAssertionsProcessorImpl implements ZeroCodeAssertionsProcessor { private static final org.slf4j.Logger LOGGER = getLogger(ZeroCodeAssertionsProcessorImpl.class); @@ -337,6 +379,49 @@ public List assertAllAndReturnFailed(List a return failedReports; } + /** + * Resolves JSON.CONTENT as object or array + * + * First the logic checks if dig-deep needed to avoid unwanted recursions. If not needed, the step definition is + * returned intact. Otherwise calls the dig deep method to perform the operation. + * + * @param thisStep + * @return The effective step definition + */ + @Override + public Step resolveJsonContent(Step thisStep, ScenarioExecutionState scenarioExecutionState) { + try { + if (!checkDigNeeded(mapper, thisStep, JSON_CONTENT)) { + return thisStep; + } + + JsonNode stepNode = mapper.convertValue(thisStep, JsonNode.class); + + Map stepMap = mapper.readValue(stepNode.toString(), new TypeReference>() { + }); + + digReplaceContent(stepMap, scenarioExecutionState); + + JsonNode jsonStepNode = mapper.valueToTree(stepMap); + + return mapper.treeToValue(jsonStepNode, Step.class); + + } catch (Exception e) { + LOGGER.error("Json content reading exception - {}", e.getMessage()); + throw new RuntimeException("Json content reading exception. Details - " + e); + } + } + + @Override + public String fieldMasksRemoved(String resolvedRequestJson) { + return getMasksRemoved(resolvedRequestJson); + } + + @Override + public String fieldMasksApplied(String resolvedRequestJson) { + return getMasksReplaced(resolvedRequestJson); + } + private void loadAnnotatedHostProperties() { try { if(isValidAbsolutePath(hostFileName)){ @@ -408,4 +493,39 @@ private boolean hasNoTypeCast(String resolvedJson) { } + void digReplaceContent(Map map, ScenarioExecutionState scenarioExecutionState) { + map.entrySet().forEach(entry -> { + Object value = entry.getValue(); + + if (value instanceof Map) { + digReplaceContent((Map) value, scenarioExecutionState); + } else { + LOGGER.debug("Leaf node found = {}, checking for any json content...", value); + if (value != null && (value.toString().contains(JSON_CONTENT))) { + LOGGER.debug("Found JSON content place holder = {}. Replacing with content", value); + String valueString = value.toString(); + String token = getJsonFilePhToken(valueString); + + if (token != null && (token.startsWith(JSON_CONTENT))) { + try { + String resolvedRequestJson = resolveStringJson( + "${" + token.substring(JSON_CONTENT.length()) + "}", + scenarioExecutionState.getResolvedScenarioState()); + resolvedRequestJson = resolvedRequestJson.replaceAll("\\\\", ""); + try { + JsonNode jsonNode = mapper.readTree(resolvedRequestJson); + entry.setValue(jsonNode); + } catch (JsonParseException e) { + //value is not a json string, but a string value + entry.setValue(resolvedRequestJson); + } + } catch (Exception exx) { + LOGGER.error("External file reference exception - {}", exx.getMessage()); + throw new RuntimeException(exx); + } + } + } + } + }); + } } diff --git a/core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeExternalFileProcessorImpl.java b/core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeExternalFileProcessorImpl.java index daf3ac329..b34fd1c00 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeExternalFileProcessorImpl.java +++ b/core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeExternalFileProcessorImpl.java @@ -13,12 +13,15 @@ import com.google.inject.name.Named; import org.jsmart.zerocode.core.domain.Step; +import org.jsmart.zerocode.core.utils.SmartUtils; import org.slf4j.Logger; import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.JSON_PAYLOAD_FILE; import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.YAML_PAYLOAD_FILE; import static org.jsmart.zerocode.core.utils.SmartUtils.readJsonAsString; import static org.jsmart.zerocode.core.utils.SmartUtils.readYamlAsString; +import static org.jsmart.zerocode.core.utils.SmartUtils.checkDigNeeded; +import static org.jsmart.zerocode.core.utils.SmartUtils.getJsonFilePhToken; import static org.jsmart.zerocode.core.utils.TokenUtils.getTestCaseTokens; import static org.slf4j.LoggerFactory.getLogger; @@ -69,7 +72,7 @@ public Step resolveExtJsonFile(Step thisStep) { try { - if (!checkDigNeeded(thisStep)) { + if (!checkDigNeeded(objectMapper, thisStep, JSON_PAYLOAD_FILE, YAML_PAYLOAD_FILE)) { return thisStep; } @@ -176,22 +179,4 @@ else if (token != null && token.startsWith(OTHER_FILE)) { } }); } - - private String getJsonFilePhToken(String valueString) { - if (valueString != null) { - List allTokens = getTestCaseTokens(valueString); - if (allTokens != null && !allTokens.isEmpty()) { - return allTokens.get(0); - } - } - return null; - } - - boolean checkDigNeeded(Step thisStep) throws JsonProcessingException { - String stepJson = objectMapper.writeValueAsString(thisStep); - List allTokens = getTestCaseTokens(stepJson); - - return allTokens.toString().contains(JSON_PAYLOAD_FILE) || allTokens.toString().contains(YAML_PAYLOAD_FILE); - } - } diff --git a/core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeParameterizedProcessorImpl.java b/core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeParameterizedProcessorImpl.java index 63466366f..7c899c664 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeParameterizedProcessorImpl.java +++ b/core/src/main/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeParameterizedProcessorImpl.java @@ -4,17 +4,19 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.univocity.parsers.csv.CsvParser; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.text.StrSubstitutor; +import org.jsmart.zerocode.core.domain.ScenarioSpec; +import org.slf4j.Logger; + import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; -import org.apache.commons.lang.text.StrSubstitutor; -import org.jsmart.zerocode.core.domain.ScenarioSpec; -import org.slf4j.Logger; -import static org.jsmart.zerocode.core.di.provider.CsvParserProvider.LINE_SEPARATOR; import static org.jsmart.zerocode.core.constants.ZerocodeConstants.DSL_FORMAT; +import static org.jsmart.zerocode.core.di.provider.CsvParserProvider.LINE_SEPARATOR; import static org.slf4j.LoggerFactory.getLogger; /** @@ -25,17 +27,17 @@ *

* Parameters can be * "parameterized": [ - * 200, - * "Hello", - * true + * 200, + * "Hello", + * true * ] *

* -or- *

* "parameterizedCsv": [ - * "1, 2, 200", - * "11, 22, 400", - * "21, 31, 500" + * "1, 2, 200", + * "11, 22, 400", + * "21, 31, 500" * ] *

* In each the above cases, the step will execute 3 times. @@ -64,7 +66,7 @@ public ZeroCodeParameterizedProcessorImpl(ObjectMapper objectMapper, CsvParser c @Override public ScenarioSpec resolveParameterized(ScenarioSpec scenario, int iteration) { - if(scenario.getParameterized() == null){ + if (scenario.getParameterized() == null) { return scenario; @@ -72,7 +74,7 @@ public ScenarioSpec resolveParameterized(ScenarioSpec scenario, int iteration) { return resolveParamsValues(scenario, iteration); - } else if (scenario.getParameterized().getCsvSource() != null) { + } else if (CollectionUtils.isNotEmpty(scenario.getParameterized().getCsvSource())) { return resolveParamsCsv(scenario, iteration); diff --git a/core/src/main/java/org/jsmart/zerocode/core/engine/tokens/ZeroCodeValueTokens.java b/core/src/main/java/org/jsmart/zerocode/core/engine/tokens/ZeroCodeValueTokens.java index 496fda747..f73b0b072 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/engine/tokens/ZeroCodeValueTokens.java +++ b/core/src/main/java/org/jsmart/zerocode/core/engine/tokens/ZeroCodeValueTokens.java @@ -31,11 +31,16 @@ public class ZeroCodeValueTokens { public static final String SYSTEM_ENV = "SYSTEM.ENV:"; public static final String $VALUE = ".$VALUE"; public static final String ABS_PATH = "ABS.PATH:"; + public static final String JSON_CONTENT = "JSON.CONTENT:"; + public static final String MASKED = "MASKED:"; + public static final String MASKED_STR = "***masked***"; + public static Map globalTokenCache = new HashMap<>(); public static List getKnownTokens() { return asList( + MASKED, PREFIX_ASU, RANDOM_NUMBER, GLOBAL_RANDOM_NUMBER, diff --git a/core/src/main/java/org/jsmart/zerocode/core/httpclient/utils/FileUploadUtils.java b/core/src/main/java/org/jsmart/zerocode/core/httpclient/utils/FileUploadUtils.java index 8774de198..3c86220fe 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/httpclient/utils/FileUploadUtils.java +++ b/core/src/main/java/org/jsmart/zerocode/core/httpclient/utils/FileUploadUtils.java @@ -57,7 +57,7 @@ public static void buildOtherRequestParams(Map fileFieldNameValu if (entry.getKey().equals(FILES_FIELD) || entry.getKey().equals(BOUNDARY_FIELD)) { continue; } - multipartEntityBuilder.addPart(entry.getKey(), new StringBody((String) entry.getValue(), TEXT_PLAIN)); + multipartEntityBuilder.addPart(entry.getKey(), new StringBody(entry.getValue().toString(), TEXT_PLAIN)); } } diff --git a/core/src/main/java/org/jsmart/zerocode/core/kafka/consume/ConsumerLocalConfigs.java b/core/src/main/java/org/jsmart/zerocode/core/kafka/consume/ConsumerLocalConfigs.java index 865992518..86b49df10 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/kafka/consume/ConsumerLocalConfigs.java +++ b/core/src/main/java/org/jsmart/zerocode/core/kafka/consume/ConsumerLocalConfigs.java @@ -23,6 +23,9 @@ public class ConsumerLocalConfigs { private final String protoClassType; private final Boolean cacheByTopic; private final String filterByJsonPath; + private final String seekEpoch; + private final SeekTimestamp seekTimestamp; + @JsonCreator public ConsumerLocalConfigs( @@ -36,9 +39,11 @@ public ConsumerLocalConfigs( @JsonProperty("pollingTime") Long pollingTime, @JsonProperty("cacheByTopic") Boolean cacheByTopic, @JsonProperty("filterByJsonPath") String filterByJsonPath, - @JsonProperty("seek") String seek) { + @JsonProperty("seek") String seek, + @JsonProperty("seekEpoch") String seekEpoch, + @JsonProperty("seekTimestamp") SeekTimestamp seekTimestamp) { this.recordType = recordType; - this.protoClassType= protobufMessageClassType; + this.protoClassType = protobufMessageClassType; this.fileDumpTo = fileDumpTo; this.commitAsync = commitAsync; this.commitSync = commitSync; @@ -48,39 +53,45 @@ public ConsumerLocalConfigs( this.cacheByTopic = cacheByTopic; this.filterByJsonPath = filterByJsonPath; this.seek = seek; + this.seekEpoch = seekEpoch; + this.seekTimestamp = seekTimestamp; } - + public ConsumerLocalConfigs( - String recordType, - String fileDumpTo, - Boolean commitAsync, + String recordType, + String fileDumpTo, + Boolean commitAsync, Boolean commitSync, Boolean showRecordsConsumed, Integer maxNoOfRetryPollsOrTimeouts, Long pollingTime, Boolean cacheByTopic, String filterByJsonPath, - String seek) { - this(recordType, null, + String seek, + String seekEpoch, + SeekTimestamp seekTimestamp) { + this(recordType, null, fileDumpTo, commitAsync, commitSync, showRecordsConsumed, maxNoOfRetryPollsOrTimeouts, - pollingTime, + pollingTime, cacheByTopic, filterByJsonPath, - seek); + seek, + seekEpoch, + seekTimestamp); } public String getRecordType() { return recordType != null ? recordType : RAW; } - - public String getProtoClassType() { - return protoClassType; - } + + public String getProtoClassType() { + return protoClassType; + } public String getFileDumpTo() { @@ -119,6 +130,14 @@ public String getSeek() { return seek; } + public String getSeekEpoch() { + return seekEpoch; + } + + public SeekTimestamp getSeekTimestamp() { + return seekTimestamp; + } + @JsonIgnore public String[] getSeekTopicPartitionOffset() { return seek.split(","); @@ -130,7 +149,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; ConsumerLocalConfigs that = (ConsumerLocalConfigs) o; return Objects.equals(recordType, that.recordType) && - Objects.equals(protoClassType, that.protoClassType) && + Objects.equals(protoClassType, that.protoClassType) && Objects.equals(fileDumpTo, that.fileDumpTo) && Objects.equals(commitAsync, that.commitAsync) && Objects.equals(commitSync, that.commitSync) && @@ -139,13 +158,14 @@ public boolean equals(Object o) { Objects.equals(pollingTime, that.pollingTime) && Objects.equals(filterByJsonPath, that.filterByJsonPath) && Objects.equals(cacheByTopic, that.cacheByTopic) && - Objects.equals(seek, that.seek); + Objects.equals(seek, that.seek) && + Objects.equals(seekEpoch, that.seekEpoch); } @Override public int hashCode() { - return Objects.hash(recordType, fileDumpTo, commitAsync, commitSync, showRecordsConsumed, maxNoOfRetryPollsOrTimeouts, pollingTime,cacheByTopic, filterByJsonPath, seek); + return Objects.hash(recordType, fileDumpTo, commitAsync, commitSync, showRecordsConsumed, maxNoOfRetryPollsOrTimeouts, pollingTime, cacheByTopic, filterByJsonPath, seek); } @Override @@ -162,6 +182,8 @@ public String toString() { ", cacheByTopic=" + cacheByTopic + ", filterByJsonPath=" + filterByJsonPath + ", seek=" + seek + + ", seekEpoch=" + seekEpoch + + ", seekTimestamp=" + seekTimestamp + '}'; } } diff --git a/core/src/main/java/org/jsmart/zerocode/core/kafka/consume/SeekTimestamp.java b/core/src/main/java/org/jsmart/zerocode/core/kafka/consume/SeekTimestamp.java new file mode 100644 index 000000000..6381ee252 --- /dev/null +++ b/core/src/main/java/org/jsmart/zerocode/core/kafka/consume/SeekTimestamp.java @@ -0,0 +1,38 @@ +package org.jsmart.zerocode.core.kafka.consume; + + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class SeekTimestamp { + + private final String timestamp; + private final String format; + + + @JsonCreator + public SeekTimestamp( + @JsonProperty("timestamp") String timestamp, + @JsonProperty("format") String format) { + this.timestamp = timestamp; + this.format = format; + } + + public String getTimestamp() { + return timestamp; + } + + public String getFormat() { + return format; + } + + @Override + public String toString() { + return "SeekTimestamp{" + + "timestamp='" + timestamp + '\'' + + ", format='" + format + '\'' + + '}'; + } +} diff --git a/core/src/main/java/org/jsmart/zerocode/core/kafka/helper/KafkaConsumerHelper.java b/core/src/main/java/org/jsmart/zerocode/core/kafka/helper/KafkaConsumerHelper.java index 1e5a76ae2..0ea2d4424 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/kafka/helper/KafkaConsumerHelper.java +++ b/core/src/main/java/org/jsmart/zerocode/core/kafka/helper/KafkaConsumerHelper.java @@ -1,9 +1,32 @@ package org.jsmart.zerocode.core.kafka.helper; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.io.Resources; +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Message; +import com.google.protobuf.MessageOrBuilder; +import com.google.protobuf.util.JsonFormat; +import com.jayway.jsonpath.JsonPath; import static java.lang.Integer.parseInt; import static java.lang.Long.parseLong; import static java.util.Optional.ofNullable; import static org.apache.commons.lang3.StringUtils.isEmpty; +import static org.apache.commons.lang3.StringUtils.isNumeric; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndTimestamp; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.Headers; +import org.jsmart.zerocode.core.di.provider.GsonSerDeProvider; +import org.jsmart.zerocode.core.di.provider.ObjectMapperProvider; +import org.jsmart.zerocode.core.kafka.KafkaConstants; import static org.jsmart.zerocode.core.kafka.KafkaConstants.AVRO; import static org.jsmart.zerocode.core.kafka.KafkaConstants.DEFAULT_POLLING_TIME_MILLI_SEC; import static org.jsmart.zerocode.core.kafka.KafkaConstants.JSON; @@ -11,58 +34,44 @@ import static org.jsmart.zerocode.core.kafka.KafkaConstants.PROTO; import static org.jsmart.zerocode.core.kafka.KafkaConstants.RAW; import static org.jsmart.zerocode.core.kafka.common.KafkaCommonUtils.resolveValuePlaceHolders; +import org.jsmart.zerocode.core.kafka.consume.ConsumerLocalConfigs; +import org.jsmart.zerocode.core.kafka.consume.ConsumerLocalConfigsWrap; +import org.jsmart.zerocode.core.kafka.consume.SeekTimestamp; +import org.jsmart.zerocode.core.kafka.receive.ConsumerCommonConfigs; +import org.jsmart.zerocode.core.kafka.receive.message.ConsumerJsonRecord; +import org.jsmart.zerocode.core.kafka.receive.message.ConsumerJsonRecords; +import org.jsmart.zerocode.core.kafka.receive.message.ConsumerRawRecords; import static org.jsmart.zerocode.core.utils.SmartUtils.prettyPrintJson; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.time.Duration; import java.time.temporal.ChronoUnit; import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Properties; import java.util.Set; - -import com.jayway.jsonpath.JsonPath; -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.Headers; -import org.jsmart.zerocode.core.di.provider.GsonSerDeProvider; -import org.jsmart.zerocode.core.di.provider.ObjectMapperProvider; -import org.jsmart.zerocode.core.kafka.KafkaConstants; -import org.jsmart.zerocode.core.kafka.consume.ConsumerLocalConfigs; -import org.jsmart.zerocode.core.kafka.consume.ConsumerLocalConfigsWrap; -import org.jsmart.zerocode.core.kafka.receive.ConsumerCommonConfigs; -import org.jsmart.zerocode.core.kafka.receive.message.ConsumerJsonRecord; -import org.jsmart.zerocode.core.kafka.receive.message.ConsumerJsonRecords; -import org.jsmart.zerocode.core.kafka.receive.message.ConsumerRawRecords; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.io.Resources; -import com.google.gson.Gson; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Message; -import com.google.protobuf.MessageOrBuilder; -import com.google.protobuf.util.JsonFormat; +import java.util.function.Function; +import java.util.stream.Collectors; public class KafkaConsumerHelper { + public static final String CONSUMER = "CONSUMER"; private static final Logger LOGGER = LoggerFactory.getLogger(KafkaConsumerHelper.class); private static final Gson gson = new GsonSerDeProvider().get(); private static final ObjectMapper objectMapper = new ObjectMapperProvider().get(); - public static final String CONSUMER = "CONSUMER"; public static Map consumerCacheByTopicMap = new HashMap<>(); public static Consumer createConsumer(String bootStrapServers, String consumerPropertyFile, String topic, Boolean consumerToBeCached) { @@ -81,9 +90,9 @@ public static Consumer createConsumer(String bootStrapServers, String consumerPr final Consumer consumer = new KafkaConsumer(properties); consumer.subscribe(Collections.singletonList(topic)); - if(consumerToBeCached == true){ + if (consumerToBeCached) { consumerCacheByTopicMap.forEach((xTopic, xConsumer) -> { - if(!xTopic.equals(topic)){ + if (!xTopic.equals(topic)) { // close the earlier consumer if in the same group for safety. // (even if not in the same group, closing it anyway will not do any harm) // Otherwise rebalance will fail while rejoining/joining the same group for a new consumer @@ -111,18 +120,18 @@ public static Consumer createConsumer(String bootStrapServers, String consumerPr public static ConsumerRecords initialPollWaitingForConsumerGroupJoin(Consumer consumer, ConsumerLocalConfigs effectiveLocalConfigs) { - for (int run = 0; run < 50; run++) { - if (!consumer.assignment().isEmpty()) { - LOGGER.debug("==> WaitingForConsumerGroupJoin - Partition now assigned. No records yet consumed"); - return new ConsumerRecords(new HashMap()); - } - LOGGER.debug("==> WaitingForConsumerGroupJoin - Partition not assigned. Polling once"); - ConsumerRecords records = consumer.poll(Duration.of(getPollTime(effectiveLocalConfigs), ChronoUnit.MILLIS)); - LOGGER.debug("==> WaitingForConsumerGroupJoin - polled records length={}", records.count()); - if (!records.isEmpty()) { - return records; - } + for (int run = 0; run < 50; run++) { + if (!consumer.assignment().isEmpty()) { + LOGGER.debug("==> WaitingForConsumerGroupJoin - Partition now assigned. No records yet consumed"); + return new ConsumerRecords(new HashMap()); + } + LOGGER.debug("==> WaitingForConsumerGroupJoin - Partition not assigned. Polling once"); + ConsumerRecords records = consumer.poll(Duration.of(getPollTime(effectiveLocalConfigs), ChronoUnit.MILLIS)); + LOGGER.debug("==> WaitingForConsumerGroupJoin - polled records length={}", records.count()); + if (!records.isEmpty()) { + return records; } + } throw new RuntimeException("\n********* Kafka Consumer unable to join in time - try increasing consumer polling time setting *********\n"); } @@ -135,6 +144,31 @@ public static void validateLocalConfigs(ConsumerLocalConfigs localConfigs) { validateCommitFlags(localCommitSync, localCommitAsync); validateSeekConfig(localConfigs); + validateSeekToTimestamp(localConfigs); + } + } + + private static void validateSeekToTimestamp(ConsumerLocalConfigs localConfigs) { + String seekToTimestamp = localConfigs.getSeekEpoch(); + if (isEmpty(seekToTimestamp)) { + if (isNumeric(seekToTimestamp) && (Long.parseLong(seekToTimestamp) > System.currentTimeMillis() || Long.parseLong(seekToTimestamp) < 0L)) { + throw new RuntimeException("\n------> 'seekEpoch' is not a valid epoch/Unix timestamp"); + } + if (!isEmpty(localConfigs.getSeek()) && Objects.nonNull(localConfigs.getSeekTimestamp())) { + throw new RuntimeException("Only one of 'seek', 'seekEpoch' and 'seekTimestamp' should be provided, but not both. Please fix and rerun"); + } + } + if (Objects.nonNull(localConfigs.getSeekTimestamp())) { + DateFormat dateFormat = new SimpleDateFormat(localConfigs.getSeekTimestamp().getFormat()); + try { + Date date = dateFormat.parse(localConfigs.getSeekTimestamp().getTimestamp()); + long epochMillis = date.toInstant().toEpochMilli(); + if (epochMillis > System.currentTimeMillis() || epochMillis < 0L) { + throw new RuntimeException("\n------> 'seekTimestamp' is not a valid epoch/Unix timestamp " + epochMillis); + } + } catch (ParseException e) { + throw new RuntimeException("Timestamp and format provided in 'seekTimestamp' cannot be parsed ", e); + } } } @@ -163,7 +197,9 @@ public static ConsumerLocalConfigs createEffective(ConsumerCommonConfigs consume consumerCommon.getPollingTime(), consumerCommon.getCacheByTopic(), consumerCommon.getFilterByJsonPath(), - consumerCommon.getSeek()); + null, + null, + null); } // Handle recordType @@ -188,9 +224,6 @@ public static ConsumerLocalConfigs createEffective(ConsumerCommonConfigs consume // Handle pollingTime String filterByJsonPath = ofNullable(consumerLocal.getFilterByJsonPath()).orElse(consumerCommon.getFilterByJsonPath()); - // Handle pollingTime - String effectiveSeek = ofNullable(consumerLocal.getSeek()).orElse(consumerCommon.getSeek()); - // Handle consumerCache by topic Boolean effectiveConsumerCacheByTopic = ofNullable(consumerLocal.getCacheByTopic()) .orElse(consumerCommon.getCacheByTopic()); @@ -222,7 +255,9 @@ public static ConsumerLocalConfigs createEffective(ConsumerCommonConfigs consume effectivePollingTime, effectiveConsumerCacheByTopic, filterByJsonPath, - effectiveSeek); + consumerLocal.getSeek(), + consumerLocal.getSeekEpoch(), + consumerLocal.getSeekTimestamp()); } public static ConsumerLocalConfigs readConsumerLocalTestProperties(String requestJsonWithConfigWrapped) { @@ -264,7 +299,7 @@ public static void readJson(List jsonRecords, Object key = thisRecord.key(); Object valueObj = thisRecord.value(); Headers headers = thisRecord.headers(); - String keyStr = thisRecord.key() != null ? thisRecord.key().toString() : ""; + String keyStr = thisRecord.key() != null ? thisRecord.key().toString() : ""; String valueStr = consumerLocalConfig != null && KafkaConstants.PROTO.equalsIgnoreCase(consumerLocalConfig.getRecordType()) ? convertProtobufToJson(thisRecord, consumerLocalConfig) : valueObj.toString(); LOGGER.debug("\nRecord Key - {} , Record value - {}, Record partition - {}, Record offset - {}, Headers - {}", key, valueStr, thisRecord.partition(), thisRecord.offset(), headers); @@ -289,7 +324,7 @@ private static String convertProtobufToJson(ConsumerRecord thisRecord, ConsumerL throw new IllegalArgumentException( "[protoClassType] is required consumer config for recordType PROTO."); } - MessageOrBuilder builderOrMessage = (MessageOrBuilder) createMessageOrBuilder( + MessageOrBuilder builderOrMessage = createMessageOrBuilder( consumerLocalConfig.getProtoClassType(), (byte[]) thisRecord.value()); try { return JsonFormat.printer().includingDefaultValueFields().preservingProtoFieldNames().print(builderOrMessage); @@ -301,10 +336,10 @@ private static String convertProtobufToJson(ConsumerRecord thisRecord, ConsumerL private static MessageOrBuilder createMessageOrBuilder(String messageClass, byte[] value) { try { Class msgClass = (Class) Class.forName(messageClass); - Method method = msgClass.getMethod("parseFrom", new Class[]{byte[].class}); + Method method = msgClass.getMethod("parseFrom", byte[].class); return (MessageOrBuilder) method.invoke(null, value); } catch (IllegalAccessException | ClassNotFoundException | NoSuchMethodException | SecurityException - | IllegalArgumentException | InvocationTargetException e) { + | IllegalArgumentException | InvocationTargetException e) { throw new IllegalArgumentException(e); } @@ -316,7 +351,7 @@ public static String prepareResult(ConsumerLocalConfigs testConfigs, String result; - if (testConfigs != null && testConfigs.getShowRecordsConsumed() == false) { + if (testConfigs != null && !testConfigs.getShowRecordsConsumed()) { int size = jsonRecords.size(); result = prettyPrintJson(gson.toJson(new ConsumerRawRecords(size == 0 ? rawRecords.size() : size))); @@ -326,7 +361,7 @@ public static String prepareResult(ConsumerLocalConfigs testConfigs, } else if (testConfigs != null && (JSON.equals(testConfigs.getRecordType()) || PROTO.equalsIgnoreCase(testConfigs.getRecordType()) || AVRO.equalsIgnoreCase(testConfigs.getRecordType()))) { result = prettyPrintJson(objectMapper.writeValueAsString(new ConsumerJsonRecords(jsonRecords))); - }else { + } else { result = "{\"error\" : \"recordType Undecided, Please chose recordType as JSON or RAW\"}"; } @@ -363,10 +398,10 @@ public static void handleCommitSyncAsync(Consumer consumer, effectiveCommitAsync = localCommitAsync; } - if (effectiveCommitSync != null && effectiveCommitSync == true) { + if (effectiveCommitSync != null && effectiveCommitSync) { consumer.commitSync(); - } else if (effectiveCommitAsync != null && effectiveCommitAsync == true) { + } else if (effectiveCommitAsync != null && effectiveCommitAsync) { consumer.commitAsync(); } else { @@ -378,32 +413,78 @@ public static void handleCommitSyncAsync(Consumer consumer, // -------------------------------------------------------- } - public static void handleSeekOffset(ConsumerLocalConfigs effectiveLocal, Consumer consumer) { + public static void handleSeek(ConsumerLocalConfigs effectiveLocal, Consumer consumer, String topicName) { String seek = effectiveLocal.getSeek(); if (!isEmpty(seek)) { - String[] seekParts = effectiveLocal.getSeekTopicPartitionOffset(); - String topic = seekParts[0]; - int partition = parseInt(seekParts[1]); - long offset = parseLong(seekParts[2]); + handleSeekByOffset(effectiveLocal, consumer); + } else if (!isEmpty(effectiveLocal.getSeekEpoch())) { + handleSeekByEpoch(Long.parseLong(effectiveLocal.getSeekEpoch()), consumer, topicName); + } else if (Objects.nonNull(effectiveLocal.getSeekTimestamp())) { + handleSeekByTimestamp(effectiveLocal.getSeekTimestamp(), consumer, topicName); + } + } + + private static void handleSeekByTimestamp(SeekTimestamp seekTimestamp, Consumer consumer, String topicName) { + if (Objects.nonNull(seekTimestamp)) { + DateFormat dateFormat = new SimpleDateFormat(seekTimestamp.getFormat()); + Date date = null; + try { + date = dateFormat.parse(seekTimestamp.getTimestamp()); + } catch (ParseException e) { + throw new RuntimeException("Could not parse timestamp", e); + } + handleSeekByEpoch(date.toInstant().toEpochMilli(), consumer, topicName); + } + } + + private static void handleSeekByEpoch(Long epoch, Consumer consumer, String topicName) { + if (Objects.nonNull(epoch)) { + List partitionInfos = consumer.partitionsFor(topicName); - TopicPartition topicPartition = new TopicPartition(topic, partition); - Set topicPartitions = new HashSet<>(); - topicPartitions.add(topicPartition); + //fetch partitions on topic + List topicPartitions = partitionInfos.stream() + .map(info -> new TopicPartition(info.topic(), info.partition())) + .collect(Collectors.toList()); + //fetch offsets for each partition-timestamp pair + Map topicPartitionTimestampMap = topicPartitions.stream() + .collect(Collectors.toMap(Function.identity(), ignore -> epoch)); + Map topicPartitionOffsetAndTimestampMap = consumer.offsetsForTimes(topicPartitionTimestampMap); + + //assign to fetched partitions consumer.unsubscribe(); - consumer.assign(topicPartitions); + consumer.assign(topicPartitionOffsetAndTimestampMap.keySet()); - if (offset <= -1) { - consumer.seekToEnd(topicPartitions); - consumer.seek(topicPartition, consumer.position(topicPartition) + offset); - } else { - consumer.seek(topicPartition, offset); + //seek to fetched offsets for partitions + for (Map.Entry topicOffsetEntry : topicPartitionOffsetAndTimestampMap.entrySet()) { + consumer.seek(topicOffsetEntry.getKey(), topicOffsetEntry.getValue().offset()); } } } + private static void handleSeekByOffset(ConsumerLocalConfigs effectiveLocal, Consumer consumer) { + String[] seekParts = effectiveLocal.getSeekTopicPartitionOffset(); + String topic = seekParts[0]; + int partition = parseInt(seekParts[1]); + long offset = parseLong(seekParts[2]); + + TopicPartition topicPartition = new TopicPartition(topic, partition); + Set topicPartitions = new HashSet<>(); + topicPartitions.add(topicPartition); + + consumer.unsubscribe(); + consumer.assign(topicPartitions); + + if (offset <= -1) { + consumer.seekToEnd(topicPartitions); + consumer.seek(topicPartition, consumer.position(topicPartition) + offset); + } else { + consumer.seek(topicPartition, offset); + } + } + private static void validateCommitFlags(Boolean commitSync, Boolean commitAsync) { - if ((commitSync != null && commitAsync != null) && commitSync == true && commitAsync == true) { + if ((commitSync != null && commitAsync != null) && commitSync && commitAsync) { throw new RuntimeException("\n********* Both commitSync and commitAsync can not be true *********\n"); } } @@ -419,7 +500,7 @@ private static void validateSeekConfig(ConsumerLocalConfigs localConfigs) { } private static Consumer getCachedConsumer(String topic, Boolean consumerToBeCached) { - if(consumerToBeCached){ + if (consumerToBeCached) { return consumerCacheByTopicMap.get(topic); } return null; diff --git a/core/src/main/java/org/jsmart/zerocode/core/kafka/receive/ConsumerCommonConfigs.java b/core/src/main/java/org/jsmart/zerocode/core/kafka/receive/ConsumerCommonConfigs.java index 2b6640d4e..d34fe087c 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/kafka/receive/ConsumerCommonConfigs.java +++ b/core/src/main/java/org/jsmart/zerocode/core/kafka/receive/ConsumerCommonConfigs.java @@ -49,11 +49,6 @@ public class ConsumerCommonConfigs { @Named("consumer.filterByJsonPath") private String filterByJsonPath; - // TODO- Remove this from Global properties, as it doesn't make sense - @Inject(optional = true) - @Named("consumer.seek") - private String seek; - public ConsumerCommonConfigs() { } @@ -66,8 +61,7 @@ public ConsumerCommonConfigs(Boolean commitSync, Integer maxNoOfRetryPollsOrTimeouts, Long pollingTime, Boolean cacheByTopic, - String filterByJsonPath, - String seek + String filterByJsonPath ) { this.commitSync = commitSync; @@ -80,7 +74,6 @@ public ConsumerCommonConfigs(Boolean commitSync, this.pollingTime = pollingTime; this.cacheByTopic = cacheByTopic; this.filterByJsonPath = filterByJsonPath; - this.seek = seek; } public ConsumerCommonConfigs(Boolean commitSync, @@ -91,8 +84,7 @@ public ConsumerCommonConfigs(Boolean commitSync, Integer maxNoOfRetryPollsOrTimeouts, Long pollingTime, Boolean cacheByTopic, - String filterByJsonPath, - String seek + String filterByJsonPath ) { this(commitSync, @@ -104,8 +96,7 @@ public ConsumerCommonConfigs(Boolean commitSync, maxNoOfRetryPollsOrTimeouts, pollingTime, cacheByTopic, - filterByJsonPath, - seek); + filterByJsonPath); } public Boolean getCommitSync() { @@ -136,9 +127,6 @@ public String getRecordType() { return recordType; } - public String getSeek() { - return seek; - } public String getProtoClassType() { return protoClassType; } @@ -165,7 +153,6 @@ public String toString() { ", pollingTime=" + pollingTime + ", cacheByTopic=" + cacheByTopic + ", filterByJsonPath=" + filterByJsonPath + - ", seek=" + seek + '}'; } } diff --git a/core/src/main/java/org/jsmart/zerocode/core/kafka/receive/KafkaReceiver.java b/core/src/main/java/org/jsmart/zerocode/core/kafka/receive/KafkaReceiver.java index 37f893161..327250242 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/kafka/receive/KafkaReceiver.java +++ b/core/src/main/java/org/jsmart/zerocode/core/kafka/receive/KafkaReceiver.java @@ -25,7 +25,7 @@ import static org.jsmart.zerocode.core.kafka.helper.KafkaConsumerHelper.getMaxTimeOuts; import static org.jsmart.zerocode.core.kafka.helper.KafkaConsumerHelper.getPollTime; import static org.jsmart.zerocode.core.kafka.helper.KafkaConsumerHelper.handleCommitSyncAsync; -import static org.jsmart.zerocode.core.kafka.helper.KafkaConsumerHelper.handleSeekOffset; +import static org.jsmart.zerocode.core.kafka.helper.KafkaConsumerHelper.handleSeek; import static org.jsmart.zerocode.core.kafka.helper.KafkaConsumerHelper.initialPollWaitingForConsumerGroupJoin; import static org.jsmart.zerocode.core.kafka.helper.KafkaConsumerHelper.prepareResult; import static org.jsmart.zerocode.core.kafka.helper.KafkaConsumerHelper.readConsumerLocalTestProperties; @@ -62,7 +62,7 @@ public String receive(String kafkaServers, String topicName, String requestJsonW int noOfTimeOuts = 0; - handleSeekOffset(effectiveLocal, consumer); + handleSeek(effectiveLocal, consumer, topicName); LOGGER.debug("initial polling to trigger ConsumerGroupJoin"); diff --git a/core/src/main/java/org/jsmart/zerocode/core/logbuilder/ZerocodeCorrelationshipLogger.java b/core/src/main/java/org/jsmart/zerocode/core/logbuilder/ZerocodeCorrelationshipLogger.java index 974ab29fa..53c807b7c 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/logbuilder/ZerocodeCorrelationshipLogger.java +++ b/core/src/main/java/org/jsmart/zerocode/core/logbuilder/ZerocodeCorrelationshipLogger.java @@ -16,6 +16,7 @@ import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.RESULT_FAIL; import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.RESULT_PASS; import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.TEST_STEP_CORRELATION_ID; +import static org.jsmart.zerocode.core.utils.TokenUtils.getMasksReplaced; public class ZerocodeCorrelationshipLogger { private static final String DISPLAY_DEMARCATION_ = "\n--------- " + TEST_STEP_CORRELATION_ID + " %s ---------"; @@ -139,11 +140,12 @@ public void print() { buildResponseDelay(); String customLog = responseLogBuilder.getCustomLog(); + String assertionsWithMaskRemoved = getMasksReplaced(responseLogBuilder.getAssertion()); logger.warn(format("%s %s \n*Response delay:%s milli-secs \n%s \n%s \n-done-\n", requestLogBuilder.toString(), responseLogBuilder.toString(), responseDelay, - "---------> Expected Response: <----------\n" + responseLogBuilder.getAssertion(), + "---------> Expected Response: <----------\n" + assertionsWithMaskRemoved, customLog == null ? "" : "---------> Custom Log: <----------\n" +customLog ) ); diff --git a/core/src/main/java/org/jsmart/zerocode/core/report/ZeroCodeReportGeneratorImpl.java b/core/src/main/java/org/jsmart/zerocode/core/report/ZeroCodeReportGeneratorImpl.java index db5a3d4af..e80fb0760 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/report/ZeroCodeReportGeneratorImpl.java +++ b/core/src/main/java/org/jsmart/zerocode/core/report/ZeroCodeReportGeneratorImpl.java @@ -12,12 +12,7 @@ import com.fasterxml.jackson.dataformat.csv.CsvSchema; import com.google.inject.Inject; import com.google.inject.name.Named; -import org.apache.commons.lang.StringUtils; -import org.jsmart.zerocode.core.domain.builders.ExtentReportsFactory; -import org.jsmart.zerocode.core.domain.builders.HighChartColumnHtmlBuilder; -import org.jsmart.zerocode.core.domain.builders.ZeroCodeChartKeyValueArrayBuilder; -import org.jsmart.zerocode.core.domain.builders.ZeroCodeChartKeyValueBuilder; -import org.jsmart.zerocode.core.domain.builders.ZeroCodeCsvReportBuilder; +import org.jsmart.zerocode.core.domain.builders.*; import org.jsmart.zerocode.core.domain.reports.ZeroCodeExecResult; import org.jsmart.zerocode.core.domain.reports.ZeroCodeReport; import org.jsmart.zerocode.core.domain.reports.ZeroCodeReportStep; @@ -35,21 +30,8 @@ import static java.util.Collections.emptyList; import static java.util.Optional.ofNullable; -import static org.apache.commons.lang.StringUtils.substringBetween; -import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.AUTHOR_MARKER_NEW; -import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.CATEGORY_MARKER; +import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.*; import static org.jsmart.zerocode.core.domain.builders.ExtentReportsFactory.getReportName; -import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.ANONYMOUS_CAT; -import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.AUTHOR_MARKER_OLD; -import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.DEFAULT_REGRESSION_CATEGORY; -import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.HIGH_CHART_HTML_FILE_NAME; -import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.LINK_LABEL_NAME; -import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.RESULT_PASS; -import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.TARGET_FILE_NAME; -import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.TARGET_FULL_REPORT_CSV_FILE_NAME; -import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.TARGET_FULL_REPORT_DIR; -import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.TARGET_REPORT_DIR; -import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.TEST_STEP_CORRELATION_ID; public class ZeroCodeReportGeneratorImpl implements ZeroCodeReportGenerator { private static final Logger LOGGER = LoggerFactory.getLogger(ZeroCodeReportGeneratorImpl.class); @@ -115,10 +97,21 @@ public void generateExtentReport() { thisReport.getResults().forEach(thisScenario -> { ExtentTest test = extentReports.createTest(thisScenario.getScenarioName()); + + // Assign Category test.assignCategory(DEFAULT_REGRESSION_CATEGORY); //Super set - test.assignCategory(optionalCategory(thisScenario.getScenarioName())); //Sub sets + String[] hashTagsArray = optionalCategories(thisScenario.getScenarioName()).toArray(new String[0]); + if(hashTagsArray.length > 0) { + test.assignCategory(hashTagsArray); //Sub categories + } + + // Assign Authors + test.assignAuthor(DEFAULT_REGRESSION_AUTHOR); //Super set + String[] authorsArray = optionalAuthors(thisScenario.getScenarioName()).toArray(new String[0]); + if(authorsArray.length > 0) { + test.assignAuthor(authorsArray); //Sub authors + } - test.assignAuthor(optionalAuthor(thisScenario.getScenarioName())); List thisScenarioUniqueSteps = getUniqueSteps(thisScenario.getSteps()); thisScenarioUniqueSteps.forEach(thisStep -> { test.getModel().setStartTime(utilDateOf(thisStep.getRequestTimeStamp())); @@ -166,42 +159,33 @@ public void linkToSpikeChartIfEnabled() { /** * @param scenarioName String containing a name of an author - * @return author of the test scenario + * @return authors of the test scenario */ - protected String optionalAuthor(String scenarioName) { - String authorName = deriveName(scenarioName, AUTHOR_MARKER_OLD); - authorName = ANONYMOUS_CAT.equals(authorName) ? deriveName(scenarioName, AUTHOR_MARKER_NEW) : authorName; - return authorName; + protected List optionalAuthors(String scenarioName) { + return deriveNames(scenarioName, AUTHOR_MARKER_NEW); } /** - * @param scenarioName String containing hash tags of a category - * @return category of the test scenario + * @param scenarioName String containing hashtags of a category + * @return hashtags aka categories of the test scenario */ - protected String optionalCategory(String scenarioName) { - return deriveName(scenarioName, CATEGORY_MARKER); + protected List optionalCategories(String scenarioName) { + return deriveNames(scenarioName, CATEGORY_MARKER); } - private String deriveName(String scenarioName, String marker) { - String authorName = substringBetween(scenarioName, marker, marker); - - if (authorName == null) { - authorName = substringBetween(scenarioName, marker, ","); - } - - if (authorName == null) { - authorName = substringBetween(scenarioName, marker, " "); - } - - if (authorName == null) { - authorName = scenarioName.substring(scenarioName.lastIndexOf(marker) + marker.length()); - } - - if (scenarioName.lastIndexOf(marker) == -1 || StringUtils.isEmpty(authorName)) { - authorName = ANONYMOUS_CAT; + private List deriveNames(String scenarioName, String marker) { + List nameList = new ArrayList<>(); + for(String thisName : scenarioName.trim().split(" ")){ + if(thisName.startsWith(marker) && !thisName.startsWith(AUTHOR_MARKER_OLD)){ + nameList.add(thisName); + } + // Depreciated, but still supports. Remove this via a new ticket + if(thisName.startsWith(AUTHOR_MARKER_OLD)){ + nameList.add(thisName); + } } + return nameList; - return authorName; } protected String onlyScenarioName(String scenarioName) { diff --git a/core/src/main/java/org/jsmart/zerocode/core/runner/ZeroCodeMultiStepsScenarioRunnerImpl.java b/core/src/main/java/org/jsmart/zerocode/core/runner/ZeroCodeMultiStepsScenarioRunnerImpl.java index 4f7e7d05b..3bef7ba2d 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/runner/ZeroCodeMultiStepsScenarioRunnerImpl.java +++ b/core/src/main/java/org/jsmart/zerocode/core/runner/ZeroCodeMultiStepsScenarioRunnerImpl.java @@ -181,6 +181,8 @@ private Boolean executeRetryWithSteps(RunNotifier notifier, ScenarioExecutionState scenarioExecutionState, ScenarioSpec scenario, Step thisStep) { thisStep = extFileProcessor.resolveExtJsonFile(thisStep); + thisStep = zeroCodeAssertionsProcessor.resolveJsonContent(thisStep, scenarioExecutionState); + List thisSteps = extFileProcessor.createFromStepFile(thisStep, thisStep.getId()); if(null == thisSteps || thisSteps.isEmpty()) thisSteps.add(thisStep); Boolean wasExecSuccess = null; @@ -412,6 +414,13 @@ private String executeApi(String logPrefixRelationshipId, // -------------------------------- url = zeroCodeAssertionsProcessor.resolveStringJson(url, scenarioExecutionState.getResolvedScenarioState()); + // ------------------------------------------------ + // 1) Removed the MASKED wrapper for API execution (For logging) + // 2) Replace the MASKED field with masked content (For API executions) + // ------------------------------------------------ + String resolvedRequestJsonMaskRemoved = zeroCodeAssertionsProcessor.fieldMasksRemoved(resolvedRequestJson); + String resolvedRequestJsonMaskApplied = zeroCodeAssertionsProcessor.fieldMasksApplied(resolvedRequestJson); + final LocalDateTime requestTimeStamp = LocalDateTime.now(); String executionResult; @@ -426,9 +435,9 @@ private String executeApi(String logPrefixRelationshipId, .url(url) .method(operationName) .id(stepId) - .request(prettyPrintJson(resolvedRequestJson)); + .request(prettyPrintJson(resolvedRequestJsonMaskApplied)); - executionResult = apiExecutor.executeHttpApi(url, operationName, resolvedRequestJson); + executionResult = apiExecutor.executeHttpApi(url, operationName, resolvedRequestJsonMaskRemoved); break; case JAVA_CALL: @@ -439,10 +448,10 @@ private String executeApi(String logPrefixRelationshipId, .id(stepId) .url(url) .method(operationName) - .request(prettyPrintJson(resolvedRequestJson)); + .request(prettyPrintJson(resolvedRequestJsonMaskApplied)); url = apiTypeUtils.getQualifiedJavaApi(url); - executionResult = apiExecutor.executeJavaOperation(url, operationName, resolvedRequestJson); + executionResult = apiExecutor.executeJavaOperation(url, operationName, resolvedRequestJsonMaskRemoved); break; case KAFKA_CALL: @@ -457,10 +466,10 @@ private String executeApi(String logPrefixRelationshipId, .url(url) .method(operationName.toUpperCase()) .id(stepId) - .request(prettyPrintJson(resolvedRequestJson)); + .request(prettyPrintJson(resolvedRequestJsonMaskApplied)); String topicName = url.substring(KAFKA_TOPIC.length()); - executionResult = apiExecutor.executeKafkaService(kafkaServers, topicName, operationName, resolvedRequestJson, scenarioExecutionState); + executionResult = apiExecutor.executeKafkaService(kafkaServers, topicName, operationName, resolvedRequestJsonMaskRemoved, scenarioExecutionState); break; case NONE: @@ -471,14 +480,14 @@ private String executeApi(String logPrefixRelationshipId, .id(stepId) .url(url) .method(operationName) - .request(prettyPrintJson(resolvedRequestJson)); + .request(prettyPrintJson(resolvedRequestJsonMaskApplied)); - executionResult = prettyPrintJson(resolvedRequestJson); + executionResult = prettyPrintJson(resolvedRequestJsonMaskApplied); break; default: throw new RuntimeException("Oops! API Type Undecided. If it is intentional, " + - "then keep the value as empty to receive the request in the response"); + "then keep the value as empty to receive the request as response"); } return executionResult; @@ -532,6 +541,8 @@ private int deriveScenarioLoopTimes(ScenarioSpec scenario) { private List compareStepResults(Step thisStep, String actualResult, String expectedResult, String resolvedScenarioState) { List failureResults = new ArrayList<>(); + expectedResult = zeroCodeAssertionsProcessor.fieldMasksRemoved(expectedResult); + // -------------------- // Validators (pyrest) // -------------------- diff --git a/core/src/main/java/org/jsmart/zerocode/core/utils/SmartUtils.java b/core/src/main/java/org/jsmart/zerocode/core/utils/SmartUtils.java index 528659a26..819746ff0 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/utils/SmartUtils.java +++ b/core/src/main/java/org/jsmart/zerocode/core/utils/SmartUtils.java @@ -27,10 +27,10 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.commons.lang.text.StrSubstitutor; import org.jsmart.zerocode.core.di.provider.ObjectMapperProvider; import org.jsmart.zerocode.core.domain.ScenarioSpec; +import org.jsmart.zerocode.core.domain.Step; import org.jsmart.zerocode.core.engine.assertion.FieldAssertionMatcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,7 +39,10 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.jsmart.zerocode.core.engine.assertion.FieldAssertionMatcher.aMatchingMessage; import static org.jsmart.zerocode.core.engine.assertion.FieldAssertionMatcher.aNotMatchingMessage; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.JSON_PAYLOAD_FILE; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.YAML_PAYLOAD_FILE; import static org.jsmart.zerocode.core.utils.PropertiesProviderUtils.loadAbsoluteProperties; +import static org.jsmart.zerocode.core.utils.TokenUtils.getTestCaseTokens; import static org.skyscreamer.jsonassert.JSONAssert.assertEquals; import static org.skyscreamer.jsonassert.JSONCompareMode.STRICT; @@ -240,4 +243,42 @@ public static String getEnvPropertyValue(String envPropertyKey) { return System.getenv(envPropertyKey); } } + + /** + * + * @param thisStep --> Currently executing step + * @param tokenString --> JSON_PAYLAOD_FILE or JSON_CONTENT + * @return if there is a match for the token, then the json traversal will happen + * @throws JsonProcessingException + */ + public static boolean checkDigNeeded(ObjectMapper mapper, Step thisStep, String tokenString) throws JsonProcessingException { + String stepJson = mapper.writeValueAsString(thisStep); + List allTokens = getTestCaseTokens(stepJson); + + return allTokens.toString().contains(tokenString); + } + public static boolean checkDigNeeded(ObjectMapper mapper, Step thisStep, String tokenString, String alternateTokenString) throws JsonProcessingException { + String stepJson = mapper.writeValueAsString(thisStep); + List allTokens = getTestCaseTokens(stepJson); + + return allTokens.toString().contains(tokenString) || allTokens.toString().contains(alternateTokenString); + } + + /** + * Retrieves the first token from the given value string that matches the format "${token}". + * Ph = Placeholder (e.g. ${JSON.FILE:unit_test_files/filebody_unit_test/common/common_content.json} ) + * + * @param valueString The string from which to extract the jsonfile path + * @return The extracted token, or null if no token is found + */ + public static String getJsonFilePhToken(String valueString) { + if (valueString != null) { + List allTokens = getTestCaseTokens(valueString); + if (allTokens != null && !allTokens.isEmpty()) { + return allTokens.get(0); + } + } + return null; + } + } diff --git a/core/src/main/java/org/jsmart/zerocode/core/utils/TokenUtils.java b/core/src/main/java/org/jsmart/zerocode/core/utils/TokenUtils.java index e54a5036d..367c1ff42 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/utils/TokenUtils.java +++ b/core/src/main/java/org/jsmart/zerocode/core/utils/TokenUtils.java @@ -19,23 +19,7 @@ import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric; import static org.apache.commons.lang.StringEscapeUtils.escapeJava; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.ABS_PATH; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.GLOBAL_RANDOM_NUMBER; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.GQL_FILE; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.LOCALDATETIME_NOW; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.LOCALDATE_TODAY; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.RANDOM_NUMBER; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.RANDOM_NUMBER_FIXED; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.RANDOM_STRING_ALPHA; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.RANDOM_STRING_ALPHA_NUMERIC; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.RANDOM_UU_ID; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.RANDOM_UU_ID_FIXED; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.STATIC_ALPHABET; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.SYSTEM_ENV; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.SYSTEM_PROPERTY; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.XML_FILE; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.getKnownTokens; -import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.globalTokenCache; +import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.*; public class TokenUtils { @@ -162,6 +146,29 @@ public static List getTestCaseTokens(String aString) { return keyTokens; } + public static String getMasksReplaced(String aString) { + String regex = "\\$\\{MASKED:([^\\}]*)\\}"; + Matcher maskMatcher = Pattern.compile(regex).matcher(aString); + while(maskMatcher.find()) { + String foundMatch = maskMatcher.group(0); + aString = aString.replace(foundMatch, MASKED_STR); + } + + return aString; + } + + public static String getMasksRemoved(String aString) { + String regex = "\\$\\{MASKED:([^\\}]*)\\}"; + Matcher maskMatcher = Pattern.compile(regex).matcher(aString); + while(maskMatcher.find()) { + String foundFullMatch = maskMatcher.group(0); + String innerContent = maskMatcher.group(1); + aString = aString.replace(foundFullMatch, innerContent); + } + + return aString; + } + public static String createRandomAlphaString(int length) { return randomAlphabetic(length); } diff --git a/core/src/test/java/org/jsmart/zerocode/core/domain/ParameterizedTest.java b/core/src/test/java/org/jsmart/zerocode/core/domain/ParameterizedTest.java index a2be5436d..09569bd21 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/domain/ParameterizedTest.java +++ b/core/src/test/java/org/jsmart/zerocode/core/domain/ParameterizedTest.java @@ -10,7 +10,9 @@ import org.junit.Test; import org.junit.runner.RunWith; -import static org.hamcrest.CoreMatchers.hasItem; +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.*; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; @@ -58,4 +60,33 @@ public void testSerDe_csvSource() throws Exception { assertThat(parameterized.getCsvSource(), hasItem("11, 22, 400")); } + @Test + public void shouldReadCsvSourceFromCsvFile() throws IOException { + //given + String jsonDocumentAsString = + smartUtils.getJsonDocumentAsString("unit_test_files/engine_unit_test_jsons/08.1_parameterized_csv_source_from_file.json"); + + //when + Parameterized parameterized = mapper.readValue(jsonDocumentAsString, Parameterized.class); + + //then + assertThat(parameterized.getCsvSource(), hasItem("octocat,The Octocat,San Francisco,583231")); + assertThat(parameterized.getCsvSource(), hasItem("siddhagalaxy,Sidd,UK,33847730")); + } + + @Test + public void shouldReadCsvSourceFromCsvFileIgnoringHeader() throws IOException { + //given + String jsonDocumentAsString = + smartUtils.getJsonDocumentAsString("unit_test_files/engine_unit_test_jsons/08.2_parameterized_csv_source_from_file_containing_header.json"); + + //when + Parameterized parameterized = mapper.readValue(jsonDocumentAsString, Parameterized.class); + + //then + assertThat(parameterized.getCsvSource(), hasItem("octocat,The Octocat,San Francisco,583231")); + assertThat(parameterized.getCsvSource(), hasItem("siddhagalaxy,Sidd,UK,33847730")); + assertThat(parameterized.getCsvSource(), everyItem(not(is("user,name,city,userid"))));//assert header is ignored + } + } \ No newline at end of file diff --git a/core/src/test/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeAssertionsProcessorImplTest.java b/core/src/test/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeAssertionsProcessorImplTest.java index 531878755..8d569eb74 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeAssertionsProcessorImplTest.java +++ b/core/src/test/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeAssertionsProcessorImplTest.java @@ -1,26 +1,40 @@ package org.jsmart.zerocode.core.engine.preprocessor; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Guice; import com.google.inject.Injector; import com.jayway.jsonpath.JsonPath; + +import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; + +import org.apache.commons.lang.text.StrSubstitutor; +import org.hamcrest.core.Is; import org.jsmart.zerocode.core.di.main.ApplicationMainModule; import org.jsmart.zerocode.core.di.provider.ObjectMapperProvider; import org.jsmart.zerocode.core.domain.ScenarioSpec; +import org.jsmart.zerocode.core.domain.Step; import org.jsmart.zerocode.core.engine.assertion.FieldAssertionMatcher; import org.jsmart.zerocode.core.engine.assertion.JsonAsserter; +import org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens; import org.jsmart.zerocode.core.utils.SmartUtils; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import static com.jayway.jsonpath.JsonPath.read; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; +import static org.jsmart.zerocode.core.utils.SmartUtils.checkDigNeeded; +import static org.jsmart.zerocode.core.utils.SmartUtils.readJsonAsString; import static org.jsmart.zerocode.core.utils.TokenUtils.getTestCaseTokens; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertThat; @@ -1449,4 +1463,175 @@ public void testLeafValuesArray_badIndex() { expectedException.expect(IndexOutOfBoundsException.class); jsonPreProcessor.resolveLeafOnlyNodeValue(scenarioState, paramMap, thisPath); } + + @Test(expected = RuntimeException.class) + public void test_wrongJsonPathBy_JSONCONTENT_Exception() throws JsonProcessingException { + String jsonAsString = readJsonAsString("unit_test_files/json_content_unit_test/json_step_test_wrong_json_path.json"); + Map map = mapper.readValue(jsonAsString, new TypeReference>() {}); + + jsonPreProcessor.digReplaceContent(map, new ScenarioExecutionState()); + } + + @Test + public void test_JSONCONTENT_leafNode() throws IOException { + ScenarioExecutionState scenarioExecutionState = new ScenarioExecutionState(); + + final String step1 = createStepWithRequestAndResponse("create_emp", "\"body\" : {\n \"id\" : 39001,\n \"ldapId\" : \"emmanorton\"\n }\n}\n }"); + scenarioExecutionState.addStepState(step1); + + ScenarioSpec scenarioSpec = + smartUtils.scenarioFileToJava( + "unit_test_files/json_content_unit_test/json_step_test_json_content.json", ScenarioSpec.class); + Step thisStep = scenarioSpec.getSteps().get(1); + JsonNode stepNode = mapper.convertValue(thisStep, JsonNode.class); + Map map = mapper.readValue(stepNode.toString(), new TypeReference>() {}); + + jsonPreProcessor.digReplaceContent(map, scenarioExecutionState); + + String jsonResult = mapper.writeValueAsString(map); + + assertThat(JsonPath.read(jsonResult, "$.request.body.addressId"), is(39001)); + } + + + @Test + public void test_JSONCONTENT_stringArray() throws IOException { + ScenarioExecutionState scenarioExecutionState = new ScenarioExecutionState(); + + final String step1 = createStepWithRequestAndResponse("create_emp", "\"body\": {\"id\": 38001,\n \"names\": [\"test1\", \"test2\"]\n}"); + scenarioExecutionState.addStepState(step1); + + ScenarioSpec scenarioSpec = + smartUtils.scenarioFileToJava( + "unit_test_files/json_content_unit_test/json_step_test_json_content_array.json", ScenarioSpec.class); + Step thisStep = scenarioSpec.getSteps().get(1); + JsonNode stepNode = mapper.convertValue(thisStep, JsonNode.class); + Map map = mapper.readValue(stepNode.toString(), new TypeReference>() {}); + + jsonPreProcessor.digReplaceContent(map, scenarioExecutionState); + + String jsonResult = mapper.writeValueAsString(map); + + String result = "[\"test1\",\"test2\"]"; + + Object jsonPathValue = JsonPath.read(jsonResult, + "$.request.body.names"); + + Assert.assertEquals(result, jsonPathValue.toString()); + } + + @Test + public void test_JSONCONTENT_objectArray() throws IOException { + ScenarioExecutionState scenarioExecutionState = new ScenarioExecutionState(); + /** + * { + * "id": 38001, + * "allAddresses": [ + * { + * "type": "Home", + * "line1": "North Lon", + * "id": 43 + * }, + * { + * "type": "Office", + * "line1": "Central Lon" + * } + * ] + * } + */ + final String step1 = createStepWithRequestAndResponse("create_emp", + "\"body\": {\"id\": 38001, \"allAddresses\": [{\"type\": \"Home\", \"line1\": \"North Lon\", \"id\": 47}, {\"type\": \"Office\", \"line1\": \"Central Lon\"}]}"); + scenarioExecutionState.addStepState(step1); + + ScenarioSpec scenarioSpec = + smartUtils.scenarioFileToJava( + "unit_test_files/json_content_unit_test/json_step_test_json_content_objectarray.json", ScenarioSpec.class); + + Step thisStep = scenarioSpec.getSteps().get(1); + JsonNode stepNode = mapper.convertValue(thisStep, JsonNode.class); + Map map = mapper.readValue(stepNode.toString(), new TypeReference>() {}); + + jsonPreProcessor.digReplaceContent(map, scenarioExecutionState); + + String jsonResult = mapper.writeValueAsString(map); + + assertThat(JsonPath.read(jsonResult, "$.request.body.allAddresses[0].id"), is(47)); + assertThat(JsonPath.read(jsonResult, "$.request.body.allAddresses[0].type"), is("Home")); + assertThat(JsonPath.read(jsonResult, "$.request.body.allAddresses[1].type"), is("Office")); + assertThat(JsonPath.read(jsonResult, "$.request.body.allAddresses[0].line1"), is("North Lon")); + assertThat(JsonPath.read(jsonResult, "$.request.body.allAddresses[1].line1"), is("Central Lon")); + } + + @Test + public void test_JSONCONTENT_jsonBlock() throws IOException { + ScenarioExecutionState scenarioExecutionState = new ScenarioExecutionState(); + + final String step1 = createStepWithRequestAndResponse("create_emp", + "\"body\": {\n" + + " \"id\": 38001,\n" + + " \"address\": {\n" + + " \"type\": \"Home\",\n" + + " \"line1\": \"River Side\"\n" + + " }\n" + + "}"); + scenarioExecutionState.addStepState(step1); + + ScenarioSpec scenarioSpec = + smartUtils.scenarioFileToJava( + "unit_test_files/json_content_unit_test/json_step_test_json_content_block.json", ScenarioSpec.class); + Step thisStep = scenarioSpec.getSteps().get(1); + JsonNode stepNode = mapper.convertValue(thisStep, JsonNode.class); + Map map = mapper.readValue(stepNode.toString(), new TypeReference>() {}); + + jsonPreProcessor.digReplaceContent(map, scenarioExecutionState); + + String jsonResult = mapper.writeValueAsString(map); + + assertThat(JsonPath.read(jsonResult, "$.request.body.address.type"), is("Home")); + assertThat(JsonPath.read(jsonResult, "$.request.body.address.line1"), is("River Side")); + } + + @Test + public void test_NoJSONContentCheckDigNeeded() throws IOException { + String jsonAsString = readJsonAsString("unit_test_files/json_content_unit_test/json_step_no_json_content_test.json"); + Step step = mapper.readValue(jsonAsString, Step.class); + assertThat(checkDigNeeded(mapper, step, ZeroCodeValueTokens.JSON_CONTENT), Is.is(false)); + + + ScenarioSpec scenarioSpec = + smartUtils.scenarioFileToJava( + "unit_test_files/json_content_unit_test/json_step_test_json_content.json", ScenarioSpec.class); + step = scenarioSpec.getSteps().get(1); + assertThat(checkDigNeeded(mapper, step, ZeroCodeValueTokens.JSON_CONTENT), Is.is(true)); + } + + @Test + public void test_textNode() throws IOException { + String jsonAsString = readJsonAsString("unit_test_files/filebody_unit_test/json_step_text_request.json"); + Step step = mapper.readValue(jsonAsString, Step.class); + + jsonPreProcessor.resolveJsonContent(step, new ScenarioExecutionState()); + String resultJsonStep = mapper.writeValueAsString(step); + + assertThat(read(resultJsonStep, "$.request"), Is.is("I am a simple text")); + } + + + protected String createStepWithRequestAndResponse(String stepName, String body) { + Map parammap = new HashMap<>(); + + parammap.put("STEP.NAME", stepName); + parammap.put("STEP.REQUEST", "{\n" + + " \"customer\": {\n" + + " \"firstName\": \"FIRST_NAME\"\n" + + " }\n" + + "}"); + parammap.put("STEP.RESPONSE", "{\n" + + body + + "}"); + + StrSubstitutor sub = new StrSubstitutor(parammap); + + return sub.replace((new StepExecutionState()).getRequestResponseState()); + } } diff --git a/core/src/test/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeExternalFileProcessorImplTest.java b/core/src/test/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeExternalFileProcessorImplTest.java index 2cd3f7da2..e30837331 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeExternalFileProcessorImplTest.java +++ b/core/src/test/java/org/jsmart/zerocode/core/engine/preprocessor/ZeroCodeExternalFileProcessorImplTest.java @@ -4,7 +4,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.jsmart.zerocode.core.di.provider.ObjectMapperProvider; import org.jsmart.zerocode.core.domain.Step; +import org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens; import org.junit.Test; +import static org.jsmart.zerocode.core.utils.SmartUtils.checkDigNeeded; + import java.io.IOException; import java.util.Map; @@ -79,11 +82,11 @@ public void test_NoExtJsonFile() throws IOException { public void test_NoExtFileCheckDigNeeded() throws IOException { String jsonAsString = readJsonAsString("unit_test_files/filebody_unit_test/json_step_no_ext_json_test_file.json"); Step step = objectMapper.readValue(jsonAsString, Step.class); - assertThat(externalFileProcessor.checkDigNeeded(step), is(false)); + assertThat(checkDigNeeded(objectMapper, step, ZeroCodeValueTokens.JSON_PAYLOAD_FILE, ZeroCodeValueTokens.YAML_PAYLOAD_FILE), is(false)); jsonAsString = readJsonAsString("unit_test_files/filebody_unit_test/json_step_text_node_ext_json_file_test.json"); step = objectMapper.readValue(jsonAsString, Step.class); - assertThat(externalFileProcessor.checkDigNeeded(step), is(true)); + assertThat(checkDigNeeded(objectMapper, step, ZeroCodeValueTokens.JSON_PAYLOAD_FILE, ZeroCodeValueTokens.YAML_PAYLOAD_FILE), is(true)); } @Test diff --git a/core/src/test/java/org/jsmart/zerocode/core/kafka/consume/ConsumerLocalConfigsWrapTest.java b/core/src/test/java/org/jsmart/zerocode/core/kafka/consume/ConsumerLocalConfigsWrapTest.java index 4a9de75df..63e2bf93f 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/kafka/consume/ConsumerLocalConfigsWrapTest.java +++ b/core/src/test/java/org/jsmart/zerocode/core/kafka/consume/ConsumerLocalConfigsWrapTest.java @@ -15,6 +15,86 @@ public class ConsumerLocalConfigsWrapTest { ObjectMapper objectMapper = new ObjectMapperProvider().get(); + @Test + public void testSerDeser_seekEpoch() throws IOException { + ConsumerLocalConfigsWrap javaObject = new ConsumerLocalConfigsWrap( + new ConsumerLocalConfigs("RAW", + "RAW:/target/ttt", + true, + null, + true, + 3, + 50L, + false, + "$.JSON.Path", + null, + "1706940293669", + null)); + ObjectMapper objectMapper = new ObjectMapperProvider().get(); + + String json = objectMapper.writeValueAsString(javaObject); + assertEquals("{\n" + + " \"consumerLocalConfigs\":\n" + + " {\n" + + " \"recordType\": \"RAW\",\n" + + " \"fileDumpTo\": \"RAW:/target/ttt\",\n" + + " \"commitAsync\": true,\n" + + " \"showRecordsConsumed\": true,\n" + + " \"maxNoOfRetryPollsOrTimeouts\": 3,\n" + + " \"pollingTime\": 50,\n" + + " \"cacheByTopic\": false,\n" + + " \"filterByJsonPath\": \"$.JSON.Path\",\n" + + " \"seekEpoch\": \"1706940293669\"\n" + + " }\n" + + "}", + json, LENIENT); + + ConsumerLocalConfigsWrap javaPojo = objectMapper.readValue(json, ConsumerLocalConfigsWrap.class); + assertThat(javaPojo, is(javaObject)); + } + + @Test + public void testSerDeser_seekTimestamp() throws IOException { + ConsumerLocalConfigsWrap javaObject = new ConsumerLocalConfigsWrap( + new ConsumerLocalConfigs("RAW", + "RAW:/target/ttt", + true, + null, + true, + 3, + 50L, + false, + "$.JSON.Path", + null, + null, + new SeekTimestamp("2024-01-29T19:35:21.959340", "yyyy-MM-dd'T'HH:mm:ss.ssssss"))); + ObjectMapper objectMapper = new ObjectMapperProvider().get(); + + String json = objectMapper.writeValueAsString(javaObject); + assertEquals("{\n" + + " \"consumerLocalConfigs\":\n" + + " {\n" + + " \"recordType\": \"RAW\",\n" + + " \"fileDumpTo\": \"RAW:/target/ttt\",\n" + + " \"commitAsync\": true,\n" + + " \"showRecordsConsumed\": true,\n" + + " \"maxNoOfRetryPollsOrTimeouts\": 3,\n" + + " \"pollingTime\": 50,\n" + + " \"cacheByTopic\": false,\n" + + " \"filterByJsonPath\": \"$.JSON.Path\",\n" + + " \"seekTimestamp\":\n" + + " {\n" + + " \"timestamp\": \"2024-01-29T19:35:21.959340\",\n" + + " \"format\": \"yyyy-MM-dd'T'HH:mm:ss.ssssss\"\n" + + " }\n" + + " }\n" + + "}", + json, LENIENT); + + ConsumerLocalConfigsWrap javaPojo = objectMapper.readValue(json, ConsumerLocalConfigsWrap.class); + assertThat(javaPojo, is(javaObject)); + } + @Test public void testSerDeser() throws IOException { ConsumerLocalConfigsWrap javaObject = new ConsumerLocalConfigsWrap( @@ -27,7 +107,8 @@ public void testSerDeser() throws IOException { 50L, false, "$.JSON.Path", - "1,0,test-topic")); + "1,0,test-topic", + null, null)); ObjectMapper objectMapper = new ObjectMapperProvider().get(); String json = objectMapper.writeValueAsString(javaObject); @@ -58,7 +139,8 @@ public void testSerDeser_oneFieldOnly() throws IOException { null, false, null, - "1,0,test-topic")); + "1,0,test-topic", + null, null)); String json = objectMapper.writeValueAsString(javaObject); assertEquals("{\n" + diff --git a/core/src/test/java/org/jsmart/zerocode/core/kafka/helper/KafkaConsumerHelperTest.java b/core/src/test/java/org/jsmart/zerocode/core/kafka/helper/KafkaConsumerHelperTest.java index e1b2eefd0..63699055c 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/kafka/helper/KafkaConsumerHelperTest.java +++ b/core/src/test/java/org/jsmart/zerocode/core/kafka/helper/KafkaConsumerHelperTest.java @@ -44,7 +44,7 @@ public class KafkaConsumerHelperTest { public void test_syncAsyncTrueCommon() throws Exception { consumerCommon = new ConsumerCommonConfigs(true, true, "aTestFile", "JSON", true, 3, 50L, - false,"$JSON.Path", ""); + false,"$JSON.Path"); expectedException.expectMessage("Both commitSync and commitAsync can not be true"); ConsumerLocalConfigs consumerEffectiveConfigs = deriveEffectiveConfigs(null, consumerCommon); @@ -52,8 +52,8 @@ public void test_syncAsyncTrueCommon() throws Exception { @Test public void test_syncAsyncTrueLocal() throws Exception { - consumerCommon = new ConsumerCommonConfigs(true, false, "aTestFile", "JSON", true, 3, 50L,false,"$JSON.Path", ""); - consumerLocal = new ConsumerLocalConfigs("RAW", "sTestLocalFile", true, true, false, 3, 50L,false,"$JSON.Path", "1,0,test-topic"); + consumerCommon = new ConsumerCommonConfigs(true, false, "aTestFile", "JSON", true, 3, 50L,false,"$JSON.Path"); + consumerLocal = new ConsumerLocalConfigs("RAW", "sTestLocalFile", true, true, false, 3, 50L,false,"$JSON.Path", "1,0,test-topic", null, null); ConsumerLocalConfigsWrap localConfigsWrap = new ConsumerLocalConfigsWrap(consumerLocal); expectedException.expectMessage("Both commitSync and commitAsync can not be true"); @@ -63,8 +63,8 @@ public void test_syncAsyncTrueLocal() throws Exception { @Test public void test_effectiveConfigsIsLocal() throws Exception { - consumerCommon = new ConsumerCommonConfigs(true, false, "aTestFile", "JSON", true, 3, 50L,false,"$JSON.Path", ""); - consumerLocal = new ConsumerLocalConfigs("RAW", "sTestLocalFile", true, false, false, 3, 150L,false,"$JSON.Path", "1,0,test-topic"); + consumerCommon = new ConsumerCommonConfigs(true, false, "aTestFile", "JSON", true, 3, 50L,false,"$JSON.Path"); + consumerLocal = new ConsumerLocalConfigs("RAW", "sTestLocalFile", true, false, false, 3, 150L,false,"$JSON.Path", "1,0,test-topic", null, null); ConsumerLocalConfigs consumerEffectiveConfigs = deriveEffectiveConfigs(consumerLocal, consumerCommon); @@ -79,7 +79,7 @@ public void test_effectiveConfigsIsLocal() throws Exception { @Test public void test_effectiveConfigsIsCentral() throws Exception { - consumerCommon = new ConsumerCommonConfigs(true, false, "aTestFile", "JSON", true, 3, 50L,false,"$JSON.Path", ""); + consumerCommon = new ConsumerCommonConfigs(true, false, "aTestFile", "JSON", true, 3, 50L,false,"$JSON.Path"); consumerLocal = null; ConsumerLocalConfigs consumerEffectiveConfigs = deriveEffectiveConfigs(consumerLocal, consumerCommon); @@ -94,8 +94,8 @@ public void test_effectiveConfigsIsCentral() throws Exception { @Test public void test_effectiveCommitAsync_true() throws Exception { - consumerCommon = new ConsumerCommonConfigs(true, null, "aTestFile", "JSON", true, 3, 50L,false,"$JSON.Path", ""); - consumerLocal = new ConsumerLocalConfigs("RAW", "sTestLocalFile", true, false, false, 3, 50L,false,"$JSON.Path", "1,0,test-topic"); + consumerCommon = new ConsumerCommonConfigs(true, null, "aTestFile", "JSON", true, 3, 50L,false,"$JSON.Path"); + consumerLocal = new ConsumerLocalConfigs("RAW", "sTestLocalFile", true, false, false, 3, 50L,false,"$JSON.Path", "1,0,test-topic", null, null); ConsumerLocalConfigs consumerEffectiveConfigs = deriveEffectiveConfigs(consumerLocal, consumerCommon); @@ -106,8 +106,8 @@ public void test_effectiveCommitAsync_true() throws Exception { @Test public void test_effectiveCommitSync_true() throws Exception { - consumerCommon = new ConsumerCommonConfigs(null, true, "aTestFile", "JSON", true, 3, 50L,false,"$JSON.Path", ""); - consumerLocal = new ConsumerLocalConfigs("RAW", "sTestLocalFile", null, true, false, 3, 50L,false,"$JSON.Path", "1,0,test-topic"); + consumerCommon = new ConsumerCommonConfigs(null, true, "aTestFile", "JSON", true, 3, 50L,false,"$JSON.Path"); + consumerLocal = new ConsumerLocalConfigs("RAW", "sTestLocalFile", null, true, false, 3, 50L,false,"$JSON.Path", "1,0,test-topic", null, null); ConsumerLocalConfigs consumerEffectiveConfigs = deriveEffectiveConfigs(consumerLocal, consumerCommon); @@ -118,8 +118,8 @@ public void test_effectiveCommitSync_true() throws Exception { @Test public void test_effectiveCommitSyncFromCommon_true() throws Exception { - consumerCommon = new ConsumerCommonConfigs(true, false, "aTestFile", "JSON", true, 3, 50L,false,"$JSON.Path", ""); - consumerLocal = new ConsumerLocalConfigs("RAW", "sTestLocalFile", null, null, false, 3, 50L,false,"$JSON.Path", "1,0,test-topic"); + consumerCommon = new ConsumerCommonConfigs(true, false, "aTestFile", "JSON", true, 3, 50L,false,"$JSON.Path"); + consumerLocal = new ConsumerLocalConfigs("RAW", "sTestLocalFile", null, null, false, 3, 50L,false,"$JSON.Path", "1,0,test-topic", null, null); ConsumerLocalConfigs consumerEffectiveConfigs = deriveEffectiveConfigs(consumerLocal, consumerCommon); @@ -130,8 +130,8 @@ public void test_effectiveCommitSyncFromCommon_true() throws Exception { @Test public void test_effectiveCommitAsyncFromCommon_true() throws Exception { - consumerCommon = new ConsumerCommonConfigs(null, true, "aTestFile", "JSON", true, 3, 50L,false,"$JSON.Path", ""); - consumerLocal = new ConsumerLocalConfigs("RAW", "sTestLocalFile", true, false, false, 3, 150L,false,"$JSON.Path", "1,0,test-topic"); + consumerCommon = new ConsumerCommonConfigs(null, true, "aTestFile", "JSON", true, 3, 50L,false,"$JSON.Path"); + consumerLocal = new ConsumerLocalConfigs("RAW", "sTestLocalFile", true, false, false, 3, 150L,false,"$JSON.Path", "1,0,test-topic", null, null); ConsumerLocalConfigs consumerEffectiveConfigs = deriveEffectiveConfigs(consumerLocal, consumerCommon); @@ -166,7 +166,7 @@ public void should_read_json_with_headers_in_record() throws IOException { public void test_firstPoll_exits_early_on_assignment() { // given - consumerCommon = new ConsumerCommonConfigs(true, false, "aTestFile", "JSON", true, 3, 1000L,false,"$JSON.Path", ""); + consumerCommon = new ConsumerCommonConfigs(true, false, "aTestFile", "JSON", true, 3, 1000L,false,"$JSON.Path"); consumerLocal = null; ConsumerLocalConfigs consumerEffectiveConfigs = deriveEffectiveConfigs(consumerLocal, consumerCommon); Consumer consumer = Mockito.mock(Consumer.class); @@ -185,8 +185,8 @@ public void test_firstPoll_exits_early_on_assignment() { public void test_firstPoll_exits_on_receiving_records() { // given - consumerCommon = new ConsumerCommonConfigs(true, false, "aTestFile", "JSON", true, 3, 5000L,false,"$JSON.Path", ""); - consumerLocal = new ConsumerLocalConfigs("RAW", "sTestLocalFile", true, false, false, 3, 50L,false,"$JSON.Path", "1,0,test-topic"); + consumerCommon = new ConsumerCommonConfigs(true, false, "aTestFile", "JSON", true, 3, 5000L,false,"$JSON.Path"); + consumerLocal = new ConsumerLocalConfigs("RAW", "sTestLocalFile", true, false, false, 3, 50L,false,"$JSON.Path", "1,0,test-topic", null, null); ConsumerLocalConfigs consumerEffectiveConfigs = deriveEffectiveConfigs(consumerLocal, consumerCommon); Consumer consumer = Mockito.mock(Consumer.class); Mockito.when(consumer.assignment()).thenReturn(new HashSet()); @@ -208,8 +208,8 @@ public void test_firstPoll_exits_on_receiving_records() { public void test_firstPoll_throws_after_timeout() throws Exception { // given - consumerCommon = new ConsumerCommonConfigs(true, false, "aTestFile", "JSON", true, 3, null,false,"$JSON.Path", ""); - consumerLocal = new ConsumerLocalConfigs("RAW", "sTestLocalFile", true, false, false, 3, 50L,false,"$JSON.Path", "1,0,test-topic"); + consumerCommon = new ConsumerCommonConfigs(true, false, "aTestFile", "JSON", true, 3, null,false,"$JSON.Path"); + consumerLocal = new ConsumerLocalConfigs("RAW", "sTestLocalFile", true, false, false, 3, 50L,false,"$JSON.Path", "1,0,test-topic", null, null); ConsumerLocalConfigs consumerEffectiveConfigs = deriveEffectiveConfigs(consumerLocal, consumerCommon); Consumer consumer = Mockito.mock(Consumer.class); diff --git a/core/src/test/java/org/jsmart/zerocode/core/report/ZeroCodeReportGeneratorImplTest.java b/core/src/test/java/org/jsmart/zerocode/core/report/ZeroCodeReportGeneratorImplTest.java index f573f38e5..60dfa5ea3 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/report/ZeroCodeReportGeneratorImplTest.java +++ b/core/src/test/java/org/jsmart/zerocode/core/report/ZeroCodeReportGeneratorImplTest.java @@ -58,84 +58,85 @@ public void testReportFolderPresentInTargetNormalFlow() throws Exception { @Test - public void testAuthorJiraStyle() throws Exception { - String author; + public void testAuthorJiraStyle_LEGACYMARKER() throws Exception { + List authors; // OLD - Deprecated - author = zeroCodeReportGenerator.optionalAuthor("PayPal One touch payment @@Peter@@"); - assertThat(author, is("Peter")); + authors = zeroCodeReportGenerator.optionalAuthors("PayPal One touch payment @@Peter"); + assertThat(authors.get(0), is("@@Peter")); - author = zeroCodeReportGenerator.optionalAuthor("PayPal One touch @@payment @@Peter@@"); - assertThat(author, is("payment ")); + authors = zeroCodeReportGenerator.optionalAuthors("PayPal One touch payment"); + assertThat(authors.size(), is(0)); - author = zeroCodeReportGenerator.optionalAuthor("PayPal One touch payment @@Peter Gibson@@"); - assertThat(author, is("Peter Gibson")); + authors = zeroCodeReportGenerator.optionalAuthors("PayPal One touch @@payment @@Peter"); + assertThat(authors.get(0), is("@@payment")); + assertThat(authors.get(1), is("@@Peter")); + assertThat(authors.size(), is(2)); - author = zeroCodeReportGenerator.optionalAuthor("PayPal One touch payment @@Peter Gibson"); - assertThat(author, is("Peter")); + authors = zeroCodeReportGenerator.optionalAuthors("PayPal One touch payment @@Peter-Gibson"); + assertThat(authors.get(0), is("@@Peter-Gibson")); - author = zeroCodeReportGenerator.optionalAuthor("PayPal One touch payment @@Peter"); - assertThat(author, is("Peter")); - - author = zeroCodeReportGenerator.optionalAuthor("@@Peter, PayPal One touch payment "); - assertThat(author, is("Peter")); - - author = zeroCodeReportGenerator.optionalAuthor("PayPal One touch payment"); - assertThat(author, is("Anonymous")); + authors = zeroCodeReportGenerator.optionalAuthors("PayPal One touch payment @@Peter Gibson"); + assertThat(authors.get(0), is("@@Peter")); + authors = zeroCodeReportGenerator.optionalAuthors("@@Peter- PayPal One touch payment "); + assertThat(authors.get(0), is("@@Peter-")); } @Test public void testAuthorJiraStyle_new() throws Exception { - String author; + List authors; - author = zeroCodeReportGenerator.optionalAuthor("PayPal One touch payment @Peter@"); - assertThat(author, is("Peter")); - - author = zeroCodeReportGenerator.optionalAuthor("PayPal One touch @payment @Peter@"); - assertThat(author, is("payment ")); + // OLD - Deprecated + authors = zeroCodeReportGenerator.optionalAuthors("PayPal One touch payment @Peter"); + assertThat(authors.get(0), is("@Peter")); - author = zeroCodeReportGenerator.optionalAuthor("PayPal One touch payment @Peter Gibson@"); - assertThat(author, is("Peter Gibson")); + authors = zeroCodeReportGenerator.optionalAuthors("PayPal One touch payment"); + assertThat(authors.size(), is(0)); - author = zeroCodeReportGenerator.optionalAuthor("PayPal One touch payment @Peter Gibson"); - assertThat(author, is("Peter")); + authors = zeroCodeReportGenerator.optionalAuthors("PayPal One touch @payment @Peter"); + assertThat(authors.get(0), is("@payment")); + assertThat(authors.get(1), is("@Peter")); + assertThat(authors.size(), is(2)); - author = zeroCodeReportGenerator.optionalAuthor("PayPal One touch payment @Peter"); - assertThat(author, is("Peter")); + authors = zeroCodeReportGenerator.optionalAuthors("PayPal One touch payment @Peter-Gibson"); + assertThat(authors.get(0), is("@Peter-Gibson")); - author = zeroCodeReportGenerator.optionalAuthor("@Peter, PayPal One touch payment "); - assertThat(author, is("Peter")); + authors = zeroCodeReportGenerator.optionalAuthors("PayPal One touch payment @Peter Gibson"); + assertThat(authors.get(0), is("@Peter")); - author = zeroCodeReportGenerator.optionalAuthor("PayPal One touch payment"); - assertThat(author, is("Anonymous")); + authors = zeroCodeReportGenerator.optionalAuthors("@Peter- PayPal One touch payment "); + assertThat(authors.get(0), is("@Peter-")); } @Test public void testCategoryHashTag() throws Exception { - String author; + List categories; - author = zeroCodeReportGenerator.optionalCategory("PayPal One touch payment #Smoke#"); - assertThat(author, is("Smoke")); + categories = zeroCodeReportGenerator.optionalCategories("PayPal One touch payment #Smoke"); + assertThat(categories.get(0), is("#Smoke")); + assertThat(categories.size(), is(1)); - author = zeroCodeReportGenerator.optionalCategory("PayPal One touch #Smoke #PDC#"); - assertThat(author, is("Smoke ")); + categories = zeroCodeReportGenerator.optionalCategories("PayPal One touch #Smoke #PDC"); + assertThat(categories.get(0), is("#Smoke")); + assertThat(categories.get(1), is("#PDC")); + assertThat(categories.size(), is(2)); - author = zeroCodeReportGenerator.optionalCategory("PayPal One touch payment #SIT Smoke#"); - assertThat(author, is("SIT Smoke")); + categories = zeroCodeReportGenerator.optionalCategories("PayPal One touch payment #SIT_Smoke"); + assertThat(categories.get(0), is("#SIT_Smoke")); - author = zeroCodeReportGenerator.optionalCategory("PayPal One touch payment #PDC Gibson"); - assertThat(author, is("PDC")); + categories = zeroCodeReportGenerator.optionalCategories("PayPal One touch payment #PDC-Gibson"); + assertThat(categories.get(0), is("#PDC-Gibson")); - author = zeroCodeReportGenerator.optionalCategory("PayPal One touch payment #PDC"); - assertThat(author, is("PDC")); + categories = zeroCodeReportGenerator.optionalCategories("PayPal One touch payment #PDC"); + assertThat(categories.get(0), is("#PDC")); - author = zeroCodeReportGenerator.optionalCategory("#PDC, PayPal One touch payment "); - assertThat(author, is("PDC")); + categories = zeroCodeReportGenerator.optionalCategories("#PDC, PayPal One touch payment "); + assertThat(categories.get(0), is("#PDC,")); - author = zeroCodeReportGenerator.optionalCategory("PayPal One touch payment"); - assertThat(author, is("Anonymous")); + categories = zeroCodeReportGenerator.optionalCategories("PayPal One touch payment"); + assertThat(categories.size(), is(0)); } diff --git a/core/src/test/java/org/jsmart/zerocode/core/utils/FileUploadUtilsTest.java b/core/src/test/java/org/jsmart/zerocode/core/utils/FileUploadUtilsTest.java new file mode 100644 index 000000000..54e45a6a9 --- /dev/null +++ b/core/src/test/java/org/jsmart/zerocode/core/utils/FileUploadUtilsTest.java @@ -0,0 +1,27 @@ +package org.jsmart.zerocode.core.utils; + +import org.apache.http.entity.mime.MultipartEntityBuilder; +import org.jsmart.zerocode.core.httpclient.utils.FileUploadUtils; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.junit.Assert.fail; + +public class FileUploadUtilsTest { + + @Test + public void buildOtherRequestParamsTest1() { + Map fileFieldNameValueMapStub = new LinkedHashMap<>(); + fileFieldNameValueMapStub.put("name", "name"); + fileFieldNameValueMapStub.put("fileName", "test.wav"); + fileFieldNameValueMapStub.put("location", "location"); + MultipartEntityBuilder multipartEntityBuilderStub = MultipartEntityBuilder.create(); + try { + FileUploadUtils.buildOtherRequestParams(fileFieldNameValueMapStub, multipartEntityBuilderStub); + } catch (Exception e) { + fail("Should not have thrown any exception"); + } + } +} diff --git a/core/src/test/java/org/jsmart/zerocode/core/utils/SmartUtilsTest.java b/core/src/test/java/org/jsmart/zerocode/core/utils/SmartUtilsTest.java index 7b5012127..d8e22b66f 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/utils/SmartUtilsTest.java +++ b/core/src/test/java/org/jsmart/zerocode/core/utils/SmartUtilsTest.java @@ -74,14 +74,14 @@ public void willGetJsonFileIntoA_JavaString() throws Exception { } @Test - public void willReadAllfileNamesFrom_TestResource() throws Exception { + public void willReadAllfileNamesFrom_TestResource() { List allTestCaseFiles = SmartUtils.getAllEndPointFiles("unit_test_files/engine_unit_test_jsons"); - assertThat(allTestCaseFiles.size(), is(20)); + assertThat(allTestCaseFiles.size(), is(22)); assertThat(allTestCaseFiles.get(0), is("unit_test_files/engine_unit_test_jsons/00_test_json_single_step_verifications.json")); } @Test - public void willReadAllfileNames_AND_return_FlowSpecList() throws Exception { + public void willReadAllfileNames_AND_return_FlowSpecList() { List allTestCaseFiles = smartUtils.getScenarioSpecListByPackage("unit_test_files/test_scenario_cases"); assertThat(allTestCaseFiles.size(), is(3)); @@ -91,19 +91,19 @@ public void willReadAllfileNames_AND_return_FlowSpecList() throws Exception { @Test(expected = RuntimeException.class) - public void willReadAllfiles_find_DuplicatesScenarioNamenames_old_style() throws Exception { + public void willReadAllfiles_find_DuplicatesScenarioNamenames_old_style() { smartUtils.checkDuplicateScenarios("unit_test_files/test_scenario_cases"); } @Test - public void willReadAllfiles_find_DuplicateScenarioNames() throws Exception { + public void willReadAllfiles_find_DuplicateScenarioNames() { expectedException.expect(RuntimeException.class); expectedException.expectMessage("Oops! Can not run with multiple Scenarios with same name."); smartUtils.checkDuplicateScenarios("unit_test_files/test_scenario_cases"); } @Test - public void willEvaluatePlaceHolder() throws Exception { + public void willEvaluatePlaceHolder() { String aString = "Hello_${WORLD}"; List placeHolders = getTestCaseTokens(aString); @@ -118,7 +118,7 @@ public void willEvaluatePlaceHolder() throws Exception { } @Test - public void testNullOrEmptyString_withPlaceHolders() throws Exception { + public void testNullOrEmptyString_withPlaceHolders() { String aString = ""; List placeHolders = getTestCaseTokens(aString); @@ -130,7 +130,7 @@ public void testNullOrEmptyString_withPlaceHolders() throws Exception { } @Test - public void testReplaceTokensOrPlaceHolders() throws Exception { + public void testReplaceTokensOrPlaceHolders() { String aString = "_${ENV_PROPERTY_NAME}"; Map paramMap = new HashMap<>(); @@ -142,7 +142,7 @@ public void testReplaceTokensOrPlaceHolders() throws Exception { } @Test - public void testEnvValue() throws Exception { + public void testEnvValue() { final String javaHomeValue = SmartUtils.getEnvPropertyValue("JAVA_HOME"); assertThat(javaHomeValue, notNullValue()); @@ -228,7 +228,7 @@ public void testScenarioFile_absolutePath() throws Exception { @Ignore("Tested in local laptop. Ignored for Ci build. Follow testSuiteFolder_absolutePath() like flow ") @Test - public void testSuiteFolder_symAbsolutePath() throws Exception { + public void testSuiteFolder_symAbsolutePath() { String absPath = "~/dev/ZEROCODE_REPOS/zerocode/core/src/test/resources/unit_test_files/cherry_pick_tests"; List allScenarios = SmartUtils.retrieveScenariosByAbsPath(absPath); assertThat(allScenarios.size(), is(2)); @@ -242,9 +242,7 @@ private static File createCascadeIfNotExisting(String fileName) { Path path = Paths.get(fileName); Files.createDirectories(path.getParent()); - File file = new File(fileName); - - return file; + return new File(fileName); } catch (IOException exx) { throw new RuntimeException("Create file '" + fileName + "' Exception" + exx); } diff --git a/core/src/test/java/org/jsmart/zerocode/core/utils/TokenUtilsTest.java b/core/src/test/java/org/jsmart/zerocode/core/utils/TokenUtilsTest.java index 9566c4dc9..a4dedeadd 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/utils/TokenUtilsTest.java +++ b/core/src/test/java/org/jsmart/zerocode/core/utils/TokenUtilsTest.java @@ -10,6 +10,8 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.jsmart.zerocode.core.utils.TokenUtils.absolutePathOf; +import static org.jsmart.zerocode.core.utils.TokenUtils.getMasksReplaced; +import static org.jsmart.zerocode.core.utils.TokenUtils.getMasksRemoved; import static org.jsmart.zerocode.core.utils.TokenUtils.resolveKnownTokens; import static org.junit.Assert.*; @@ -169,4 +171,48 @@ public void testAbsolutePathOf() { assertThat(absolutePathOf("unit_test_files/jks_files/dummy_key_store.jks"), containsString("zerocode/core/target/test-classes/unit_test_files/jks_files/dummy_key_store.jks")); } + + + @Test + public void testGetMaskedTokensReplaced_multipleOccurrences(){ + assertEquals("This is a ***masked*** message with ***masked*** tokens.", getMasksReplaced("This is a ${MASKED:secret} message with ${MASKED:masked} tokens.")); + } + + @Test + public void testGetMaskedTokensReplaced_noOccurrences(){ + assertEquals("This string has no masked tokens.", getMasksReplaced("This string has no masked tokens.")); + } + + @Test + public void testGetMaskedTokensReplaced_emptyString(){ + assertEquals("", getMasksReplaced("")); + } + + @Test + public void testGetMaskedTokensReplaced_specialCharacters(){ + assertEquals("***masked*** and ***masked***", getMasksReplaced("${MASKED:abc@123} and ${MASKED:!@#$%^}")); + } + + @Test + public void testGetMaskedTokensRemoved_multipleOccurrences(){ + assertEquals("This is a secret message with masked tokens.", getMasksRemoved("This is a ${MASKED:secret} message with ${MASKED:masked} tokens.")); + } + + @Test + public void testGetMaskedTokensRemoved_noOccurrences(){ + assertEquals("This string has no masked tokens.", getMasksRemoved("This string has no masked tokens.")); + } + + @Test + public void testGetMaskedTokensRemoved_emptyString(){ + assertEquals("", getMasksRemoved("")); + } + + @Test + public void testGetMaskedTokensRemoved_specialCharacters(){ + assertEquals("abc@123 and !@#$%^", getMasksRemoved("${MASKED:abc@123} and ${MASKED:!@#$%^}")); + } + + + } \ No newline at end of file diff --git a/core/src/test/java/org/jsmart/zerocode/integrationtests/masked/MaskedSecretsInMemoryTest.java b/core/src/test/java/org/jsmart/zerocode/integrationtests/masked/MaskedSecretsInMemoryTest.java new file mode 100644 index 000000000..2ed676ad1 --- /dev/null +++ b/core/src/test/java/org/jsmart/zerocode/integrationtests/masked/MaskedSecretsInMemoryTest.java @@ -0,0 +1,32 @@ +package org.jsmart.zerocode.integrationtests.masked; + +import org.jsmart.zerocode.core.domain.HostProperties; +import org.jsmart.zerocode.core.domain.Scenario; +import org.jsmart.zerocode.core.tests.customrunner.TestOnlyZeroCodeUnitRunner; +import org.junit.Test; +import org.junit.runner.RunWith; + +@HostProperties(host="http://localhost", port=9998, context = "") +@RunWith(TestOnlyZeroCodeUnitRunner.class) +public class MaskedSecretsInMemoryTest { + + /** + * Mock end points are in test/resources: simulators/test_purpose_end_points.json. + * @RunWith(TestOnlyZeroCodeUnitRunner.class) : starts these mocks first before running the tests + */ + + @Test + @Scenario("integration_test_files/masked/password_or_secrets_masked_in_log_n_console.json") + public void testSecretPrinted_masked() throws Exception { + } + + @Test + @Scenario("integration_test_files/masked/bearer_token_or_secret_masked_reuse_example_.json") + public void testSecretPrintedAssertions_masked() throws Exception { + + } + +} + + + diff --git a/core/src/test/resources/integration_test_files/filebody/hello_world_file_request_n_response.json b/core/src/test/resources/integration_test_files/filebody/hello_world_file_request_n_response.json index 4f521a905..42f104f76 100644 --- a/core/src/test/resources/integration_test_files/filebody/hello_world_file_request_n_response.json +++ b/core/src/test/resources/integration_test_files/filebody/hello_world_file_request_n_response.json @@ -22,6 +22,23 @@ "status": 200, "body" : "${JSON.FILE:integration_test_files/filebody/bodyonly/response_body.json}" } + }, + { + "name": "create_bathroom_with_name", + "url": "/home/bathroom/21", + "operation": "PUT", + "request": { + "body" : { + "name": "${JSON.CONTENT:$.get_user_details.response.body.name}", + "availability": true + } + }, + "assertions": { + "status": 200, + "body": { + "name": "Shower-21" + } + } } ] } diff --git a/core/src/test/resources/integration_test_files/masked/bearer_token_or_secret_masked_reuse_example_.json b/core/src/test/resources/integration_test_files/masked/bearer_token_or_secret_masked_reuse_example_.json new file mode 100644 index 000000000..1871b434f --- /dev/null +++ b/core/src/test/resources/integration_test_files/masked/bearer_token_or_secret_masked_reuse_example_.json @@ -0,0 +1,48 @@ +{ + "scenarioName": "Masked Secret path as step param - Reusable Secret or Bearer token", + "steps": [ + { + "name": "get_balance_via_secret_token", + "url": "/home/accounts/123/balance", + "operation": "GET", + "request": { + "headers": { + "Authorization": "Bearer ${MASKED:token1002003004}", + "request-bank-name": "${MASKED:HSBC}", + "secretCode": "${MASKED:Amazing}" + } + + }, + "assertions": { + "status": 200, + "body" : { + "balance" : 3900, + "name" : "${$.get_balance_via_secret_token.request.headers.request-bank-name}", + "current" : true + } + } + }, + { + "name": "find_match", + "url": "/api/v1/search/persons", + "method": "GET", + "request": { + "queryParams": { + "lang": "${$.get_balance_via_secret_token.request.headers.secretCode}", + "city": "${MASKED:Lon}" + } + }, + "verify": { + "status": 200, + "body": { + "exactMatches": true, + "name": "Mr Bean", + "lang": "${$.get_balance_via_secret_token.request.headers.secretCode}", + "city": "${$.find_match.request.queryParams.city}" + } + } + } + + ] +} + diff --git a/core/src/test/resources/integration_test_files/masked/password_or_secrets_masked_in_log_n_console.json b/core/src/test/resources/integration_test_files/masked/password_or_secrets_masked_in_log_n_console.json new file mode 100644 index 000000000..00044a409 --- /dev/null +++ b/core/src/test/resources/integration_test_files/masked/password_or_secrets_masked_in_log_n_console.json @@ -0,0 +1,31 @@ +{ + "scenarioName": "As simple GET API - Inetgration Test - Local Server", + "steps": [ + { + "name": "find_match", + "url": "/api/v1/search/persons", + "method": "GET", + "request": { + "queryParams": { + "lang": "${MASKED:Amazing}", + "city": "Lon" + }, + "body": { + "api_secret" : "${MASKED:pass123}", + "emotion": "${MASKED:Amazing}", + "state": "Amazing" + } + }, + "verify": { + "status": 200, + "body": { + "exactMatches": true, + "name": "Mr Bean", + "lang": "${$.find_match.request.body.emotion}", + "city": "Lon" + } + } + } + ] +} + diff --git a/core/src/test/resources/simulators/test_purpose_end_points.json b/core/src/test/resources/simulators/test_purpose_end_points.json index 3971527b6..2c738d8ff 100644 --- a/core/src/test/resources/simulators/test_purpose_end_points.json +++ b/core/src/test/resources/simulators/test_purpose_end_points.json @@ -1,6 +1,19 @@ { "name": "Mock endpoints Simulator - API Stubs", "apis": [ + { + "name": "Get My Bank Balance", + "operation": "GET", + "url": "/home/accounts/123/balance", + "response": { + "status": 200, + "body": { + "balance": 3900, + "name": "HSBC", + "current": true + } + } + }, { "name": "Get Bank Account by Id", "operation": "GET", @@ -43,6 +56,20 @@ } } }, + { + "name": "PUT Bathroom - Update", + "operation": "PUT", + "url": "/home/bathroom/21", + "ignoreBody": true, + "response": { + "status": 200, + "body": { + "id": 21, + "name": "Shower-21", + "availability": true + } + } + }, { "name": "Get Bathroom by Id 21", "operation": "GET", diff --git a/core/src/test/resources/unit_test_files/engine_unit_test_jsons/08.1_parameterized_csv_source_from_file.json b/core/src/test/resources/unit_test_files/engine_unit_test_jsons/08.1_parameterized_csv_source_from_file.json new file mode 100755 index 000000000..5fe1f5aee --- /dev/null +++ b/core/src/test/resources/unit_test_files/engine_unit_test_jsons/08.1_parameterized_csv_source_from_file.json @@ -0,0 +1,8 @@ +{ + "valueSource": [ + "hello", + 123, + true + ], + "csvSource": "unit_test_files/engine_unit_test_jsons/params.csv" +} diff --git a/core/src/test/resources/unit_test_files/engine_unit_test_jsons/08.2_parameterized_csv_source_from_file_containing_header.json b/core/src/test/resources/unit_test_files/engine_unit_test_jsons/08.2_parameterized_csv_source_from_file_containing_header.json new file mode 100755 index 000000000..bb456137f --- /dev/null +++ b/core/src/test/resources/unit_test_files/engine_unit_test_jsons/08.2_parameterized_csv_source_from_file_containing_header.json @@ -0,0 +1,9 @@ +{ + "valueSource": [ + "hello", + 123, + true + ], + "ignoreHeader": true, + "csvSource": "unit_test_files/engine_unit_test_jsons/params_with_header.csv" +} diff --git a/core/src/test/resources/unit_test_files/engine_unit_test_jsons/params.csv b/core/src/test/resources/unit_test_files/engine_unit_test_jsons/params.csv new file mode 100644 index 000000000..faef91767 --- /dev/null +++ b/core/src/test/resources/unit_test_files/engine_unit_test_jsons/params.csv @@ -0,0 +1,2 @@ +octocat,The Octocat,San Francisco,583231 +siddhagalaxy,Sidd,UK,33847730 \ No newline at end of file diff --git a/core/src/test/resources/unit_test_files/engine_unit_test_jsons/params_with_header.csv b/core/src/test/resources/unit_test_files/engine_unit_test_jsons/params_with_header.csv new file mode 100644 index 000000000..3df16ea8d --- /dev/null +++ b/core/src/test/resources/unit_test_files/engine_unit_test_jsons/params_with_header.csv @@ -0,0 +1,3 @@ +user,name,city,userid +octocat,The Octocat,San Francisco,583231 +siddhagalaxy,Sidd,UK,33847730 \ No newline at end of file diff --git a/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_no_json_content_test.json b/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_no_json_content_test.json new file mode 100644 index 000000000..271d22bf6 --- /dev/null +++ b/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_no_json_content_test.json @@ -0,0 +1,21 @@ +{ + "name": "get_user_details", + "url": "/api/v1/employees/${$.create_emp.response.body.id}", + "method": "GET", + "request": { + "body" : { + "fullName":"Bob Luis", + "empLogin":"bolu_lon", + "type":"CONTRACT" + } + }, + "verify": { + "status": 200, + "body": { + "id": 39001, + "ldapId": "emmanorton", + "name": "Emma", + "surName": "Norton" + } + } +} diff --git a/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_test_json_content.json b/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_test_json_content.json new file mode 100644 index 000000000..1a8af01e0 --- /dev/null +++ b/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_test_json_content.json @@ -0,0 +1,41 @@ +{ + "scenarioName": "POST API - File json as request content - Reuse body", + "steps": [ + { + "name": "create_emp", + "url": "/api/v1/employees", + "method": "POST", + "request": { + "body": { + "name": "Emma", + "surName": "Norton" + } + }, + "verify": { + "status": 201 + } + }, + { + "name": "get_user_details", + "url": "/api/v1/employees/${$.create_emp.response.body.id}", + "method": "GET", + "request": { + "body": { + "fullName": "Bob Luis", + "empLogin": "bolu_lon", + "type": "CONTRACT", + "addressId": "${JSON.CONTENT:$.create_emp.response.body.id}" + } + }, + "verify": { + "status": 200, + "body": { + "id": 39001, + "ldapId": "emmanorton", + "name": "Emma", + "surName": "Norton" + } + } + } + ] +} diff --git a/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_test_json_content_array.json b/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_test_json_content_array.json new file mode 100644 index 000000000..8c0f619af --- /dev/null +++ b/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_test_json_content_array.json @@ -0,0 +1,41 @@ +{ + "scenarioName": "POST API - File json as request content - Reuse body", + "steps": [ + { + "name": "create_emp", + "url": "/api/v1/employees", + "method": "POST", + "request": { + "body": { + "name": "Emma", + "surName": "Norton" + } + }, + "verify": { + "status": 201 + } + }, + { + "name": "get_user_details", + "url": "/api/v1/employees/${$.create_emp.response.body.id}", + "method": "GET", + "request": { + "body": { + "fullName": "Bob Luis", + "empLogin": "bolu_lon", + "type": "CONTRACT", + "names": "${JSON.CONTENT:$.create_emp.response.body.names}" + } + }, + "verify": { + "status": 200, + "body": { + "id": 39001, + "ldapId": "emmanorton", + "name": "Emma", + "surName": "Norton" + } + } + } + ] +} diff --git a/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_test_json_content_block.json b/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_test_json_content_block.json new file mode 100644 index 000000000..b287b99d3 --- /dev/null +++ b/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_test_json_content_block.json @@ -0,0 +1,41 @@ +{ + "scenarioName": "POST API - File json as request content - Reuse body", + "steps": [ + { + "name": "create_emp", + "url": "/api/v1/employees", + "method": "POST", + "request": { + "body": { + "name": "Emma", + "surName": "Norton" + } + }, + "verify": { + "status": 201 + } + }, + { + "name": "get_user_details", + "url": "/api/v1/employees/${$.create_emp.response.body.id}", + "method": "GET", + "request": { + "body": { + "fullName": "Bob Luis", + "empLogin": "bolu_lon", + "type": "CONTRACT", + "address": "${JSON.CONTENT:$.create_emp.response.body.address}" + } + }, + "verify": { + "status": 200, + "body": { + "id": 39001, + "ldapId": "emmanorton", + "name": "Emma", + "surName": "Norton" + } + } + } + ] +} diff --git a/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_test_json_content_objectarray.json b/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_test_json_content_objectarray.json new file mode 100644 index 000000000..daf4b95f4 --- /dev/null +++ b/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_test_json_content_objectarray.json @@ -0,0 +1,41 @@ +{ + "scenarioName": "POST API - File json as request content - Reuse body", + "steps": [ + { + "name": "create_emp", + "url": "/api/v1/employees", + "method": "POST", + "request": { + "body": { + "name": "Emma", + "surName": "Norton" + } + }, + "verify": { + "status": 201 + } + }, + { + "name": "get_user_details", + "url": "/api/v1/employees/${$.create_emp.response.body.id}", + "method": "GET", + "request": { + "body": { + "fullName": "Bob Luis", + "empLogin": "bolu_lon", + "type": "CONTRACT", + "allAddresses": "${JSON.CONTENT:$.create_emp.response.body.allAddresses}" + } + }, + "verify": { + "status": 200, + "body": { + "id": 39001, + "ldapId": "emmanorton", + "name": "Emma", + "surName": "Norton" + } + } + } + ] +} diff --git a/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_test_wrong_json_path.json b/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_test_wrong_json_path.json new file mode 100644 index 000000000..5fe61831b --- /dev/null +++ b/core/src/test/resources/unit_test_files/json_content_unit_test/json_step_test_wrong_json_path.json @@ -0,0 +1,22 @@ +{ + "name": "get_user_details", + "url": "/api/v1/employees/${$.create_emp.response.body.id}", + "method": "GET", + "request": { + "body" : { + "fullName":"Bob Luis", + "empLogin":"bolu_lon", + "type":"CONTRACT", + "address":"${JSON.CONTENT:$.create_emp.response.body.id}" + } + }, + "verify": { + "status": 200, + "body": { + "id": 39001, + "ldapId": "emmanorton", + "name": "Emma", + "surName": "Norton" + } + } +} diff --git a/docker/compose/kafka-schema-registry-m3.yml b/docker/compose/kafka-schema-registry-m3.yml new file mode 100644 index 000000000..c7224d4c3 --- /dev/null +++ b/docker/compose/kafka-schema-registry-m3.yml @@ -0,0 +1,49 @@ +--- +version: '3' +services: + zookeeper: + image: confluentinc/cp-zookeeper:5.5.1 + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + + kafka: + image: confluentinc/cp-kafka:5.5.1 + depends_on: + - zookeeper + ports: + - 9092:9092 + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + + schema-registry: + image: confluentinc/cp-schema-registry:5.5.1 + depends_on: + - kafka + - zookeeper + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181 + SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 + ports: + - "8081:8081" + + rest-proxy: + image: confluentinc/cp-kafka-rest:5.5.1 + depends_on: + - zookeeper + - kafka + - schema-registry + environment: + KAFKA_REST_HOST_NAME: rest-proxy + KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_REST_BOOTSTRAP_SERVERS: kafka:29092 + KAFKA_REST_LISTENERS: http://0.0.0.0:8082 + KAFKA_REST_SCHEMA_REGISTRY_URL: http://schema-registry:8081 + ports: + - "8082:8082" diff --git a/http-testing/src/main/java/org/jsmart/zerocode/zerocodejavaexec/httpclient/JustHelloWorldSuite.java b/http-testing/src/main/java/org/jsmart/zerocode/zerocodejavaexec/httpclient/JustHelloWorldSuite.java index 413cf203d..97e3e7d75 100644 --- a/http-testing/src/main/java/org/jsmart/zerocode/zerocodejavaexec/httpclient/JustHelloWorldSuite.java +++ b/http-testing/src/main/java/org/jsmart/zerocode/zerocodejavaexec/httpclient/JustHelloWorldSuite.java @@ -1,11 +1,8 @@ package org.jsmart.zerocode.zerocodejavaexec.httpclient; -import org.jsmart.zerocode.core.domain.Scenario; import org.jsmart.zerocode.core.domain.TargetEnv; import org.jsmart.zerocode.core.domain.TestPackageRoot; import org.jsmart.zerocode.core.runner.ZeroCodePackageRunner; -import org.jsmart.zerocode.core.runner.ZeroCodeUnitRunner; -import org.junit.Test; import org.junit.runner.RunWith; @TargetEnv("hello_github_host.properties") diff --git a/http-testing/src/test/java/org/jsmart/zerocode/testhelp/tests/helloworldjsoncontent/HelloJsonContentAsBodyTest.java b/http-testing/src/test/java/org/jsmart/zerocode/testhelp/tests/helloworldjsoncontent/HelloJsonContentAsBodyTest.java new file mode 100644 index 000000000..148a5a91e --- /dev/null +++ b/http-testing/src/test/java/org/jsmart/zerocode/testhelp/tests/helloworldjsoncontent/HelloJsonContentAsBodyTest.java @@ -0,0 +1,18 @@ +package org.jsmart.zerocode.testhelp.tests.helloworldjsoncontent; + +import org.jsmart.zerocode.core.domain.JsonTestCase; +import org.jsmart.zerocode.core.domain.TargetEnv; +import org.jsmart.zerocode.core.runner.ZeroCodeUnitRunner; +import org.junit.Test; +import org.junit.runner.RunWith; + +@TargetEnv("hello_world_host.properties") +@RunWith(ZeroCodeUnitRunner.class) +public class HelloJsonContentAsBodyTest { + + @Test + @JsonTestCase("helloworld_json_content/hello_world_json_content_as_request_body.json") + public void testHelloWorld_jsonContentAsBody() throws Exception { + } + +} diff --git a/http-testing/src/test/java/org/jsmart/zerocode/testhelp/tests/helloworldparameterizedcsv/HelloWorldParameterizedCsvTest.java b/http-testing/src/test/java/org/jsmart/zerocode/testhelp/tests/helloworldparameterizedcsv/HelloWorldParameterizedCsvTest.java index e169a8a24..6baea481e 100644 --- a/http-testing/src/test/java/org/jsmart/zerocode/testhelp/tests/helloworldparameterizedcsv/HelloWorldParameterizedCsvTest.java +++ b/http-testing/src/test/java/org/jsmart/zerocode/testhelp/tests/helloworldparameterizedcsv/HelloWorldParameterizedCsvTest.java @@ -15,4 +15,14 @@ public class HelloWorldParameterizedCsvTest { public void testGetByUserNames_csv() throws Exception { } + @Test + @Scenario("parameterized_csv/hello_world_test_parameterized_csv_source_files.json") + public void testGetByUserNames_csvSourceFiles() throws Exception { + } + + @Test + @Scenario("parameterized_csv/hello_world_test_parameterized_csv_source_file_ignore_header.json") + public void testGetByUserNames_csvSourceFiles_ignoringHeader() throws Exception { + } + } diff --git a/http-testing/src/test/resources/helloworld_json_content/hello_world_json_content_as_request_body.json b/http-testing/src/test/resources/helloworld_json_content/hello_world_json_content_as_request_body.json new file mode 100644 index 000000000..bf3cf2a75 --- /dev/null +++ b/http-testing/src/test/resources/helloworld_json_content/hello_world_json_content_as_request_body.json @@ -0,0 +1,39 @@ +{ + "scenarioName": "POST API - JSON content as request body", + "steps": [ + { + "name": "get_screening", + "url": "/api/v1/employees/screening/SCRUNIQUEID5003", + "method": "GET", + "request": {}, + "assertions": { + "status": 200, + "body": { + "id": "SCRUNIQUEID5003", + "empId": "EMP39001", + "originAddress": { + "addressId":"lon-hsbc-5432", + "countryOfOrigin":"UK" + } + } + } + }, + { + "name": "create_screening", + "url": "/api/v1/employees/screening", + "method": "POST", + "request": { + "body": { + "empId": "${JSON.CONTENT:$.get_screening.response.body.empId}", + "originAddress":"${JSON.CONTENT:$.get_screening.response.body.originAddress}" + } + }, + "assertions": { + "status": 201, + "body": { + "id": "SCRUNIQUEID5003" + } + } + } + ] +} diff --git a/http-testing/src/test/resources/parameterized_csv/hello_world_test_parameterized_csv_source_file_ignore_header.json b/http-testing/src/test/resources/parameterized_csv/hello_world_test_parameterized_csv_source_file_ignore_header.json new file mode 100644 index 000000000..e1bada989 --- /dev/null +++ b/http-testing/src/test/resources/parameterized_csv/hello_world_test_parameterized_csv_source_file_ignore_header.json @@ -0,0 +1,26 @@ +{ + "scenarioName": "Fetch and assert GitHub userIds by their userNames", + "steps": [ + { + "name": "get_user_details", + "url": "/users/${0}", + "method": "GET", + "request": { + }, + "assertions": { + "status": 200, + "body": { + "login" : "${0}", + "type" : "User", + "name" : "${1}", + "location" : "${2}", + "id" : "$EQ.${3}" + } + } + } + ], + "parameterized": { + "ignoreHeader": true, + "csvSource":"parameterized_csv/params_with_header.csv" + } +} diff --git a/http-testing/src/test/resources/parameterized_csv/hello_world_test_parameterized_csv_source_files.json b/http-testing/src/test/resources/parameterized_csv/hello_world_test_parameterized_csv_source_files.json new file mode 100644 index 000000000..bce4e78d2 --- /dev/null +++ b/http-testing/src/test/resources/parameterized_csv/hello_world_test_parameterized_csv_source_files.json @@ -0,0 +1,25 @@ +{ + "scenarioName": "Fetch and assert GitHub userIds by their userNames", + "steps": [ + { + "name": "get_user_details", + "url": "/users/${0}", + "method": "GET", + "request": { + }, + "assertions": { + "status": 200, + "body": { + "login" : "${0}", + "type" : "User", + "name" : "${1}", + "location" : "${2}", + "id" : "$EQ.${3}" + } + } + } + ], + "parameterized": { + "csvSource":"parameterized_csv/params.csv" + } +} diff --git a/http-testing/src/test/resources/parameterized_csv/params.csv b/http-testing/src/test/resources/parameterized_csv/params.csv new file mode 100644 index 000000000..faef91767 --- /dev/null +++ b/http-testing/src/test/resources/parameterized_csv/params.csv @@ -0,0 +1,2 @@ +octocat,The Octocat,San Francisco,583231 +siddhagalaxy,Sidd,UK,33847730 \ No newline at end of file diff --git a/http-testing/src/test/resources/parameterized_csv/params_with_header.csv b/http-testing/src/test/resources/parameterized_csv/params_with_header.csv new file mode 100644 index 000000000..3df16ea8d --- /dev/null +++ b/http-testing/src/test/resources/parameterized_csv/params_with_header.csv @@ -0,0 +1,3 @@ +user,name,city,userid +octocat,The Octocat,San Francisco,583231 +siddhagalaxy,Sidd,UK,33847730 \ No newline at end of file diff --git a/kafka-testing/src/main/java/org/jsmart/zerocode/zerocodejavaexec/utils/ExampleUtils.java b/kafka-testing/src/main/java/org/jsmart/zerocode/zerocodejavaexec/utils/ExampleUtils.java index 860646189..90be7b93d 100644 --- a/kafka-testing/src/main/java/org/jsmart/zerocode/zerocodejavaexec/utils/ExampleUtils.java +++ b/kafka-testing/src/main/java/org/jsmart/zerocode/zerocodejavaexec/utils/ExampleUtils.java @@ -1,5 +1,16 @@ package org.jsmart.zerocode.zerocodejavaexec.utils; +import org.jsmart.zerocode.core.kafka.consume.SeekTimestamp; + +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; + public class ExampleUtils { + public String seekTimestampToEpoch(SeekTimestamp seekTimestamp) throws ParseException { + DateFormat dateFormat = new SimpleDateFormat(seekTimestamp.getFormat()); + return String.valueOf(dateFormat.parse(seekTimestamp.getTimestamp()).toInstant().toEpochMilli()); + } + } diff --git a/kafka-testing/src/test/java/org/jsmart/zerocode/integration/tests/kafka/consume/KafkaConsumeSeekOffsetTest.java b/kafka-testing/src/test/java/org/jsmart/zerocode/integration/tests/kafka/consume/KafkaConsumeSeekOffsetTest.java index bb48d0f15..690be202b 100644 --- a/kafka-testing/src/test/java/org/jsmart/zerocode/integration/tests/kafka/consume/KafkaConsumeSeekOffsetTest.java +++ b/kafka-testing/src/test/java/org/jsmart/zerocode/integration/tests/kafka/consume/KafkaConsumeSeekOffsetTest.java @@ -26,4 +26,10 @@ public void testKafkaConsume_seekOffset() throws Exception { public void testKafkaConsume_seekOffsetLatest() throws Exception { } + @Test + @Scenario("kafka/consume/test_kafka_consume_seek_epoch_and_timestamp.json") + public void testKafkaConsume_seekEpochAndTimestamp() { + + } + } diff --git a/kafka-testing/src/test/resources/kafka/consume/test_kafka_consume_seek_epoch_and_timestamp.json b/kafka-testing/src/test/resources/kafka/consume/test_kafka_consume_seek_epoch_and_timestamp.json new file mode 100755 index 000000000..b71aac928 --- /dev/null +++ b/kafka-testing/src/test/resources/kafka/consume/test_kafka_consume_seek_epoch_and_timestamp.json @@ -0,0 +1,105 @@ +{ + "scenarioName": "Consume message after timestamp/epoch", + "steps": [ + { + "name": "load_kafka_before_timestamp", + "url": "kafka-topic:demo-seekTime", + "operation": "PRODUCE", + "request": { + "records": [ + { + "key": "${RANDOM.NUMBER}", + "value": "Before Timestamp 1" + }, + { + "key": "${RANDOM.NUMBER}", + "value": "Before Timestamp 2" + } + ] + }, + "assertions": { + "status": "Ok" + } + }, + { + "name": "load_timestamp_and_epoch", + "url": "org.jsmart.zerocode.zerocodejavaexec.utils.ExampleUtils", + "operation": "seekTimestampToEpoch", + "request": { + "timestamp": "${LOCAL.DATETIME.NOW:yyyy-MM-dd'T'HH:mm:ss.SSS}", + "format": "yyyy-MM-dd'T'HH:mm:ss.SSS" + }, + "assertions": {} + }, + { + "name": "load_kafka_after_timestamp", + "url": "kafka-topic:demo-seekTime", + "operation": "PRODUCE", + "request": { + "records": [ + { + "key": "${RANDOM.NUMBER}", + "value": "After Timestamp 1" + }, + { + "key": "${RANDOM.NUMBER}", + "value": "After Timestamp 2" + } + ] + }, + "assertions": { + "status": "Ok" + } + }, + { + "name": "consume_seekEpoch", + "url": "kafka-topic:demo-seekTime", + "operation": "CONSUME", + "request": { + "consumerLocalConfigs": { + "seekEpoch": "${$.load_timestamp_and_epoch.response}", + "commitSync": true, + "recordType": "RAW", + "showRecordsConsumed": true, + "maxNoOfRetryPollsOrTimeouts": 3 + } + }, + "verify": { + "records": [ + { + "value": "After Timestamp 1" + }, + { + "value": "After Timestamp 2" + } + ] + }}, + { + "name": "consume_seekTimestamp", + "url": "kafka-topic:demo-seekTime", + "operation": "CONSUME", + "request": { + "consumerLocalConfigs": { + "seekTimestamp": { + "timestamp": "${$.load_timestamp_and_epoch.request.timestamp}", + "format": "${$.load_timestamp_and_epoch.request.format}" + }, + "commitSync": true, + "recordType": "RAW", + "showRecordsConsumed": true, + "maxNoOfRetryPollsOrTimeouts": 3 + } + }, + "verify": { + "records": [ + { + "value": "After Timestamp 1" + }, + { + "value": "After Timestamp 2" + } + ] + } + } + ] +}