Skip to content

Commit

Permalink
Merge pull request #618 from authorjapps/ISSUE-603_masked_secrets
Browse files Browse the repository at this point in the history
ISSUES-603 Secrets not logged but used for API executions
  • Loading branch information
authorjapps authored Feb 7, 2024
2 parents a27729c + 0c56321 commit 48fd9d3
Show file tree
Hide file tree
Showing 12 changed files with 285 additions and 43 deletions.
5 changes: 5 additions & 0 deletions BUILDING.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ or
mvn -pl core clean test
```

## Runing Only The Integration tests(core)
Right click and run the "integrationtests" package. It picks the tests ending with "*Test.java"
<img width="632" alt="integration_tests_only_running_" src="https://github.com/authorjapps/zerocode/assets/12598420/6b6e8e33-16c1-43ce-8179-62a18e9a2290">


## With tests executed(kafka)
Some tests require a running Kafka (and some related components like kafka-rest, and kafka-schema-registry).

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,8 @@ public interface ZeroCodeAssertionsProcessor {

Step resolveJsonContent(Step thisStep, ScenarioExecutionState scenarioExecutionState);

String fieldMasksRemoved(String resolvedRequestJson);

String fieldMasksApplied(String resolvedRequestJson);

}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,28 @@
import com.google.inject.Inject;
import com.google.inject.name.Named;
import com.jayway.jsonpath.JsonPath;
import net.minidev.json.JSONArray;
import org.apache.commons.lang.text.StrSubstitutor;
import org.jsmart.zerocode.core.domain.Step;
import org.jsmart.zerocode.core.engine.assertion.FieldAssertionMatcher;
import org.jsmart.zerocode.core.engine.assertion.JsonAsserter;
import org.jsmart.zerocode.core.engine.assertion.array.ArrayIsEmptyAsserterImpl;
import org.jsmart.zerocode.core.engine.assertion.array.ArraySizeAsserterImpl;
import org.jsmart.zerocode.core.engine.assertion.field.FieldContainsStringAsserter;
import org.jsmart.zerocode.core.engine.assertion.field.FieldContainsStringIgnoreCaseAsserter;
import org.jsmart.zerocode.core.engine.assertion.field.FieldHasDateAfterValueAsserter;
import org.jsmart.zerocode.core.engine.assertion.field.FieldHasDateBeforeValueAsserter;
import org.jsmart.zerocode.core.engine.assertion.field.FieldHasEqualNumberValueAsserter;
import org.jsmart.zerocode.core.engine.assertion.field.FieldHasExactValueAsserter;
import org.jsmart.zerocode.core.engine.assertion.field.FieldHasGreaterThanValueAsserter;
import org.jsmart.zerocode.core.engine.assertion.field.FieldHasInEqualNumberValueAsserter;
import org.jsmart.zerocode.core.engine.assertion.field.FieldHasLesserThanValueAsserter;
import org.jsmart.zerocode.core.engine.assertion.field.FieldIsNotNullAsserter;
import org.jsmart.zerocode.core.engine.assertion.field.FieldIsNullAsserter;
import org.jsmart.zerocode.core.engine.assertion.field.FieldIsOneOfValueAsserter;
import org.jsmart.zerocode.core.engine.assertion.field.FieldMatchesCustomAsserter;
import org.jsmart.zerocode.core.engine.assertion.field.FieldMatchesRegexPatternAsserter;

import java.io.IOException;
import java.math.BigDecimal;
import java.time.LocalDateTime;
Expand All @@ -19,37 +41,47 @@
import java.util.List;
import java.util.Map;
import java.util.Properties;
import net.minidev.json.JSONArray;
import org.apache.commons.lang.text.StrSubstitutor;
import org.jsmart.zerocode.core.domain.Step;
import org.jsmart.zerocode.core.engine.assertion.FieldAssertionMatcher;
import org.jsmart.zerocode.core.engine.assertion.JsonAsserter;
import org.jsmart.zerocode.core.engine.assertion.array.ArrayIsEmptyAsserterImpl;
import org.jsmart.zerocode.core.engine.assertion.array.ArraySizeAsserterImpl;
import org.jsmart.zerocode.core.engine.assertion.field.*;
import org.jsmart.zerocode.core.utils.SmartUtils;

import static java.lang.Integer.valueOf;
import static java.lang.String.format;
import static org.apache.commons.lang.StringEscapeUtils.escapeJava;
import static org.apache.commons.lang.StringUtils.substringBetween;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.*;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_LOCAL_DATETIME_AFTER;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_LOCAL_DATETIME_BEFORE;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_PATH_SIZE;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_CONTAINS_STRING;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_CONTAINS_STRING_IGNORE_CASE;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_CUSTOM_ASSERT;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_EMPTY_ARRAY;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_EQUAL_TO_NUMBER;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_GREATER_THAN;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_IS_NOT_NULL;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_IS_NULL;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_IS_ONE_OF;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_LESSER_THAN;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_MATCHES_STRING;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_NOT_EQUAL_TO_NUMBER;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_NOT_NULL;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_NULL;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.ASSERT_VALUE_ONE_OF;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeAssertionTokens.RAW_BODY;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.$VALUE;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.JSON_CONTENT;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.JSON_PAYLOAD_FILE;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.YAML_PAYLOAD_FILE;
import static org.jsmart.zerocode.core.utils.FieldTypeConversionUtils.deepTypeCast;
import static org.jsmart.zerocode.core.utils.FieldTypeConversionUtils.fieldTypes;
import static org.jsmart.zerocode.core.utils.PropertiesProviderUtils.loadAbsoluteProperties;
import static org.jsmart.zerocode.core.utils.SmartUtils.checkDigNeeded;
import static org.jsmart.zerocode.core.utils.SmartUtils.getJsonFilePhToken;
import static org.jsmart.zerocode.core.utils.SmartUtils.isValidAbsolutePath;
import static org.jsmart.zerocode.core.utils.SmartUtils.readJsonAsString;
import static org.jsmart.zerocode.core.utils.SmartUtils.readYamlAsString;
import static org.jsmart.zerocode.core.utils.SmartUtils.checkDigNeeded;;
import static org.jsmart.zerocode.core.utils.SmartUtils.getJsonFilePhToken;;
import static org.jsmart.zerocode.core.utils.TokenUtils.getMasksRemoved;
import static org.jsmart.zerocode.core.utils.TokenUtils.getMasksReplaced;
import static org.jsmart.zerocode.core.utils.TokenUtils.getTestCaseTokens;
import static org.jsmart.zerocode.core.utils.TokenUtils.populateParamMap;
import static org.slf4j.LoggerFactory.getLogger;

;
;

public class ZeroCodeAssertionsProcessorImpl implements ZeroCodeAssertionsProcessor {

private static final org.slf4j.Logger LOGGER = getLogger(ZeroCodeAssertionsProcessorImpl.class);
Expand Down Expand Up @@ -380,6 +412,16 @@ public Step resolveJsonContent(Step thisStep, ScenarioExecutionState scenarioExe
}
}

@Override
public String fieldMasksRemoved(String resolvedRequestJson) {
return getMasksRemoved(resolvedRequestJson);
}

@Override
public String fieldMasksApplied(String resolvedRequestJson) {
return getMasksReplaced(resolvedRequestJson);
}

private void loadAnnotatedHostProperties() {
try {
if(isValidAbsolutePath(hostFileName)){
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,15 @@ public class ZeroCodeValueTokens {
public static final String $VALUE = ".$VALUE";
public static final String ABS_PATH = "ABS.PATH:";
public static final String JSON_CONTENT = "JSON.CONTENT:";
public static final String MASKED = "MASKED:";
public static final String MASKED_STR = "***masked***";


public static Map<String, Object> globalTokenCache = new HashMap<>();

public static List<String> getKnownTokens() {
return asList(
MASKED,
PREFIX_ASU,
RANDOM_NUMBER,
GLOBAL_RANDOM_NUMBER,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.RESULT_FAIL;
import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.RESULT_PASS;
import static org.jsmart.zerocode.core.constants.ZeroCodeReportConstants.TEST_STEP_CORRELATION_ID;
import static org.jsmart.zerocode.core.utils.TokenUtils.getMasksReplaced;

public class ZerocodeCorrelationshipLogger {
private static final String DISPLAY_DEMARCATION_ = "\n--------- " + TEST_STEP_CORRELATION_ID + " %s ---------";
Expand Down Expand Up @@ -139,11 +140,12 @@ public void print() {
buildResponseDelay();

String customLog = responseLogBuilder.getCustomLog();
String assertionsWithMaskRemoved = getMasksReplaced(responseLogBuilder.getAssertion());
logger.warn(format("%s %s \n*Response delay:%s milli-secs \n%s \n%s \n-done-\n",
requestLogBuilder.toString(),
responseLogBuilder.toString(),
responseDelay,
"---------> Expected Response: <----------\n" + responseLogBuilder.getAssertion(),
"---------> Expected Response: <----------\n" + assertionsWithMaskRemoved,
customLog == null ? "" : "---------> Custom Log: <----------\n" +customLog
)
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,13 @@ private String executeApi(String logPrefixRelationshipId,
// --------------------------------
url = zeroCodeAssertionsProcessor.resolveStringJson(url, scenarioExecutionState.getResolvedScenarioState());

// ------------------------------------------------
// 1) Removed the MASKED wrapper for API execution (For logging)
// 2) Replace the MASKED field with masked content (For API executions)
// ------------------------------------------------
String resolvedRequestJsonMaskRemoved = zeroCodeAssertionsProcessor.fieldMasksRemoved(resolvedRequestJson);
String resolvedRequestJsonMaskApplied = zeroCodeAssertionsProcessor.fieldMasksApplied(resolvedRequestJson);

final LocalDateTime requestTimeStamp = LocalDateTime.now();

String executionResult;
Expand All @@ -428,9 +435,9 @@ private String executeApi(String logPrefixRelationshipId,
.url(url)
.method(operationName)
.id(stepId)
.request(prettyPrintJson(resolvedRequestJson));
.request(prettyPrintJson(resolvedRequestJsonMaskApplied));

executionResult = apiExecutor.executeHttpApi(url, operationName, resolvedRequestJson);
executionResult = apiExecutor.executeHttpApi(url, operationName, resolvedRequestJsonMaskRemoved);
break;

case JAVA_CALL:
Expand All @@ -441,10 +448,10 @@ private String executeApi(String logPrefixRelationshipId,
.id(stepId)
.url(url)
.method(operationName)
.request(prettyPrintJson(resolvedRequestJson));
.request(prettyPrintJson(resolvedRequestJsonMaskApplied));

url = apiTypeUtils.getQualifiedJavaApi(url);
executionResult = apiExecutor.executeJavaOperation(url, operationName, resolvedRequestJson);
executionResult = apiExecutor.executeJavaOperation(url, operationName, resolvedRequestJsonMaskRemoved);
break;

case KAFKA_CALL:
Expand All @@ -459,10 +466,10 @@ private String executeApi(String logPrefixRelationshipId,
.url(url)
.method(operationName.toUpperCase())
.id(stepId)
.request(prettyPrintJson(resolvedRequestJson));
.request(prettyPrintJson(resolvedRequestJsonMaskApplied));

String topicName = url.substring(KAFKA_TOPIC.length());
executionResult = apiExecutor.executeKafkaService(kafkaServers, topicName, operationName, resolvedRequestJson, scenarioExecutionState);
executionResult = apiExecutor.executeKafkaService(kafkaServers, topicName, operationName, resolvedRequestJsonMaskRemoved, scenarioExecutionState);
break;

case NONE:
Expand All @@ -473,14 +480,14 @@ private String executeApi(String logPrefixRelationshipId,
.id(stepId)
.url(url)
.method(operationName)
.request(prettyPrintJson(resolvedRequestJson));
.request(prettyPrintJson(resolvedRequestJsonMaskApplied));

executionResult = prettyPrintJson(resolvedRequestJson);
executionResult = prettyPrintJson(resolvedRequestJsonMaskApplied);
break;

default:
throw new RuntimeException("Oops! API Type Undecided. If it is intentional, " +
"then keep the value as empty to receive the request in the response");
"then keep the value as empty to receive the request as response");
}

return executionResult;
Expand Down Expand Up @@ -534,6 +541,8 @@ private int deriveScenarioLoopTimes(ScenarioSpec scenario) {
private List<FieldAssertionMatcher> compareStepResults(Step thisStep, String actualResult, String expectedResult, String resolvedScenarioState) {
List<FieldAssertionMatcher> failureResults = new ArrayList<>();

expectedResult = zeroCodeAssertionsProcessor.fieldMasksRemoved(expectedResult);

// --------------------
// Validators (pyrest)
// --------------------
Expand Down
41 changes: 24 additions & 17 deletions core/src/main/java/org/jsmart/zerocode/core/utils/TokenUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,23 +19,7 @@
import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
import static org.apache.commons.lang.StringEscapeUtils.escapeJava;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.ABS_PATH;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.GLOBAL_RANDOM_NUMBER;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.GQL_FILE;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.LOCALDATETIME_NOW;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.LOCALDATE_TODAY;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.RANDOM_NUMBER;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.RANDOM_NUMBER_FIXED;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.RANDOM_STRING_ALPHA;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.RANDOM_STRING_ALPHA_NUMERIC;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.RANDOM_UU_ID;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.RANDOM_UU_ID_FIXED;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.STATIC_ALPHABET;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.SYSTEM_ENV;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.SYSTEM_PROPERTY;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.XML_FILE;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.getKnownTokens;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.globalTokenCache;
import static org.jsmart.zerocode.core.engine.tokens.ZeroCodeValueTokens.*;

public class TokenUtils {

Expand Down Expand Up @@ -162,6 +146,29 @@ public static List<String> getTestCaseTokens(String aString) {
return keyTokens;
}

public static String getMasksReplaced(String aString) {
String regex = "\\$\\{MASKED:([^\\}]*)\\}";
Matcher maskMatcher = Pattern.compile(regex).matcher(aString);
while(maskMatcher.find()) {
String foundMatch = maskMatcher.group(0);
aString = aString.replace(foundMatch, MASKED_STR);
}

return aString;
}

public static String getMasksRemoved(String aString) {
String regex = "\\$\\{MASKED:([^\\}]*)\\}";
Matcher maskMatcher = Pattern.compile(regex).matcher(aString);
while(maskMatcher.find()) {
String foundFullMatch = maskMatcher.group(0);
String innerContent = maskMatcher.group(1);
aString = aString.replace(foundFullMatch, innerContent);
}

return aString;
}

public static String createRandomAlphaString(int length) {
return randomAlphabetic(length);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.is;
import static org.jsmart.zerocode.core.utils.TokenUtils.absolutePathOf;
import static org.jsmart.zerocode.core.utils.TokenUtils.getMasksReplaced;
import static org.jsmart.zerocode.core.utils.TokenUtils.getMasksRemoved;
import static org.jsmart.zerocode.core.utils.TokenUtils.resolveKnownTokens;
import static org.junit.Assert.*;

Expand Down Expand Up @@ -169,4 +171,48 @@ public void testAbsolutePathOf() {
assertThat(absolutePathOf("unit_test_files/jks_files/dummy_key_store.jks"),
containsString("zerocode/core/target/test-classes/unit_test_files/jks_files/dummy_key_store.jks"));
}


@Test
public void testGetMaskedTokensReplaced_multipleOccurrences(){
assertEquals("This is a ***masked*** message with ***masked*** tokens.", getMasksReplaced("This is a ${MASKED:secret} message with ${MASKED:masked} tokens."));
}

@Test
public void testGetMaskedTokensReplaced_noOccurrences(){
assertEquals("This string has no masked tokens.", getMasksReplaced("This string has no masked tokens."));
}

@Test
public void testGetMaskedTokensReplaced_emptyString(){
assertEquals("", getMasksReplaced(""));
}

@Test
public void testGetMaskedTokensReplaced_specialCharacters(){
assertEquals("***masked*** and ***masked***", getMasksReplaced("${MASKED:abc@123} and ${MASKED:!@#$%^}"));
}

@Test
public void testGetMaskedTokensRemoved_multipleOccurrences(){
assertEquals("This is a secret message with masked tokens.", getMasksRemoved("This is a ${MASKED:secret} message with ${MASKED:masked} tokens."));
}

@Test
public void testGetMaskedTokensRemoved_noOccurrences(){
assertEquals("This string has no masked tokens.", getMasksRemoved("This string has no masked tokens."));
}

@Test
public void testGetMaskedTokensRemoved_emptyString(){
assertEquals("", getMasksRemoved(""));
}

@Test
public void testGetMaskedTokensRemoved_specialCharacters(){
assertEquals("abc@123 and !@#$%^", getMasksRemoved("${MASKED:abc@123} and ${MASKED:!@#$%^}"));
}



}
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package org.jsmart.zerocode.integrationtests.masked;

import org.jsmart.zerocode.core.domain.HostProperties;
import org.jsmart.zerocode.core.domain.Scenario;
import org.jsmart.zerocode.core.tests.customrunner.TestOnlyZeroCodeUnitRunner;
import org.junit.Test;
import org.junit.runner.RunWith;

@HostProperties(host="http://localhost", port=9998, context = "")
@RunWith(TestOnlyZeroCodeUnitRunner.class)
public class MaskedSecretsInMemoryTest {

/**
* Mock end points are in test/resources: simulators/test_purpose_end_points.json.
* @RunWith(TestOnlyZeroCodeUnitRunner.class) : starts these mocks first before running the tests
*/

@Test
@Scenario("integration_test_files/masked/password_or_secrets_masked_in_log_n_console.json")
public void testSecretPrinted_masked() throws Exception {
}

@Test
@Scenario("integration_test_files/masked/bearer_token_or_secret_masked_reuse_example_.json")
public void testSecretPrintedAssertions_masked() throws Exception {

}

}



Loading

0 comments on commit 48fd9d3

Please sign in to comment.