diff --git a/syncers/auth_history_syncer/pom.xml b/syncers/auth_history_syncer/pom.xml index 8e661e6a487..8f3187008f1 100644 --- a/syncers/auth_history_syncer/pom.xml +++ b/syncers/auth_history_syncer/pom.xml @@ -32,7 +32,7 @@ Authentication History Syncer (Cloudwatch logs to DynamoDB) - 0.7738 + 0.8593 2.2.1 1.0.392 4.13.2 diff --git a/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/AuthHistoryDynamoDBRecord.java b/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/AuthHistoryDynamoDBRecord.java index c6c946601fd..45be13513a1 100644 --- a/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/AuthHistoryDynamoDBRecord.java +++ b/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/AuthHistoryDynamoDBRecord.java @@ -36,19 +36,21 @@ public class AuthHistoryDynamoDBRecord { private String principalName; private String endpoint; private String timestamp; + private String operation; private long ttl; public AuthHistoryDynamoDBRecord() { - } - public AuthHistoryDynamoDBRecord(String primaryKey, String uriDomain, String principalDomain, String principalName, String endpoint, String timestamp, long ttl) { + public AuthHistoryDynamoDBRecord(String primaryKey, String uriDomain, String principalDomain, + String principalName, String endpoint, String timestamp, String operation, long ttl) { this.primaryKey = primaryKey; this.uriDomain = uriDomain; this.principalDomain = principalDomain; this.principalName = principalName; this.endpoint = endpoint; this.timestamp = timestamp; + this.operation = operation; this.ttl = ttl; } @@ -83,6 +85,10 @@ public String getTimestamp() { return timestamp; } + public String getOperation() { + return operation; + } + // Set methods must exist for @DynamoDbBean successful marshalling public void setPrimaryKey(String primaryKey) { this.primaryKey = primaryKey; @@ -102,6 +108,9 @@ public void setEndpoint(String endpoint) { public void setTimestamp(String timestamp) { this.timestamp = timestamp; } + public void setOperation(String operation) { + this.operation = operation; + } public void setTtl(long ttl) { this.ttl = ttl; } @@ -116,11 +125,7 @@ public boolean equals(Object o) { } AuthHistoryDynamoDBRecord record = (AuthHistoryDynamoDBRecord) o; - if (getPrimaryKey() == null ? record.getPrimaryKey() != null : !getPrimaryKey().equals(record.getPrimaryKey())) { - return false; - } - - return true; + return getPrimaryKey() == null ? record.getPrimaryKey() == null : getPrimaryKey().equals(record.getPrimaryKey()); } @Override @@ -137,6 +142,7 @@ public String toString() { ", principalName='" + principalName + '\'' + ", endpoint='" + endpoint + '\'' + ", timestamp='" + timestamp + '\'' + + ", operation='" + operation + '\'' + ", ttl=" + ttl + '}'; } diff --git a/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/AuthHistorySyncer.java b/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/AuthHistorySyncer.java index 0d66c2316c3..61321fa86f7 100644 --- a/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/AuthHistorySyncer.java +++ b/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/AuthHistorySyncer.java @@ -94,7 +94,7 @@ PrivateKeyStore getPrivateKeyStore() { return pkeyFactory.create(); } - private AuthHistoryFetcher getAuthHistoryFetcher(PrivateKeyStore privateKeyStore, String region) throws Exception { + AuthHistoryFetcher getAuthHistoryFetcher(PrivateKeyStore privateKeyStore, String region) throws Exception { String authHistoryFetcherFactoryClass = System.getProperty(PROP_AUTH_HISTORY_FETCH_FACTORY_CLASS); if (authHistoryFetcherFactoryClass == null) { System.out.println("Error: " + PROP_AUTH_HISTORY_FETCH_FACTORY_CLASS + " system property is mandatory"); @@ -110,7 +110,7 @@ private AuthHistoryFetcher getAuthHistoryFetcher(PrivateKeyStore privateKeyStore } } - private AuthHistorySender getAuthHistorySender(PrivateKeyStore privateKeyStore, String region) throws ClassNotFoundException, InstantiationException, IllegalAccessException { + AuthHistorySender getAuthHistorySender(PrivateKeyStore privateKeyStore, String region) throws ClassNotFoundException, InstantiationException, IllegalAccessException { String authHistorySenderFactoryClass = System.getProperty(PROP_AUTH_HISTORY_SEND_FACTORY_CLASS, PROP_AUTH_HISTORY_SEND_FACTORY_CLASS_DEFAULT); AuthHistorySenderFactory authHistorySenderFactory; try { diff --git a/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/AuthHistorySyncerConsts.java b/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/AuthHistorySyncerConsts.java index db93dba0e05..9573837c86d 100644 --- a/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/AuthHistorySyncerConsts.java +++ b/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/AuthHistorySyncerConsts.java @@ -34,4 +34,7 @@ private AuthHistorySyncerConsts() { public static final String PROP_DYNAMODB_EXTERNAL_ID = "auth_history_syncer.dynamodb_external_id"; public static final String PROP_DYNAMODB_MIN_EXPIRY_TIME = "auth_history_syncer.dynamodb_min_expiry_time"; public static final String PROP_DYNAMODB_MAX_EXPIRY_TIME = "auth_history_syncer.dynamodb_max_expiry_time"; + + public static final String PROP_CLOUDWATCH_ZMS_LOG_GROUP = "auth_history_syncer.cloudwatch_zms_log_group"; + public static final String PROP_CLOUDWATCH_ZTS_LOG_GROUP = "auth_history_syncer.cloudwatch_zts_log_group"; } diff --git a/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/LogsParserUtils.java b/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/LogsParserUtils.java index 046ccfc4a1c..cd2d3104921 100644 --- a/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/LogsParserUtils.java +++ b/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/LogsParserUtils.java @@ -18,38 +18,68 @@ package com.yahoo.athenz.syncer.auth.history; +import com.yahoo.athenz.auth.util.AthenzUtils; + import java.net.MalformedURLException; import java.util.regex.Matcher; import java.util.regex.Pattern; public class LogsParserUtils { + + private static class PatternType { + public Pattern pattern; + public String operation; + + public PatternType(Pattern pattern, final String operation) { + this.pattern = pattern; + this.operation = operation; + } + } + private static final String DOMAIN_REGEX = "([a-zA-Z0-9_][a-zA-Z0-9_-]*\\.)*[a-zA-Z0-9_][a-zA-Z0-9_-]*"; - // regex: - // /domain/{domainName}/token - // /domain/{domainName}/role/{roleName}/token - // /access/domain/{domainName}/role/{roleName}/principal/{principal} - // /access/domain/{domainName}/principal/{principal} - private static final Pattern DOMAIN_PATH_PARAM_PATTERN = Pattern.compile("/domain/(" + DOMAIN_REGEX + ")/(token|role/|principal/)", Pattern.MULTILINE); + // regex: /domain/{domainName}/token + private static final PatternType ROLE_TOKEN_PATTERN = new PatternType( + Pattern.compile("/domain/(" + DOMAIN_REGEX + ")/token", Pattern.MULTILINE), "role-token"); + + // regex: /domain/{domainName}/role/{roleName}/token + private static final PatternType ROLE_CERT_OLD_PATTERN = new PatternType( + Pattern.compile("/domain/(" + DOMAIN_REGEX + ")/role/", Pattern.MULTILINE), "role-cert"); + + // regex: /access/domain/{domainName}/role/{roleName}/principal/{principal} + private static final PatternType ACCESS_ROLE_PATTERN = new PatternType( + Pattern.compile("/access/domain/(" + DOMAIN_REGEX + ")/role/", Pattern.MULTILINE), "access-check"); + + // regex: /access/domain/{domainName}/principal/{principal} + private static final PatternType ACCESS_PRINCIPAL_PATTERN = new PatternType( + Pattern.compile("/access/domain/(" + DOMAIN_REGEX + ")/principal/", Pattern.MULTILINE), "access-check"); // regex: /oauth2/token - private static final Pattern OAUTH_TOKEN_PATTERN = Pattern.compile("/oauth2/token.*scope=(" + DOMAIN_REGEX + ").*", Pattern.MULTILINE); + private static final PatternType OAUTH_TOKEN_PATTERN = new PatternType( + Pattern.compile("/oauth2/token.*scope=(" + DOMAIN_REGEX + ").*", Pattern.MULTILINE), "access-token"); // regex: (alternate domain for cross-domain trust relation) // /access/{action}?domain={domain} // /access/{action}/{resource}?domain={domain} - private static final Pattern ACCESS_RESOURCE_ALT_DOMAIN_PATTERN = Pattern.compile("/access/.*domain=(" + DOMAIN_REGEX + ").*", Pattern.MULTILINE); + private static final PatternType ACCESS_RESOURCE_ALT_DOMAIN_PATTERN = new PatternType( + Pattern.compile("/access/.*domain=(" + DOMAIN_REGEX + ").*", Pattern.MULTILINE), "access-check"); // regex: /access/{action}?resource={domain}:{resource} - private static final Pattern ACCESS_RESOURCE_PATTERN_QUERY_PARAM = Pattern.compile("/access/.*resource=(" + DOMAIN_REGEX + ").*", Pattern.MULTILINE); + private static final PatternType ACCESS_RESOURCE_PATTERN_QUERY_PARAM = new PatternType( + Pattern.compile("/access/.*resource=(" + DOMAIN_REGEX + ").*", Pattern.MULTILINE), "access-check"); // regex: /access/{action}/{resource} - private static final Pattern ACCESS_RESOURCE_PATTERN = Pattern.compile("/access/.*/(" + DOMAIN_REGEX + "):.*", Pattern.MULTILINE); + private static final PatternType ACCESS_RESOURCE_PATTERN = new PatternType( + Pattern.compile("/access/.*/(" + DOMAIN_REGEX + "):.*", Pattern.MULTILINE), "access-check"); // regex: /rolecert?roleName={domain}:role.{role} - private static final Pattern ROLE_CERT_PATTERN = Pattern.compile("/rolecert.roleName=(" + DOMAIN_REGEX + "):role.*", Pattern.MULTILINE); + private static final PatternType ROLE_CERT_PATTERN = new PatternType( + Pattern.compile("/rolecert.roleName=(" + DOMAIN_REGEX + "):role.*", Pattern.MULTILINE), "role-cert"); - private static final Pattern[] PATTERNS = {DOMAIN_PATH_PARAM_PATTERN, OAUTH_TOKEN_PATTERN, ACCESS_RESOURCE_ALT_DOMAIN_PATTERN, ACCESS_RESOURCE_PATTERN_QUERY_PARAM, ACCESS_RESOURCE_PATTERN, ROLE_CERT_PATTERN}; + private static final PatternType[] PATTERNS = {ROLE_TOKEN_PATTERN, ROLE_CERT_OLD_PATTERN, + ACCESS_ROLE_PATTERN, ACCESS_PRINCIPAL_PATTERN, OAUTH_TOKEN_PATTERN, + ACCESS_RESOURCE_ALT_DOMAIN_PATTERN, ACCESS_RESOURCE_PATTERN_QUERY_PARAM, + ACCESS_RESOURCE_PATTERN, ROLE_CERT_PATTERN}; private static final String PROP_TTL = "auth_history_syncer.ttl"; private static final String PROP_TTL_DEFAULT = "720"; // 30 days @@ -58,45 +88,34 @@ public class LogsParserUtils { private static final long EXPIRY_TIME = 3660 * EXPIRY_HOURS; public static AuthHistoryDynamoDBRecord getRecordFromLogEvent(String message) throws MalformedURLException { + String[] split = message.split("\\s+"); - String principalDomain = getPrincipalDomain(split[2]); - String principalName = getPrincipalName(split[2]); - String endpoint = split[5].substring(1) + " " + split[6]; - String timestamp = split[3].substring(1); - String uriDomain = getDomainFromEndpoint(split[6]); - String primaryKey = generatePrimaryKey(uriDomain, principalDomain, principalName); - return new AuthHistoryDynamoDBRecord(primaryKey, uriDomain, principalDomain, principalName, endpoint, timestamp, System.currentTimeMillis() / 1000L + EXPIRY_TIME); + AuthHistoryDynamoDBRecord record = createRecordObject(split[6]); + record.setPrincipalDomain(AthenzUtils.extractPrincipalDomainName(split[2])); + record.setPrincipalName(AthenzUtils.extractPrincipalServiceName(split[2])); + record.setEndpoint(split[5].substring(1) + " " + split[6]); + record.setTimestamp(split[3].substring(1)); + record.setPrimaryKey(generatePrimaryKey(record.getUriDomain(), record.getPrincipalDomain(), + record.getPrincipalName())); + record.setTtl(System.currentTimeMillis() / 1000L + EXPIRY_TIME); + return record; } public static String generatePrimaryKey(String uriDomain, String principalDomain, String principalName) { return uriDomain + ":" + principalDomain + ":" + principalName; } - private static String getDomainFromEndpoint(String endpoint) throws MalformedURLException { - for (Pattern pattern : PATTERNS) { - Matcher m = pattern.matcher(endpoint); + private static AuthHistoryDynamoDBRecord createRecordObject(final String endpoint) throws MalformedURLException { + for (PatternType patternType : PATTERNS) { + Matcher m = patternType.pattern.matcher(endpoint); if (m.find()) { - return m.group(1); + AuthHistoryDynamoDBRecord record = new AuthHistoryDynamoDBRecord(); + record.setUriDomain(m.group(1)); + record.setOperation(patternType.operation); + return record; } } throw new MalformedURLException("Failed to locate domain at endpoint: " + endpoint); } - - - private static String getPrincipalDomain(String principal) { - int n = principal.lastIndexOf('.'); - if (n <= 0 || n == principal.length() - 1) { - return null; - } - return principal.substring(0, n); - } - - private static String getPrincipalName(String principal) { - int n = principal.lastIndexOf('.'); - if (n <= 0 || n == principal.length() - 1) { - return null; - } - return principal.substring(n + 1); - } } diff --git a/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/impl/AwsAuthHistoryFetcher.java b/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/impl/AwsAuthHistoryFetcher.java index 613d7af1142..5fab5473003 100644 --- a/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/impl/AwsAuthHistoryFetcher.java +++ b/syncers/auth_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/impl/AwsAuthHistoryFetcher.java @@ -18,10 +18,7 @@ package com.yahoo.athenz.syncer.auth.history.impl; -import com.yahoo.athenz.syncer.auth.history.AuthHistoryDynamoDBRecord; -import com.yahoo.athenz.syncer.auth.history.AuthHistoryFetcher; -import com.yahoo.athenz.syncer.auth.history.CloudWatchClientFactory; -import com.yahoo.athenz.syncer.auth.history.LogsParserUtils; +import com.yahoo.athenz.syncer.auth.history.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.cloudwatch.model.CloudWatchException; @@ -35,24 +32,34 @@ import java.util.Set; public class AwsAuthHistoryFetcher implements AuthHistoryFetcher { + private static final Logger LOGGER = LoggerFactory.getLogger(AwsAuthHistoryFetcher.class); private final CloudWatchLogsClient cloudWatchLogsClient; + private final String zmsLogGroup = System.getProperty(AuthHistorySyncerConsts.PROP_CLOUDWATCH_ZMS_LOG_GROUP, + "athenz-zms-service-access"); + private final String ztsLogGroup = System.getProperty(AuthHistorySyncerConsts.PROP_CLOUDWATCH_ZTS_LOG_GROUP, + "athenz-zts-service-access"); + public AwsAuthHistoryFetcher(CloudWatchClientFactory cloudWatchClientFactory) { this.cloudWatchLogsClient = cloudWatchClientFactory.create(); } /** * - * @param startTime - the start of the time range, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC (for example, 1620940080) - * @param endTime - the end of the time range, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC (for example, 1620940080) - * @param useFilterPattern - if true, filter access events in request. False means that all events in the time range will return. - * @return - authorization checks and token requests history ready to be pushed to a data store. On error return null. + * @param startTime - the start of the time range, expressed as the number of milliseconds + * after Jan 1, 1970 00:00:00 UTC (for example, 1620940080) + * @param endTime - the end of the time range, expressed as the number of milliseconds + * after Jan 1, 1970 00:00:00 UTC (for example, 1620940080) + * @param useFilterPattern - if true, filter access events in request. False means that + * all events in the time range will return. + * @return - authorization checks and token requests history ready to be pushed to a + * data store. On error return null. */ @Override public Set getLogs(Long startTime, Long endTime, boolean useFilterPattern) { - Set zmsLogs = getLogs("athenz-zms-service-access", startTime, endTime, useFilterPattern); - Set ztsLogs = getLogs("athenz-zts-service-access", startTime, endTime, useFilterPattern); + Set zmsLogs = getLogs(zmsLogGroup, startTime, endTime, useFilterPattern); + Set ztsLogs = getLogs(ztsLogGroup, startTime, endTime, useFilterPattern); Set allRecords = new HashSet<>(); if (zmsLogs != null && !zmsLogs.isEmpty()) { allRecords.addAll(zmsLogs); @@ -66,13 +73,20 @@ public Set getLogs(Long startTime, Long endTime, bool /** * * @param logGroup - Log group name - * @param startTime - the start of the time range, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC (for example, 1620940080) - * @param endTime - the end of the time range, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC (for example, 1620940080) - * @param useFilterPattern - if true, filter access events in request. False means that all events in the time range will return. - * @return - authorization checks and token requests history ready to be pushed to a data store. On error return null. + * @param startTime - the start of the time range, expressed as the number of milliseconds + * after Jan 1, 1970 00:00:00 UTC (for example, 1620940080) + * @param endTime - the end of the time range, expressed as the number of milliseconds + * after Jan 1, 1970 00:00:00 UTC (for example, 1620940080) + * @param useFilterPattern - if true, filter access events in request. False means that + * all events in the time range will return. + * @return - authorization checks and token requests history ready to be pushed to a + * data store. On error return null. */ private Set getLogs(String logGroup, Long startTime, Long endTime, boolean useFilterPattern) { - LOGGER.info("Getting logs from logGroup {}, startTime(milli): {}, endTime(milli): {}, useFilterPattern: {}", logGroup, startTime, endTime, useFilterPattern); + + LOGGER.info("Getting logs from logGroup {}, startTime(milli): {}, endTime(milli): {}, useFilterPattern: {}", + logGroup, startTime, endTime, useFilterPattern); + try { String nextToken = null; Set filteredEvents = new HashSet<>(); diff --git a/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/AuthHistoryDynamoDBRecordTest.java b/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/AuthHistoryDynamoDBRecordTest.java new file mode 100644 index 00000000000..407c5064856 --- /dev/null +++ b/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/AuthHistoryDynamoDBRecordTest.java @@ -0,0 +1,81 @@ +/* + * + * * Copyright The Athenz Authors + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.yahoo.athenz.syncer.auth.history; + +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + +public class AuthHistoryDynamoDBRecordTest { + + @Test + public void testAuthHistoryDynamoDBRecord() { + AuthHistoryDynamoDBRecord authHistoryDynamoDBRecord = new AuthHistoryDynamoDBRecord(); + authHistoryDynamoDBRecord.setPrimaryKey("primaryKeyTest"); + authHistoryDynamoDBRecord.setUriDomain("uriDomainTest"); + authHistoryDynamoDBRecord.setPrincipalDomain("principalDomainTest"); + authHistoryDynamoDBRecord.setPrincipalName("principalNameTest"); + authHistoryDynamoDBRecord.setEndpoint("endpointTest"); + authHistoryDynamoDBRecord.setTimestamp("timestampTest"); + authHistoryDynamoDBRecord.setOperation("access-check"); + authHistoryDynamoDBRecord.setTtl(1000L); + + assertEquals(authHistoryDynamoDBRecord.getPrimaryKey(), "primaryKeyTest"); + assertEquals(authHistoryDynamoDBRecord.getUriDomain(), "uriDomainTest"); + assertEquals(authHistoryDynamoDBRecord.getPrincipalDomain(), "principalDomainTest"); + assertEquals(authHistoryDynamoDBRecord.getPrincipalName(), "principalNameTest"); + assertEquals(authHistoryDynamoDBRecord.getEndpoint(), "endpointTest"); + assertEquals(authHistoryDynamoDBRecord.getTimestamp(), "timestampTest"); + assertEquals(authHistoryDynamoDBRecord.getOperation(), "access-check"); + assertEquals(authHistoryDynamoDBRecord.getTtl(), 1000L); + + AuthHistoryDynamoDBRecord authHistoryDynamoDBRecord2 = new AuthHistoryDynamoDBRecord("primaryKeyTest", + "uriDomainTest", "principalDomainTest", "principalNameTest", "endpointTest", + "timestampTest", "access-check", 1000L); + assertEquals(authHistoryDynamoDBRecord, authHistoryDynamoDBRecord2); + + assertEquals(authHistoryDynamoDBRecord, authHistoryDynamoDBRecord); + assertFalse(authHistoryDynamoDBRecord.equals(null)); + assertFalse(authHistoryDynamoDBRecord.equals("test")); + + assertEquals(authHistoryDynamoDBRecord.hashCode(), authHistoryDynamoDBRecord2.hashCode()); + assertEquals(authHistoryDynamoDBRecord.toString(), + "AuthHistoryDynamoDBRecord{primaryKey='primaryKeyTest', uriDomain='uriDomainTest', principalDomain='principalDomainTest', principalName='principalNameTest', endpoint='endpointTest', timestamp='timestampTest', operation='access-check', ttl=1000}"); + + // records equal with no primary key + AuthHistoryDynamoDBRecord authHistoryDynamoDBRecord3 = new AuthHistoryDynamoDBRecord(); + AuthHistoryDynamoDBRecord authHistoryDynamoDBRecord4 = new AuthHistoryDynamoDBRecord(); + assertTrue(authHistoryDynamoDBRecord3.equals(authHistoryDynamoDBRecord4)); + + // same values - match + authHistoryDynamoDBRecord3.setPrimaryKey("primaryKeyTest"); + authHistoryDynamoDBRecord4.setPrimaryKey("primaryKeyTest"); + assertTrue(authHistoryDynamoDBRecord3.equals(authHistoryDynamoDBRecord4)); + + // different values - no match + authHistoryDynamoDBRecord3.setPrimaryKey("primaryKeyTest"); + authHistoryDynamoDBRecord4.setPrimaryKey("primaryKeyTest2"); + assertFalse(authHistoryDynamoDBRecord3.equals(authHistoryDynamoDBRecord4)); + + // one null value - no match + authHistoryDynamoDBRecord3.setPrimaryKey(null); + authHistoryDynamoDBRecord4.setPrimaryKey("primaryKeyTest2"); + assertFalse(authHistoryDynamoDBRecord3.equals(authHistoryDynamoDBRecord4)); + } +} diff --git a/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/AuthHistorySyncerTest.java b/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/AuthHistorySyncerTest.java index f7fe16719be..72e6bfd6fec 100644 --- a/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/AuthHistorySyncerTest.java +++ b/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/AuthHistorySyncerTest.java @@ -91,4 +91,39 @@ public void testGetPrivateKeyStore() { assertNotNull(privateKeyStore); System.clearProperty("auth_history_syncer.private_key_store_factory_class"); } + + @Test + public void testAuthHistorySyncerFailures() { + AuthHistorySyncer authHistorySyncer = new AuthHistorySyncer(); + assertNotNull(authHistorySyncer); + AuthHistorySyncer.usage(); + + System.setProperty("auth_history_syncer.fetch_factory_class", "unknown-class"); + try { + authHistorySyncer.getAuthHistoryFetcher(null, null); + fail(); + } catch (Exception ex) { + assertTrue(ex instanceof ClassNotFoundException); + } + + System.setProperty("auth_history_syncer.private_key_store_factory_class", "unknown-class"); + try { + authHistorySyncer.getPrivateKeyStore(); + fail(); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("Invalid private key store")); + } + + System.setProperty("auth_history_syncer.send_factory_class", "unknown-class"); + try { + authHistorySyncer.getAuthHistorySender(null, null); + fail(); + } catch (Exception ex) { + assertTrue(ex instanceof ClassNotFoundException); + } + + System.clearProperty("auth_history_syncer.fetch_factory_class"); + System.clearProperty("auth_history_syncer.send_factory_class"); + System.clearProperty("auth_history_syncer.private_key_store_factory_class"); + } } diff --git a/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/LogsParserUtilsTest.java b/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/LogsParserUtilsTest.java index 30566a0e6c1..24a95697744 100644 --- a/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/LogsParserUtilsTest.java +++ b/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/LogsParserUtilsTest.java @@ -23,12 +23,17 @@ import java.net.MalformedURLException; import static org.junit.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; import static org.testng.AssertJUnit.fail; public class LogsParserUtilsTest { @Test public void testGetRecordFromLogEvent() throws MalformedURLException { + + LogsParserUtils utils = new LogsParserUtils(); + assertNotNull(utils); + // Test /domain/{domainName}/token String message = "98.136.200.210 - user.testprincipal [19/Apr/2022:08:00:45 +0000] \"GET /zms/v1/domain/home.testuser/token HTTP/1.1\" 200 16 \"-\" \"Jersey/2.18 (HttpUrlConnection 1.8.0_302)\" - 2 Auth-X509 TLSv1.2 TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"; AuthHistoryDynamoDBRecord recordFromLogEvent = LogsParserUtils.getRecordFromLogEvent(message); diff --git a/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/impl/DynamoDBAuthHistorySenderTest.java b/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/impl/DynamoDBAuthHistorySenderTest.java index 59d1e57267f..1fa9fb84274 100644 --- a/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/impl/DynamoDBAuthHistorySenderTest.java +++ b/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/impl/DynamoDBAuthHistorySenderTest.java @@ -26,6 +26,7 @@ import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.enhanced.dynamodb.*; +import software.amazon.awssdk.enhanced.dynamodb.model.Page; import software.amazon.awssdk.enhanced.dynamodb.model.QueryConditional; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; @@ -93,20 +94,22 @@ public void testDynamoDBAuthHistorySender() throws ExecutionException, Interrupt dynamoDBAuthHistorySender.pushRecords(records); // Verify querying by primary ket and by the two domain indexes - DynamoDbTable nonAsynctable = getNonAsyncTable(LocalDynamoDbAsyncClientFactory.port); - verifyItemsByPrimaryKey(nonAsynctable, 100); - verifyItemsByUriDomainINdex(nonAsynctable, 100); - verifyItemsByPrincipalDomainINdex(nonAsynctable, 100); + DynamoDbTable nonAsyncTable = getNonAsyncTable(LocalDynamoDbAsyncClientFactory.port); + verifyItemsByPrimaryKey(nonAsyncTable, 100); + verifyItemsByUriDomainINdex(nonAsyncTable, 100); + verifyItemsByPrincipalDomainINdex(nonAsyncTable, 100); // Scan all items and verify all items are accounted for DynamoDbAsyncClient dynamoDb = dynamoDbAsyncClientFactory.create(null); DynamoDbEnhancedAsyncClient dynamoDbEnhancedAsyncClient = DynamoDbEnhancedAsyncClient.builder() .dynamoDbClient(dynamoDb) .build(); - DynamoDbAsyncTable table = dynamoDbEnhancedAsyncClient.table(DynamoDBAuthHistorySender.PROP_TABLE_NAME_DEFAULT, TableSchema.fromBean(AuthHistoryDynamoDBRecord.class)); - table.scan().items().subscribe(item -> { - records.remove(new AuthHistoryDynamoDBRecord(item.getPrimaryKey(), item.getUriDomain(), item.getPrincipalDomain(), item.getPrincipalName(), item.getEndpoint(), item.getTimestamp(), item.getTtl())); - }).get(); + DynamoDbAsyncTable table = dynamoDbEnhancedAsyncClient.table( + DynamoDBAuthHistorySender.PROP_TABLE_NAME_DEFAULT, + TableSchema.fromBean(AuthHistoryDynamoDBRecord.class)); + table.scan().items().subscribe(item -> records.remove(new AuthHistoryDynamoDBRecord(item.getPrimaryKey(), + item.getUriDomain(), item.getPrincipalDomain(), item.getPrincipalName(), item.getEndpoint(), + item.getTimestamp(), "access-token", item.getTtl()))).get(); assertEquals(0, records.size()); localDynamoDbAsyncClientFactory.terminate(); @@ -179,7 +182,7 @@ private void verifyItemsByUriDomainINdex(DynamoDbTable> records = index.query(r -> r.queryConditional(queryConditional)) .stream() - .map(record -> record.items()) + .map(Page::items) .collect(Collectors.toList()); assertEquals(records.size(), 1); assertEquals(records.get(0).size(), 1); @@ -196,7 +199,7 @@ private void verifyItemsByPrincipalDomainINdex(DynamoDbTable> records = index.query(r -> r.queryConditional(queryConditional)) .stream() - .map(record -> record.items()) + .map(Page::items) .collect(Collectors.toList()); assertEquals(records.size(), 1); assertEquals(records.get(0).size(), 1); @@ -209,7 +212,8 @@ private AuthHistoryDynamoDBRecord generateRecordForTest(int index, long ttl) { String principalDomain = "principalDomain" + index; String principalName = "principalName" + index; String primaryKey = getPrimaryKeyForTest(index); - return new AuthHistoryDynamoDBRecord(primaryKey, uriDomain, principalDomain, principalName, "https://endpoint" + index + ".com", "19/Apr/2022:08:00:45", ttl); + return new AuthHistoryDynamoDBRecord(primaryKey, uriDomain, principalDomain, principalName, + "https://endpoint" + index + ".com", "19/Apr/2022:08:00:45", "access-token", ttl); } private String getPrimaryKeyForTest(int index) { diff --git a/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/impl/MockAuthHistoryFetcher.java b/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/impl/MockAuthHistoryFetcher.java index 700caf0c9fb..d6bc492bee4 100644 --- a/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/impl/MockAuthHistoryFetcher.java +++ b/syncers/auth_history_syncer/src/test/java/com/yahoo/athenz/syncer/auth/history/impl/MockAuthHistoryFetcher.java @@ -33,7 +33,9 @@ public Set getLogs(Long startTime, Long endTime, bool return new HashSet<>(); } Set records = new HashSet<>(); - AuthHistoryDynamoDBRecord authHistoryDynamoDBRecord = new AuthHistoryDynamoDBRecord("primaryKeyTest", "uriDomainTest", "principalDomainTest", "principalNameTest", "endpointTest", "timestampTest", 0L); + AuthHistoryDynamoDBRecord authHistoryDynamoDBRecord = new AuthHistoryDynamoDBRecord("primaryKeyTest", + "uriDomainTest", "principalDomainTest", "principalNameTest", "endpointTest", + "timestampTest", "access-check", 0L); records.add(authHistoryDynamoDBRecord); return records; }