Skip to content

Commit

Permalink
store operation type as part of the auth history record (#2519)
Browse files Browse the repository at this point in the history 
Signed-off-by: Henry Avetisyan <hga@yahooinc.com>
Co-authored-by: Henry Avetisyan <hga@yahooinc.com>
  • Loading branch information
@havetisyan
havetisyan and havetisyan authored Feb 21, 2024
1 parent 4b8c08b commit 9589df2
Show file tree
Hide file tree
Showing 11 changed files with 246 additions and 77 deletions.
2 changes: 1 addition & 1 deletion syncers/auth_history_syncer/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
<description>Authentication History Syncer (Cloudwatch logs to DynamoDB)</description>

<properties>
<code.coverage.min>0.7738</code.coverage.min>
<code.coverage.min>0.8593</code.coverage.min>
<aws.dynamodb.local.version>2.2.1</aws.dynamodb.local.version>
<sqlite4java.version>1.0.392</sqlite4java.version>
<junit.version>4.13.2</junit.version>
Expand Down
20 changes: 13 additions & 7 deletions ..._syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/AuthHistoryDynamoDBRecord.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,19 +36,21 @@ public class AuthHistoryDynamoDBRecord {
private String principalName;
private String endpoint;
private String timestamp;
private String operation;
private long ttl;

public AuthHistoryDynamoDBRecord() {

}

public AuthHistoryDynamoDBRecord(String primaryKey, String uriDomain, String principalDomain, String principalName, String endpoint, String timestamp, long ttl) {
public AuthHistoryDynamoDBRecord(String primaryKey, String uriDomain, String principalDomain,
String principalName, String endpoint, String timestamp, String operation, long ttl) {
this.primaryKey = primaryKey;
this.uriDomain = uriDomain;
this.principalDomain = principalDomain;
this.principalName = principalName;
this.endpoint = endpoint;
this.timestamp = timestamp;
this.operation = operation;
this.ttl = ttl;
}

Expand Down Expand Up @@ -83,6 +85,10 @@ public String getTimestamp() {
return timestamp;
}

public String getOperation() {
return operation;
}

// Set methods must exist for @DynamoDbBean successful marshalling
public void setPrimaryKey(String primaryKey) {
this.primaryKey = primaryKey;
Expand All @@ -102,6 +108,9 @@ public void setEndpoint(String endpoint) {
public void setTimestamp(String timestamp) {
this.timestamp = timestamp;
}
public void setOperation(String operation) {
this.operation = operation;
}
public void setTtl(long ttl) {
this.ttl = ttl;
}
Expand All @@ -116,11 +125,7 @@ public boolean equals(Object o) {
}

AuthHistoryDynamoDBRecord record = (AuthHistoryDynamoDBRecord) o;
if (getPrimaryKey() == null ? record.getPrimaryKey() != null : !getPrimaryKey().equals(record.getPrimaryKey())) {
return false;
}

return true;
return getPrimaryKey() == null ? record.getPrimaryKey() == null : getPrimaryKey().equals(record.getPrimaryKey());
}

@Override
Expand All @@ -137,6 +142,7 @@ public String toString() {
", principalName='" + principalName + '\'' +
", endpoint='" + endpoint + '\'' +
", timestamp='" + timestamp + '\'' +
", operation='" + operation + '\'' +
", ttl=" + ttl +
'}';
}
Expand Down
4 changes: 2 additions & 2 deletions ..._history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/AuthHistorySyncer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ PrivateKeyStore getPrivateKeyStore() {
return pkeyFactory.create();
}

private AuthHistoryFetcher getAuthHistoryFetcher(PrivateKeyStore privateKeyStore, String region) throws Exception {
AuthHistoryFetcher getAuthHistoryFetcher(PrivateKeyStore privateKeyStore, String region) throws Exception {
String authHistoryFetcherFactoryClass = System.getProperty(PROP_AUTH_HISTORY_FETCH_FACTORY_CLASS);
if (authHistoryFetcherFactoryClass == null) {
System.out.println("Error: " + PROP_AUTH_HISTORY_FETCH_FACTORY_CLASS + " system property is mandatory");
Expand All @@ -110,7 +110,7 @@ private AuthHistoryFetcher getAuthHistoryFetcher(PrivateKeyStore privateKeyStore
}
}

private AuthHistorySender getAuthHistorySender(PrivateKeyStore privateKeyStore, String region) throws ClassNotFoundException, InstantiationException, IllegalAccessException {
AuthHistorySender getAuthHistorySender(PrivateKeyStore privateKeyStore, String region) throws ClassNotFoundException, InstantiationException, IllegalAccessException {
String authHistorySenderFactoryClass = System.getProperty(PROP_AUTH_HISTORY_SEND_FACTORY_CLASS, PROP_AUTH_HISTORY_SEND_FACTORY_CLASS_DEFAULT);
AuthHistorySenderFactory authHistorySenderFactory;
try {
Expand Down
3 changes: 3 additions & 0 deletions ...ry_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/AuthHistorySyncerConsts.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,7 @@ private AuthHistorySyncerConsts() {
public static final String PROP_DYNAMODB_EXTERNAL_ID = "auth_history_syncer.dynamodb_external_id";
public static final String PROP_DYNAMODB_MIN_EXPIRY_TIME = "auth_history_syncer.dynamodb_min_expiry_time";
public static final String PROP_DYNAMODB_MAX_EXPIRY_TIME = "auth_history_syncer.dynamodb_max_expiry_time";

public static final String PROP_CLOUDWATCH_ZMS_LOG_GROUP = "auth_history_syncer.cloudwatch_zms_log_group";
public static final String PROP_CLOUDWATCH_ZTS_LOG_GROUP = "auth_history_syncer.cloudwatch_zts_log_group";
}
99 changes: 59 additions & 40 deletions ...th_history_syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/LogsParserUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,38 +18,68 @@

package com.yahoo.athenz.syncer.auth.history;

import com.yahoo.athenz.auth.util.AthenzUtils;

import java.net.MalformedURLException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class LogsParserUtils {

private static class PatternType {
public Pattern pattern;
public String operation;

public PatternType(Pattern pattern, final String operation) {
this.pattern = pattern;
this.operation = operation;
}
}

private static final String DOMAIN_REGEX = "([a-zA-Z0-9_][a-zA-Z0-9_-]*\\.)*[a-zA-Z0-9_][a-zA-Z0-9_-]*";

// regex:
// /domain/{domainName}/token
// /domain/{domainName}/role/{roleName}/token
// /access/domain/{domainName}/role/{roleName}/principal/{principal}
// /access/domain/{domainName}/principal/{principal}
private static final Pattern DOMAIN_PATH_PARAM_PATTERN = Pattern.compile("/domain/(" + DOMAIN_REGEX + ")/(token|role/|principal/)", Pattern.MULTILINE);
// regex: /domain/{domainName}/token
private static final PatternType ROLE_TOKEN_PATTERN = new PatternType(
Pattern.compile("/domain/(" + DOMAIN_REGEX + ")/token", Pattern.MULTILINE), "role-token");

// regex: /domain/{domainName}/role/{roleName}/token
private static final PatternType ROLE_CERT_OLD_PATTERN = new PatternType(
Pattern.compile("/domain/(" + DOMAIN_REGEX + ")/role/", Pattern.MULTILINE), "role-cert");

// regex: /access/domain/{domainName}/role/{roleName}/principal/{principal}
private static final PatternType ACCESS_ROLE_PATTERN = new PatternType(
Pattern.compile("/access/domain/(" + DOMAIN_REGEX + ")/role/", Pattern.MULTILINE), "access-check");

// regex: /access/domain/{domainName}/principal/{principal}
private static final PatternType ACCESS_PRINCIPAL_PATTERN = new PatternType(
Pattern.compile("/access/domain/(" + DOMAIN_REGEX + ")/principal/", Pattern.MULTILINE), "access-check");

// regex: /oauth2/token
private static final Pattern OAUTH_TOKEN_PATTERN = Pattern.compile("/oauth2/token.*scope=(" + DOMAIN_REGEX + ").*", Pattern.MULTILINE);
private static final PatternType OAUTH_TOKEN_PATTERN = new PatternType(
Pattern.compile("/oauth2/token.*scope=(" + DOMAIN_REGEX + ").*", Pattern.MULTILINE), "access-token");

// regex: (alternate domain for cross-domain trust relation)
// /access/{action}?domain={domain}
// /access/{action}/{resource}?domain={domain}
private static final Pattern ACCESS_RESOURCE_ALT_DOMAIN_PATTERN = Pattern.compile("/access/.*domain=(" + DOMAIN_REGEX + ").*", Pattern.MULTILINE);
private static final PatternType ACCESS_RESOURCE_ALT_DOMAIN_PATTERN = new PatternType(
Pattern.compile("/access/.*domain=(" + DOMAIN_REGEX + ").*", Pattern.MULTILINE), "access-check");

// regex: /access/{action}?resource={domain}:{resource}
private static final Pattern ACCESS_RESOURCE_PATTERN_QUERY_PARAM = Pattern.compile("/access/.*resource=(" + DOMAIN_REGEX + ").*", Pattern.MULTILINE);
private static final PatternType ACCESS_RESOURCE_PATTERN_QUERY_PARAM = new PatternType(
Pattern.compile("/access/.*resource=(" + DOMAIN_REGEX + ").*", Pattern.MULTILINE), "access-check");

// regex: /access/{action}/{resource}
private static final Pattern ACCESS_RESOURCE_PATTERN = Pattern.compile("/access/.*/(" + DOMAIN_REGEX + "):.*", Pattern.MULTILINE);
private static final PatternType ACCESS_RESOURCE_PATTERN = new PatternType(
Pattern.compile("/access/.*/(" + DOMAIN_REGEX + "):.*", Pattern.MULTILINE), "access-check");

// regex: /rolecert?roleName={domain}:role.{role}
private static final Pattern ROLE_CERT_PATTERN = Pattern.compile("/rolecert.roleName=(" + DOMAIN_REGEX + "):role.*", Pattern.MULTILINE);
private static final PatternType ROLE_CERT_PATTERN = new PatternType(
Pattern.compile("/rolecert.roleName=(" + DOMAIN_REGEX + "):role.*", Pattern.MULTILINE), "role-cert");

private static final Pattern[] PATTERNS = {DOMAIN_PATH_PARAM_PATTERN, OAUTH_TOKEN_PATTERN, ACCESS_RESOURCE_ALT_DOMAIN_PATTERN, ACCESS_RESOURCE_PATTERN_QUERY_PARAM, ACCESS_RESOURCE_PATTERN, ROLE_CERT_PATTERN};
private static final PatternType[] PATTERNS = {ROLE_TOKEN_PATTERN, ROLE_CERT_OLD_PATTERN,
ACCESS_ROLE_PATTERN, ACCESS_PRINCIPAL_PATTERN, OAUTH_TOKEN_PATTERN,
ACCESS_RESOURCE_ALT_DOMAIN_PATTERN, ACCESS_RESOURCE_PATTERN_QUERY_PARAM,
ACCESS_RESOURCE_PATTERN, ROLE_CERT_PATTERN};

private static final String PROP_TTL = "auth_history_syncer.ttl";
private static final String PROP_TTL_DEFAULT = "720"; // 30 days
Expand All @@ -58,45 +88,34 @@ public class LogsParserUtils {
private static final long EXPIRY_TIME = 3660 * EXPIRY_HOURS;

public static AuthHistoryDynamoDBRecord getRecordFromLogEvent(String message) throws MalformedURLException {

String[] split = message.split("\\s+");
String principalDomain = getPrincipalDomain(split[2]);
String principalName = getPrincipalName(split[2]);
String endpoint = split[5].substring(1) + " " + split[6];
String timestamp = split[3].substring(1);
String uriDomain = getDomainFromEndpoint(split[6]);
String primaryKey = generatePrimaryKey(uriDomain, principalDomain, principalName);
return new AuthHistoryDynamoDBRecord(primaryKey, uriDomain, principalDomain, principalName, endpoint, timestamp, System.currentTimeMillis() / 1000L + EXPIRY_TIME);
AuthHistoryDynamoDBRecord record = createRecordObject(split[6]);
record.setPrincipalDomain(AthenzUtils.extractPrincipalDomainName(split[2]));
record.setPrincipalName(AthenzUtils.extractPrincipalServiceName(split[2]));
record.setEndpoint(split[5].substring(1) + " " + split[6]);
record.setTimestamp(split[3].substring(1));
record.setPrimaryKey(generatePrimaryKey(record.getUriDomain(), record.getPrincipalDomain(),
record.getPrincipalName()));
record.setTtl(System.currentTimeMillis() / 1000L + EXPIRY_TIME);
return record;
}

public static String generatePrimaryKey(String uriDomain, String principalDomain, String principalName) {
return uriDomain + ":" + principalDomain + ":" + principalName;
}

private static String getDomainFromEndpoint(String endpoint) throws MalformedURLException {
for (Pattern pattern : PATTERNS) {
Matcher m = pattern.matcher(endpoint);
private static AuthHistoryDynamoDBRecord createRecordObject(final String endpoint) throws MalformedURLException {
for (PatternType patternType : PATTERNS) {
Matcher m = patternType.pattern.matcher(endpoint);
if (m.find()) {
return m.group(1);
AuthHistoryDynamoDBRecord record = new AuthHistoryDynamoDBRecord();
record.setUriDomain(m.group(1));
record.setOperation(patternType.operation);
return record;
}
}

throw new MalformedURLException("Failed to locate domain at endpoint: " + endpoint);
}


private static String getPrincipalDomain(String principal) {
int n = principal.lastIndexOf('.');
if (n <= 0 || n == principal.length() - 1) {
return null;
}
return principal.substring(0, n);
}

private static String getPrincipalName(String principal) {
int n = principal.lastIndexOf('.');
if (n <= 0 || n == principal.length() - 1) {
return null;
}
return principal.substring(n + 1);
}
}
44 changes: 29 additions & 15 deletions ...syncer/src/main/java/com/yahoo/athenz/syncer/auth/history/impl/AwsAuthHistoryFetcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,7 @@

package com.yahoo.athenz.syncer.auth.history.impl;

import com.yahoo.athenz.syncer.auth.history.AuthHistoryDynamoDBRecord;
import com.yahoo.athenz.syncer.auth.history.AuthHistoryFetcher;
import com.yahoo.athenz.syncer.auth.history.CloudWatchClientFactory;
import com.yahoo.athenz.syncer.auth.history.LogsParserUtils;
import com.yahoo.athenz.syncer.auth.history.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.services.cloudwatch.model.CloudWatchException;
Expand All @@ -35,24 +32,34 @@
import java.util.Set;

public class AwsAuthHistoryFetcher implements AuthHistoryFetcher {

private static final Logger LOGGER = LoggerFactory.getLogger(AwsAuthHistoryFetcher.class);
private final CloudWatchLogsClient cloudWatchLogsClient;

private final String zmsLogGroup = System.getProperty(AuthHistorySyncerConsts.PROP_CLOUDWATCH_ZMS_LOG_GROUP,
"athenz-zms-service-access");
private final String ztsLogGroup = System.getProperty(AuthHistorySyncerConsts.PROP_CLOUDWATCH_ZTS_LOG_GROUP,
"athenz-zts-service-access");

public AwsAuthHistoryFetcher(CloudWatchClientFactory cloudWatchClientFactory) {
this.cloudWatchLogsClient = cloudWatchClientFactory.create();
}

/**
*
* @param startTime - the start of the time range, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC (for example, 1620940080)
* @param endTime - the end of the time range, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC (for example, 1620940080)
* @param useFilterPattern - if true, filter access events in request. False means that all events in the time range will return.
* @return - authorization checks and token requests history ready to be pushed to a data store. On error return null.
* @param startTime - the start of the time range, expressed as the number of milliseconds
* after Jan 1, 1970 00:00:00 UTC (for example, 1620940080)
* @param endTime - the end of the time range, expressed as the number of milliseconds
* after Jan 1, 1970 00:00:00 UTC (for example, 1620940080)
* @param useFilterPattern - if true, filter access events in request. False means that
* all events in the time range will return.
* @return - authorization checks and token requests history ready to be pushed to a
* data store. On error return null.
*/
@Override
public Set<AuthHistoryDynamoDBRecord> getLogs(Long startTime, Long endTime, boolean useFilterPattern) {
Set<AuthHistoryDynamoDBRecord> zmsLogs = getLogs("athenz-zms-service-access", startTime, endTime, useFilterPattern);
Set<AuthHistoryDynamoDBRecord> ztsLogs = getLogs("athenz-zts-service-access", startTime, endTime, useFilterPattern);
Set<AuthHistoryDynamoDBRecord> zmsLogs = getLogs(zmsLogGroup, startTime, endTime, useFilterPattern);
Set<AuthHistoryDynamoDBRecord> ztsLogs = getLogs(ztsLogGroup, startTime, endTime, useFilterPattern);
Set<AuthHistoryDynamoDBRecord> allRecords = new HashSet<>();
if (zmsLogs != null && !zmsLogs.isEmpty()) {
allRecords.addAll(zmsLogs);
Expand All @@ -66,13 +73,20 @@ public Set<AuthHistoryDynamoDBRecord> getLogs(Long startTime, Long endTime, bool
/**
*
* @param logGroup - Log group name
* @param startTime - the start of the time range, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC (for example, 1620940080)
* @param endTime - the end of the time range, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC (for example, 1620940080)
* @param useFilterPattern - if true, filter access events in request. False means that all events in the time range will return.
* @return - authorization checks and token requests history ready to be pushed to a data store. On error return null.
* @param startTime - the start of the time range, expressed as the number of milliseconds
* after Jan 1, 1970 00:00:00 UTC (for example, 1620940080)
* @param endTime - the end of the time range, expressed as the number of milliseconds
* after Jan 1, 1970 00:00:00 UTC (for example, 1620940080)
* @param useFilterPattern - if true, filter access events in request. False means that
* all events in the time range will return.
* @return - authorization checks and token requests history ready to be pushed to a
* data store. On error return null.
*/
private Set<AuthHistoryDynamoDBRecord> getLogs(String logGroup, Long startTime, Long endTime, boolean useFilterPattern) {
LOGGER.info("Getting logs from logGroup {}, startTime(milli): {}, endTime(milli): {}, useFilterPattern: {}", logGroup, startTime, endTime, useFilterPattern);

LOGGER.info("Getting logs from logGroup {}, startTime(milli): {}, endTime(milli): {}, useFilterPattern: {}",
logGroup, startTime, endTime, useFilterPattern);

try {
String nextToken = null;
Set<AuthHistoryDynamoDBRecord> filteredEvents = new HashSet<>();
Expand Down
Loading

0 comments on commit 9589df2

Please sign in to comment.