From f58690832b3786d42d302f16cc84585c8147e410 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 9 Dec 2024 18:55:18 -0800 Subject: [PATCH 1/4] RANGER-5064: checkstyle compliance updates - usersync modules --- ugsync-util/pom.xml | 3 + .../ugsyncutil/model/FileSyncSourceInfo.java | 166 +- .../ugsyncutil/model/GroupUserInfo.java | 2 +- .../ugsyncutil/model/LdapSyncSourceInfo.java | 268 +- .../ugsyncutil/model/UgsyncAuditInfo.java | 223 +- .../ugsyncutil/model/UnixSyncSourceInfo.java | 226 +- .../model/UsersGroupRoleAssignments.java | 102 +- .../ranger/ugsyncutil/model/XGroupInfo.java | 202 +- .../ranger/ugsyncutil/model/XUserInfo.java | 264 +- .../util/UgsyncCommonConstants.java | 9 +- .../ldapconfigcheck/pom.xml | 3 + .../ldapconfigcheck/AuthenticationCheck.java | 91 +- .../ldapconfigcheck/CommandLineOptions.java | 140 +- .../ranger/ldapconfigcheck/LdapConfig.java | 301 +- .../ldapconfigcheck/LdapConfigCheckMain.java | 79 +- .../ranger/ldapconfigcheck/UserSync.java | 753 ++-- ugsync/pom.xml | 3 + .../process/CustomSSLSocketFactory.java | 204 +- .../process/LdapUserGroupBuilder.java | 2355 ++++++----- .../config/UserGroupSyncConfig.java | 2357 +++++------ .../ha/UserSyncHAInitializerImpl.java | 48 +- .../model/GetXGroupListResponse.java | 38 +- .../model/GetXUserListResponse.java | 41 +- .../process/FileSourceUserGroupBuilder.java | 526 ++- .../process/PolicyMgrUserGroupBuilder.java | 3603 ++++++++--------- .../process/RangerUgSyncRESTClient.java | 84 +- .../process/UnixUserGroupBuilder.java | 1063 +++-- .../ranger/usergroupsync/AbstractMapper.java | 23 +- .../AbstractUserGroupSource.java | 22 +- .../apache/ranger/usergroupsync/Mapper.java | 2 +- .../apache/ranger/usergroupsync/RegEx.java | 112 +- .../ranger/usergroupsync/UserGroupSink.java | 15 +- .../ranger/usergroupsync/UserGroupSource.java | 9 +- .../ranger/usergroupsync/UserGroupSync.java | 183 +- .../UserSyncMetricsProducer.java | 136 +- .../TestFileSourceUserGroupBuilder.java | 7 +- .../process/TestUnixUserGroupBuilder.java | 34 +- .../PolicyMgrUserGroupBuilderTest.java | 161 +- .../usergroupsync/TestLdapUserGroup.java | 804 ++-- .../ranger/usergroupsync/TestRegEx.java | 35 +- unixauthclient/pom.xml | 3 + .../jaas/ConsolePromptCallbackHandler.java | 72 +- .../unix/jaas/PamLoginModule.java | 226 +- .../unix/jaas/PamPrincipal.java | 68 +- .../unix/jaas/RemoteUnixLoginModule.java | 796 ++-- .../unix/jaas/UnixGroupPrincipal.java | 23 +- .../unix/jaas/UnixUserPrincipal.java | 23 +- .../jaas/UsernamePasswordCallbackHandler.java | 20 +- .../UnixAuthenticationTester.java | 23 +- unixauthnative/pom.xml | 3 + unixauthnative/src/main/c/credValidator.c | 4 +- unixauthpam/pom.xml | 3 + unixauthservice/pom.xml | 3 + .../authentication/PasswordValidator.java | 251 +- .../UnixAuthenticationService.java | 641 ++- 55 files changed, 8041 insertions(+), 8815 deletions(-) diff --git a/ugsync-util/pom.xml b/ugsync-util/pom.xml index 5042d2097d..c6a68bb407 100644 --- a/ugsync-util/pom.xml +++ b/ugsync-util/pom.xml @@ -26,6 +26,9 @@ jar User Group Synchronizer Util User Group Synchronizer Util + + true + com.fasterxml.jackson.core diff --git a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/FileSyncSourceInfo.java b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/FileSyncSourceInfo.java index 4e122c25da..a7ba96d54b 100644 --- a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/FileSyncSourceInfo.java +++ b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/FileSyncSourceInfo.java @@ -17,89 +17,89 @@ * under the License. */ - package org.apache.ranger.ugsyncutil.model; +package org.apache.ranger.ugsyncutil.model; public class FileSyncSourceInfo { - private String fileName; - private String syncTime; - private String lastModified; - private long totalUsersSynced; - private long totalGroupsSynced; - private long totalUsersDeleted; - private long totalGroupsDeleted; - - public String getFileName() { - return fileName; - } - - public void setFileName(String fileName) { - this.fileName = fileName; - } - - public String getSyncTime() { - return syncTime; - } - - public void setSyncTime(String syncTime) { - this.syncTime = syncTime; - } - - public String getLastModified() { - return lastModified; - } - - public void setLastModified(String lastModified) { - this.lastModified = lastModified; - } - - public long getTotalUsersSynced() { - return totalUsersSynced; - } - - public void setTotalUsersSynced(long totalUsersSynced) { - this.totalUsersSynced = totalUsersSynced; - } - - public long getTotalGroupsSynced() { - return totalGroupsSynced; - } - - public void setTotalGroupsSynced(long totalGroupsSynced) { - this.totalGroupsSynced = totalGroupsSynced; - } - - public long getTotalUsersDeleted() { - return totalUsersDeleted; - } - - public void setTotalUsersDeleted(long totalUsersDeleted) { - this.totalUsersDeleted = totalUsersDeleted; - } - - public long getTotalGroupsDeleted() { - return totalGroupsDeleted; - } - - public void setTotalGroupsDeleted(long totalGroupsDeleted) { - this.totalGroupsDeleted = totalGroupsDeleted; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - toString(sb); - return sb.toString(); - } - - public StringBuilder toString(StringBuilder sb) { - sb.append("FileSycnSourceInfo [fileName= ").append(fileName); - sb.append(", syncTime= ").append(syncTime); - sb.append(", lastModified= ").append(lastModified); - sb.append(", totalUsersSynced= ").append(totalUsersSynced); - sb.append(", totalGroupsSynced= ").append(totalGroupsSynced); - sb.append(", totalUsersDeleted= ").append(totalUsersDeleted); - sb.append(", totalGroupsDeleted= ").append(totalGroupsDeleted); - sb.append("]"); - return sb; - } + private String fileName; + private String syncTime; + private String lastModified; + private long totalUsersSynced; + private long totalGroupsSynced; + private long totalUsersDeleted; + private long totalGroupsDeleted; + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public String getSyncTime() { + return syncTime; + } + + public void setSyncTime(String syncTime) { + this.syncTime = syncTime; + } + + public String getLastModified() { + return lastModified; + } + + public void setLastModified(String lastModified) { + this.lastModified = lastModified; + } + + public long getTotalUsersSynced() { + return totalUsersSynced; + } + + public void setTotalUsersSynced(long totalUsersSynced) { + this.totalUsersSynced = totalUsersSynced; + } + + public long getTotalGroupsSynced() { + return totalGroupsSynced; + } + + public void setTotalGroupsSynced(long totalGroupsSynced) { + this.totalGroupsSynced = totalGroupsSynced; + } + + public long getTotalUsersDeleted() { + return totalUsersDeleted; + } + + public void setTotalUsersDeleted(long totalUsersDeleted) { + this.totalUsersDeleted = totalUsersDeleted; + } + + public long getTotalGroupsDeleted() { + return totalGroupsDeleted; + } + + public void setTotalGroupsDeleted(long totalGroupsDeleted) { + this.totalGroupsDeleted = totalGroupsDeleted; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + toString(sb); + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("FileSycnSourceInfo [fileName= ").append(fileName); + sb.append(", syncTime= ").append(syncTime); + sb.append(", lastModified= ").append(lastModified); + sb.append(", totalUsersSynced= ").append(totalUsersSynced); + sb.append(", totalGroupsSynced= ").append(totalGroupsSynced); + sb.append(", totalUsersDeleted= ").append(totalUsersDeleted); + sb.append(", totalGroupsDeleted= ").append(totalGroupsDeleted); + sb.append("]"); + return sb; + } } diff --git a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/GroupUserInfo.java b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/GroupUserInfo.java index aaa0a84875..2f5b649d88 100644 --- a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/GroupUserInfo.java +++ b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/GroupUserInfo.java @@ -22,7 +22,7 @@ import java.util.Set; public class GroupUserInfo { - String groupName; + String groupName; Set addUsers; Set delUsers; diff --git a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/LdapSyncSourceInfo.java b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/LdapSyncSourceInfo.java index 227c897920..b9e886d9fd 100644 --- a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/LdapSyncSourceInfo.java +++ b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/LdapSyncSourceInfo.java @@ -17,140 +17,140 @@ * under the License. */ - package org.apache.ranger.ugsyncutil.model; +package org.apache.ranger.ugsyncutil.model; public class LdapSyncSourceInfo { - private String ldapUrl; - private String incrementalSycn; - private String groupSearchFirstEnabled; - private String groupSearchEnabled; - private String userSearchEnabled; - private String userSearchFilter; - private String groupSearchFilter; - private String groupHierarchyLevel; - - private long totalUsersSynced; - private long totalGroupsSynced; - private long totalUsersDeleted; - private long totalGroupsDeleted; - - public String getLdapUrl() { - return ldapUrl; - } - - public void setLdapUrl(String ldapUrl) { - this.ldapUrl = ldapUrl; - } - - public String isIncrementalSycn() { - return incrementalSycn; - } - - public void setIncrementalSycn(String incrementalSycn) { - this.incrementalSycn = incrementalSycn; - } - - public String getUserSearchFilter() { - return userSearchFilter; - } - - public void setUserSearchFilter(String userSearchFilter) { - this.userSearchFilter = userSearchFilter; - } - - public String getGroupSearchFilter() { - return groupSearchFilter; - } - - public void setGroupSearchFilter(String groupSearchFilter) { - this.groupSearchFilter = groupSearchFilter; - } - - public String getGroupHierarchyLevel() { - return groupHierarchyLevel; - } - - public void setGroupHierarchyLevel(String groupHierarchyLevel) { - this.groupHierarchyLevel = groupHierarchyLevel; - } - - public long getTotalUsersSynced() { - return totalUsersSynced; - } - - public void setTotalUsersSynced(long totalUsersSynced) { - this.totalUsersSynced = totalUsersSynced; - } - - public long getTotalGroupsSynced() { - return totalGroupsSynced; - } - - public void setTotalGroupsSynced(long totalGroupsSynced) { - this.totalGroupsSynced = totalGroupsSynced; - } - - public String getGroupSearchFirstEnabled() { - return groupSearchFirstEnabled; - } - - public void setGroupSearchFirstEnabled(String groupSearchFirstEnabled) { - this.groupSearchFirstEnabled = groupSearchFirstEnabled; - } - - public String getGroupSearchEnabled() { - return groupSearchEnabled; - } - - public void setGroupSearchEnabled(String groupSearchEnabled) { - this.groupSearchEnabled = groupSearchEnabled; - } - - public String getUserSearchEnabled() { - return userSearchEnabled; - } - - public void setUserSearchEnabled(String userSearchEnabled) { - this.userSearchEnabled = userSearchEnabled; - } - - public long getTotalUsersDeleted() { - return totalUsersDeleted; - } - - public void setTotalUsersDeleted(long totalUsersDeleted) { - this.totalUsersDeleted = totalUsersDeleted; - } - - public long getTotalGroupsDeleted() { - return totalGroupsDeleted; - } - - public void setTotalGroupsDeleted(long totalGroupsDeleted) { - this.totalGroupsDeleted = totalGroupsDeleted; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - toString(sb); - return sb.toString(); - } - - public StringBuilder toString(StringBuilder sb) { - sb.append("LdapSycnSourceInfo [ldapUrl= ").append(ldapUrl); - sb.append(", isIncrementalSync= ").append(incrementalSycn); - sb.append(", userSearchEnabled= ").append(userSearchEnabled); - sb.append(", groupSearchEnabled= ").append(groupSearchEnabled); - sb.append(", groupSearchFirstEnabled= ").append(groupSearchFirstEnabled); - sb.append(", userSearchFilter= ").append(userSearchFilter); - sb.append(", groupSearchFilter= ").append(groupSearchFilter); - sb.append(", groupHierarchyLevel= ").append(groupHierarchyLevel); - sb.append(", totalUsersSynced= ").append(totalUsersSynced); - sb.append(", totalGroupsSynced= ").append(totalGroupsSynced); - sb.append(", totalUsersDeleted= ").append(totalUsersDeleted); - sb.append(", totalGroupsDeleted= ").append(totalGroupsDeleted); - sb.append("]"); - return sb; - } + private String ldapUrl; + private String incrementalSycn; + private String groupSearchFirstEnabled; + private String groupSearchEnabled; + private String userSearchEnabled; + private String userSearchFilter; + private String groupSearchFilter; + private String groupHierarchyLevel; + + private long totalUsersSynced; + private long totalGroupsSynced; + private long totalUsersDeleted; + private long totalGroupsDeleted; + + public String getLdapUrl() { + return ldapUrl; + } + + public void setLdapUrl(String ldapUrl) { + this.ldapUrl = ldapUrl; + } + + public String isIncrementalSycn() { + return incrementalSycn; + } + + public void setIncrementalSycn(String incrementalSycn) { + this.incrementalSycn = incrementalSycn; + } + + public String getUserSearchFilter() { + return userSearchFilter; + } + + public void setUserSearchFilter(String userSearchFilter) { + this.userSearchFilter = userSearchFilter; + } + + public String getGroupSearchFilter() { + return groupSearchFilter; + } + + public void setGroupSearchFilter(String groupSearchFilter) { + this.groupSearchFilter = groupSearchFilter; + } + + public String getGroupHierarchyLevel() { + return groupHierarchyLevel; + } + + public void setGroupHierarchyLevel(String groupHierarchyLevel) { + this.groupHierarchyLevel = groupHierarchyLevel; + } + + public long getTotalUsersSynced() { + return totalUsersSynced; + } + + public void setTotalUsersSynced(long totalUsersSynced) { + this.totalUsersSynced = totalUsersSynced; + } + + public long getTotalGroupsSynced() { + return totalGroupsSynced; + } + + public void setTotalGroupsSynced(long totalGroupsSynced) { + this.totalGroupsSynced = totalGroupsSynced; + } + + public String getGroupSearchFirstEnabled() { + return groupSearchFirstEnabled; + } + + public void setGroupSearchFirstEnabled(String groupSearchFirstEnabled) { + this.groupSearchFirstEnabled = groupSearchFirstEnabled; + } + + public String getGroupSearchEnabled() { + return groupSearchEnabled; + } + + public void setGroupSearchEnabled(String groupSearchEnabled) { + this.groupSearchEnabled = groupSearchEnabled; + } + + public String getUserSearchEnabled() { + return userSearchEnabled; + } + + public void setUserSearchEnabled(String userSearchEnabled) { + this.userSearchEnabled = userSearchEnabled; + } + + public long getTotalUsersDeleted() { + return totalUsersDeleted; + } + + public void setTotalUsersDeleted(long totalUsersDeleted) { + this.totalUsersDeleted = totalUsersDeleted; + } + + public long getTotalGroupsDeleted() { + return totalGroupsDeleted; + } + + public void setTotalGroupsDeleted(long totalGroupsDeleted) { + this.totalGroupsDeleted = totalGroupsDeleted; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + toString(sb); + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("LdapSycnSourceInfo [ldapUrl= ").append(ldapUrl); + sb.append(", isIncrementalSync= ").append(incrementalSycn); + sb.append(", userSearchEnabled= ").append(userSearchEnabled); + sb.append(", groupSearchEnabled= ").append(groupSearchEnabled); + sb.append(", groupSearchFirstEnabled= ").append(groupSearchFirstEnabled); + sb.append(", userSearchFilter= ").append(userSearchFilter); + sb.append(", groupSearchFilter= ").append(groupSearchFilter); + sb.append(", groupHierarchyLevel= ").append(groupHierarchyLevel); + sb.append(", totalUsersSynced= ").append(totalUsersSynced); + sb.append(", totalGroupsSynced= ").append(totalGroupsSynced); + sb.append(", totalUsersDeleted= ").append(totalUsersDeleted); + sb.append(", totalGroupsDeleted= ").append(totalGroupsDeleted); + sb.append("]"); + return sb; + } } diff --git a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UgsyncAuditInfo.java b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UgsyncAuditInfo.java index 3407c69adb..5858a0213a 100644 --- a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UgsyncAuditInfo.java +++ b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UgsyncAuditInfo.java @@ -17,118 +17,117 @@ * under the License. */ - package org.apache.ranger.ugsyncutil.model; +package org.apache.ranger.ugsyncutil.model; public class UgsyncAuditInfo { - - private String userName; - private Long noOfNewUsers; - private Long noOfNewGroups; - private Long noOfModifiedUsers; - private Long noOfModifiedGroups; - private String syncSource; - private String sessionId; - private LdapSyncSourceInfo ldapSyncSourceInfo; - private UnixSyncSourceInfo unixSyncSourceInfo; - private FileSyncSourceInfo fileSyncSourceInfo; - - public Long getNoOfNewUsers() { - return noOfNewUsers; - } - - public void setNoOfNewUsers(Long noOfUsers) { - this.noOfNewUsers = noOfUsers; - } - - public Long getNoOfModifiedUsers() { - return noOfModifiedUsers; - } - - public void setNoOfModifiedUsers(Long noOfModifiedUsers) { - this.noOfModifiedUsers = noOfModifiedUsers; - } - - public Long getNoOfNewGroups() { - return noOfNewGroups; - } - - public void setNoOfNewGroups(Long noOfNewGroups) { - this.noOfNewGroups = noOfNewGroups; - } - - public Long getNoOfModifiedGroups() { - return noOfModifiedGroups; - } - - public void setNoOfModifiedGroups(Long noOfModifiedGroups) { - this.noOfModifiedGroups = noOfModifiedGroups; - } - - public String getSyncSource() { - return syncSource; - } - - public void setSyncSource(String syncSource) { - this.syncSource = syncSource; - } - - public LdapSyncSourceInfo getLdapSyncSourceInfo() { - return ldapSyncSourceInfo; - } - - public void setLdapSyncSourceInfo(LdapSyncSourceInfo ldapSyncSourceInfo) { - this.ldapSyncSourceInfo = ldapSyncSourceInfo; - } - - public UnixSyncSourceInfo getUnixSyncSourceInfo() { - return unixSyncSourceInfo; - } - - public void setUnixSyncSourceInfo(UnixSyncSourceInfo unixSyncSourceInfo) { - this.unixSyncSourceInfo = unixSyncSourceInfo; - } - - public FileSyncSourceInfo getFileSyncSourceInfo() { - return fileSyncSourceInfo; - } - - public void setFileSyncSourceInfo(FileSyncSourceInfo fileSyncSourceInfo) { - this.fileSyncSourceInfo = fileSyncSourceInfo; - } - - public String getUserName() { - return userName; - } - - public void setUserName(String userName) { - this.userName = userName; - } - - public String getSessionId() { - return sessionId; - } - - public void setSessionId(String sessionId) { - this.sessionId = sessionId; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - toString(sb); - return sb.toString(); - } - - public StringBuilder toString(StringBuilder sb) { - sb.append("UgsyncAuditInfo [No. of New users= ").append(noOfNewUsers); - sb.append(", No. of New groups= ").append(noOfNewGroups); - sb.append(", No. of Modified users= ").append(noOfModifiedUsers); - sb.append(", No. of Modified groups= ").append(noOfModifiedGroups); - sb.append(", syncSource= ").append(syncSource); - sb.append(", ldapSyncSourceInfo= ").append(ldapSyncSourceInfo); - sb.append(", unixSyncSourceInfo= ").append(unixSyncSourceInfo); - sb.append(", fileSyncSourceInfo= ").append(fileSyncSourceInfo); - sb.append("]"); - return sb; - } + private String userName; + private Long noOfNewUsers; + private Long noOfNewGroups; + private Long noOfModifiedUsers; + private Long noOfModifiedGroups; + private String syncSource; + private String sessionId; + private LdapSyncSourceInfo ldapSyncSourceInfo; + private UnixSyncSourceInfo unixSyncSourceInfo; + private FileSyncSourceInfo fileSyncSourceInfo; + + public Long getNoOfNewUsers() { + return noOfNewUsers; + } + + public void setNoOfNewUsers(Long noOfUsers) { + this.noOfNewUsers = noOfUsers; + } + + public Long getNoOfModifiedUsers() { + return noOfModifiedUsers; + } + + public void setNoOfModifiedUsers(Long noOfModifiedUsers) { + this.noOfModifiedUsers = noOfModifiedUsers; + } + + public Long getNoOfNewGroups() { + return noOfNewGroups; + } + + public void setNoOfNewGroups(Long noOfNewGroups) { + this.noOfNewGroups = noOfNewGroups; + } + + public Long getNoOfModifiedGroups() { + return noOfModifiedGroups; + } + + public void setNoOfModifiedGroups(Long noOfModifiedGroups) { + this.noOfModifiedGroups = noOfModifiedGroups; + } + + public String getSyncSource() { + return syncSource; + } + + public void setSyncSource(String syncSource) { + this.syncSource = syncSource; + } + + public LdapSyncSourceInfo getLdapSyncSourceInfo() { + return ldapSyncSourceInfo; + } + + public void setLdapSyncSourceInfo(LdapSyncSourceInfo ldapSyncSourceInfo) { + this.ldapSyncSourceInfo = ldapSyncSourceInfo; + } + + public UnixSyncSourceInfo getUnixSyncSourceInfo() { + return unixSyncSourceInfo; + } + + public void setUnixSyncSourceInfo(UnixSyncSourceInfo unixSyncSourceInfo) { + this.unixSyncSourceInfo = unixSyncSourceInfo; + } + + public FileSyncSourceInfo getFileSyncSourceInfo() { + return fileSyncSourceInfo; + } + + public void setFileSyncSourceInfo(FileSyncSourceInfo fileSyncSourceInfo) { + this.fileSyncSourceInfo = fileSyncSourceInfo; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getSessionId() { + return sessionId; + } + + public void setSessionId(String sessionId) { + this.sessionId = sessionId; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + toString(sb); + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("UgsyncAuditInfo [No. of New users= ").append(noOfNewUsers); + sb.append(", No. of New groups= ").append(noOfNewGroups); + sb.append(", No. of Modified users= ").append(noOfModifiedUsers); + sb.append(", No. of Modified groups= ").append(noOfModifiedGroups); + sb.append(", syncSource= ").append(syncSource); + sb.append(", ldapSyncSourceInfo= ").append(ldapSyncSourceInfo); + sb.append(", unixSyncSourceInfo= ").append(unixSyncSourceInfo); + sb.append(", fileSyncSourceInfo= ").append(fileSyncSourceInfo); + sb.append("]"); + return sb; + } } diff --git a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UnixSyncSourceInfo.java b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UnixSyncSourceInfo.java index 17b63629b8..e6cb6a630a 100644 --- a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UnixSyncSourceInfo.java +++ b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UnixSyncSourceInfo.java @@ -17,119 +17,119 @@ * under the License. */ - package org.apache.ranger.ugsyncutil.model; +package org.apache.ranger.ugsyncutil.model; public class UnixSyncSourceInfo { - private String unixBackend; - private String fileName; - private String syncTime; - private String lastModified; - private String minUserId; - private String minGroupId; - private long totalUsersSynced; - private long totalGroupsSynced; - private long totalUsersDeleted; - private long totalGroupsDeleted; - - public String getMinGroupId() { - return minGroupId; - } - - public void setMinGroupId(String minGroupId) { - this.minGroupId = minGroupId; - } - - public String getMinUserId() { - return minUserId; - } - - public void setMinUserId(String minUserId) { - this.minUserId = minUserId; - } - - public String getUnixBackend() { - return unixBackend; - } - - public void setUnixBackend(String unixBackend) { - this.unixBackend = unixBackend; - } - - public String getFileName() { - return fileName; - } - - public void setFileName(String fileName) { - this.fileName = fileName; - } - - public String getSyncTime() { - return syncTime; - } - - public void setSyncTime(String syncTime) { - this.syncTime = syncTime; - } - - public String getLastModified() { - return lastModified; - } - - public void setLastModified(String lastModified) { - this.lastModified = lastModified; - } - - public long getTotalUsersSynced() { - return totalUsersSynced; - } - - public void setTotalUsersSynced(long totalUsersSynced) { - this.totalUsersSynced = totalUsersSynced; - } - - public long getTotalGroupsSynced() { - return totalGroupsSynced; - } - - public void setTotalGroupsSynced(long totalGroupsSynced) { - this.totalGroupsSynced = totalGroupsSynced; - } - - public long getTotalUsersDeleted() { - return totalUsersDeleted; - } - - public void setTotalUsersDeleted(long totalUsersDeleted) { - this.totalUsersDeleted = totalUsersDeleted; - } - - public long getTotalGroupsDeleted() { - return totalGroupsDeleted; - } - - public void setTotalGroupsDeleted(long totalGroupsDeleted) { - this.totalGroupsDeleted = totalGroupsDeleted; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - toString(sb); - return sb.toString(); - } - - public StringBuilder toString(StringBuilder sb) { - sb.append("UnixSycnSourceInfo [unixBackend= ").append(unixBackend); - sb.append(", fileName= ").append(fileName); - sb.append(", syncTime= ").append(syncTime); - sb.append(", lastModified= ").append(lastModified); - sb.append(", minUserId= ").append(minUserId); - sb.append(", minGroupId= ").append(minGroupId); - sb.append(", totalUsersSynced= ").append(totalUsersSynced); - sb.append(", totalGroupsSynced= ").append(totalGroupsSynced); - sb.append(", totalUsersDeleted= ").append(totalUsersDeleted); - sb.append(", totalGroupsDeleted= ").append(totalGroupsDeleted); - sb.append("]"); - return sb; - } + private String unixBackend; + private String fileName; + private String syncTime; + private String lastModified; + private String minUserId; + private String minGroupId; + private long totalUsersSynced; + private long totalGroupsSynced; + private long totalUsersDeleted; + private long totalGroupsDeleted; + + public String getMinGroupId() { + return minGroupId; + } + + public void setMinGroupId(String minGroupId) { + this.minGroupId = minGroupId; + } + + public String getMinUserId() { + return minUserId; + } + + public void setMinUserId(String minUserId) { + this.minUserId = minUserId; + } + + public String getUnixBackend() { + return unixBackend; + } + + public void setUnixBackend(String unixBackend) { + this.unixBackend = unixBackend; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public String getSyncTime() { + return syncTime; + } + + public void setSyncTime(String syncTime) { + this.syncTime = syncTime; + } + + public String getLastModified() { + return lastModified; + } + + public void setLastModified(String lastModified) { + this.lastModified = lastModified; + } + + public long getTotalUsersSynced() { + return totalUsersSynced; + } + + public void setTotalUsersSynced(long totalUsersSynced) { + this.totalUsersSynced = totalUsersSynced; + } + + public long getTotalGroupsSynced() { + return totalGroupsSynced; + } + + public void setTotalGroupsSynced(long totalGroupsSynced) { + this.totalGroupsSynced = totalGroupsSynced; + } + + public long getTotalUsersDeleted() { + return totalUsersDeleted; + } + + public void setTotalUsersDeleted(long totalUsersDeleted) { + this.totalUsersDeleted = totalUsersDeleted; + } + + public long getTotalGroupsDeleted() { + return totalGroupsDeleted; + } + + public void setTotalGroupsDeleted(long totalGroupsDeleted) { + this.totalGroupsDeleted = totalGroupsDeleted; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + toString(sb); + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("UnixSycnSourceInfo [unixBackend= ").append(unixBackend); + sb.append(", fileName= ").append(fileName); + sb.append(", syncTime= ").append(syncTime); + sb.append(", lastModified= ").append(lastModified); + sb.append(", minUserId= ").append(minUserId); + sb.append(", minGroupId= ").append(minGroupId); + sb.append(", totalUsersSynced= ").append(totalUsersSynced); + sb.append(", totalGroupsSynced= ").append(totalGroupsSynced); + sb.append(", totalUsersDeleted= ").append(totalUsersDeleted); + sb.append(", totalGroupsDeleted= ").append(totalGroupsDeleted); + sb.append("]"); + return sb; + } } diff --git a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UsersGroupRoleAssignments.java b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UsersGroupRoleAssignments.java index 123db22ec0..589b5d2a5c 100644 --- a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UsersGroupRoleAssignments.java +++ b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UsersGroupRoleAssignments.java @@ -25,75 +25,77 @@ import java.util.List; import java.util.Map; + @JsonAutoDetect(getterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE, fieldVisibility = JsonAutoDetect.Visibility.ANY) @JsonInclude(JsonInclude.Include.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) public class UsersGroupRoleAssignments { + List users; + + Map groupRoleAssignments; + + Map userRoleAssignments; - List users; - - Map groupRoleAssignments; + Map whiteListGroupRoleAssignments; - Map userRoleAssignments; + Map whiteListUserRoleAssignments; - Map whiteListGroupRoleAssignments; + boolean isReset; + boolean isLastPage; - Map whiteListUserRoleAssignments; + public boolean isLastPage() { + return isLastPage; + } - boolean isReset = false; - boolean isLastPage = false; - public boolean isLastPage() { - return isLastPage; - } - public void setLastPage(boolean lastPage) { - isLastPage = lastPage; - } + public void setLastPage(boolean lastPage) { + isLastPage = lastPage; + } - public List getUsers() { - return users; - } + public List getUsers() { + return users; + } - public void setUsers(List users) { - this.users = users; - } + public void setUsers(List users) { + this.users = users; + } - public Map getGroupRoleAssignments() { - return groupRoleAssignments; - } + public Map getGroupRoleAssignments() { + return groupRoleAssignments; + } - public void setGroupRoleAssignments(Map groupRoleAssignments) { - this.groupRoleAssignments = groupRoleAssignments; - } + public void setGroupRoleAssignments(Map groupRoleAssignments) { + this.groupRoleAssignments = groupRoleAssignments; + } - public Map getUserRoleAssignments() { - return userRoleAssignments; - } + public Map getUserRoleAssignments() { + return userRoleAssignments; + } - public void setUserRoleAssignments(Map userRoleAssignments) { - this.userRoleAssignments = userRoleAssignments; - } + public void setUserRoleAssignments(Map userRoleAssignments) { + this.userRoleAssignments = userRoleAssignments; + } - public Map getWhiteListGroupRoleAssignments() { - return whiteListGroupRoleAssignments; - } + public Map getWhiteListGroupRoleAssignments() { + return whiteListGroupRoleAssignments; + } - public void setWhiteListGroupRoleAssignments(Map whiteListGroupRoleAssignments) { - this.whiteListGroupRoleAssignments = whiteListGroupRoleAssignments; - } + public void setWhiteListGroupRoleAssignments(Map whiteListGroupRoleAssignments) { + this.whiteListGroupRoleAssignments = whiteListGroupRoleAssignments; + } - public Map getWhiteListUserRoleAssignments() { - return whiteListUserRoleAssignments; - } + public Map getWhiteListUserRoleAssignments() { + return whiteListUserRoleAssignments; + } - public void setWhiteListUserRoleAssignments(Map whiteListUserRoleAssignments) { - this.whiteListUserRoleAssignments = whiteListUserRoleAssignments; - } + public void setWhiteListUserRoleAssignments(Map whiteListUserRoleAssignments) { + this.whiteListUserRoleAssignments = whiteListUserRoleAssignments; + } - public boolean isReset() { - return isReset; - } + public boolean isReset() { + return isReset; + } - public void setReset(boolean reset) { - isReset = reset; - } + public void setReset(boolean reset) { + isReset = reset; + } } diff --git a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/XGroupInfo.java b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/XGroupInfo.java index e8f3e961c0..1a69ec3ca7 100644 --- a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/XGroupInfo.java +++ b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/XGroupInfo.java @@ -17,103 +17,117 @@ * under the License. */ - package org.apache.ranger.ugsyncutil.model; +package org.apache.ranger.ugsyncutil.model; import java.util.HashMap; import java.util.Map; public class XGroupInfo { - - private String id; - private String name; - private String description; - private String groupType; - private String isVisible; - private String groupSource; - private String otherAttributes; - private String syncSource; - private Map otherAttrsMap = new HashMap<>(); - public String getId() { - return id; - } - public void setId(String id) { - this.id = id; - } - public String getName() { - return name; - } - public void setName(String name) { - this.name = name; - } - public String getDescription() { - return description; - } - public void setDescription(String description) { - this.description = description; - } - - public String getGroupType() { - return groupType; - } - public void setGroupType(String groupType) { - this.groupType = groupType; - } - - public String getIsVisible() { - return isVisible; - } - - public void setIsVisible(String isVisible) { - this.isVisible = isVisible; - } - - public String getGroupSource() { - return groupSource; - } - public void setGroupSource(String groupSource) { - this.groupSource = groupSource; - } - - public Map getOtherAttrsMap() { - return otherAttrsMap; - } - - public void setOtherAttrsMap(Map otherAttrsMap) { - if (otherAttrsMap != null) { - this.otherAttrsMap = otherAttrsMap; - } - } - - public String getOtherAttributes() { - return otherAttributes; - } - - public void setOtherAttributes(String otherAttributes) { - this.otherAttributes = otherAttributes; - } - - public String getSyncSource() { return syncSource; } - - public void setSyncSource(String syncSource) { this.syncSource = syncSource; } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - XGroupInfo groupInfo = (XGroupInfo) o; - if (name == null) { - if (groupInfo.name != null) - return false; - } else if (!name.equals(groupInfo.name)) - return false; - return true; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((name == null) ? 0 : name.hashCode()); - return result; - } + private String id; + private String name; + private String description; + private String groupType; + private String isVisible; + private String groupSource; + private String otherAttributes; + private String syncSource; + private Map otherAttrsMap = new HashMap<>(); + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getGroupType() { + return groupType; + } + + public void setGroupType(String groupType) { + this.groupType = groupType; + } + + public String getIsVisible() { + return isVisible; + } + + public void setIsVisible(String isVisible) { + this.isVisible = isVisible; + } + + public String getGroupSource() { + return groupSource; + } + + public void setGroupSource(String groupSource) { + this.groupSource = groupSource; + } + + public Map getOtherAttrsMap() { + return otherAttrsMap; + } + + public void setOtherAttrsMap(Map otherAttrsMap) { + if (otherAttrsMap != null) { + this.otherAttrsMap = otherAttrsMap; + } + } + + public String getOtherAttributes() { + return otherAttributes; + } + + public void setOtherAttributes(String otherAttributes) { + this.otherAttributes = otherAttributes; + } + + public String getSyncSource() { + return syncSource; + } + + public void setSyncSource(String syncSource) { + this.syncSource = syncSource; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((name == null) ? 0 : name.hashCode()); + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + XGroupInfo groupInfo = (XGroupInfo) o; + if (name == null) { + return groupInfo.name == null; + } else { + return name.equals(groupInfo.name); + } + } } diff --git a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/XUserInfo.java b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/XUserInfo.java index 8012f15d97..fd61ca5465 100644 --- a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/XUserInfo.java +++ b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/XUserInfo.java @@ -17,7 +17,7 @@ * under the License. */ - package org.apache.ranger.ugsyncutil.model; +package org.apache.ranger.ugsyncutil.model; import java.util.ArrayList; import java.util.HashMap; @@ -25,88 +25,92 @@ import java.util.Map; public class XUserInfo { - private String id; - private String name; - private String firstName; - private String description; - private String otherAttributes; - private String syncSource; - private Map otherAttrsMap = new HashMap<>(); - private String userSource; - private String status; - private String isVisible; - private List groupNameList = new ArrayList(); - private List userRoleList = new ArrayList(); - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getFirstName() { return firstName; } - - public void setFirstName(String firstName) { this.firstName = firstName; } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getUserSource() { - return userSource; - } - - public void setUserSource(String userSource) { - this.userSource = userSource; - } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status; - } - - public String getIsVisible() { - return isVisible; - } - - public void setIsVisible(String isVisible) { - this.isVisible = isVisible; - } - - public void setGroupNameList(List groupNameList) { - this.groupNameList = groupNameList; - } - - public List getGroupNameList() { - return groupNameList; - } - - public void deleteGroups(List delGroups) { - for (String delGroup : delGroups) { - groupNameList.remove(delGroup); - } - } - - public List getGroups() { - return groupNameList; - } + private String id; + private String name; + private String firstName; + private String description; + private String otherAttributes; + private String syncSource; + private Map otherAttrsMap = new HashMap<>(); + private String userSource; + private String status; + private String isVisible; + private List groupNameList = new ArrayList(); + private List userRoleList = new ArrayList(); + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getFirstName() { + return firstName; + } + + public void setFirstName(String firstName) { + this.firstName = firstName; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getUserSource() { + return userSource; + } + + public void setUserSource(String userSource) { + this.userSource = userSource; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getIsVisible() { + return isVisible; + } + + public void setIsVisible(String isVisible) { + this.isVisible = isVisible; + } + + public List getGroupNameList() { + return groupNameList; + } + + public void setGroupNameList(List groupNameList) { + this.groupNameList = groupNameList; + } + + public void deleteGroups(List delGroups) { + for (String delGroup : delGroups) { + groupNameList.remove(delGroup); + } + } + + public List getGroups() { + return groupNameList; + } public List getUserRoleList() { return userRoleList; @@ -116,53 +120,67 @@ public void setUserRoleList(List userRoleList) { this.userRoleList = userRoleList; } - public Map getOtherAttrsMap() { - return otherAttrsMap; - } + public Map getOtherAttrsMap() { + return otherAttrsMap; + } + + public void setOtherAttrsMap(Map otherAttrsMap) { + if (otherAttrsMap != null) { + this.otherAttrsMap = otherAttrsMap; + } + } - public void setOtherAttrsMap(Map otherAttrsMap) { - if (otherAttrsMap != null) { - this.otherAttrsMap = otherAttrsMap; - } - } + public String getOtherAttributes() { + return otherAttributes; + } - public String getOtherAttributes() { - return otherAttributes; - } + public void setOtherAttributes(String otherAttributes) { + this.otherAttributes = otherAttributes; + } - public void setOtherAttributes(String otherAttributes) { - this.otherAttributes = otherAttributes; - } + public String getSyncSource() { + return syncSource; + } - public String getSyncSource() { return syncSource; } + public void setSyncSource(String syncSource) { + this.syncSource = syncSource; + } - public void setSyncSource(String syncSource) { this.syncSource = syncSource; } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((name == null) ? 0 : name.hashCode()); + return result; + } - @Override - public String toString() { - return "XUserInfo [id=" + id + ", name=" + name + ", firstName=" + firstName + ", description=" - + description + ", groupNameList=" + groupNameList - + ", userRoleList=" + userRoleList + "]"; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + XUserInfo userInfo = (XUserInfo) o; + if (name == null) { + return userInfo.name == null; + } else { + return name.equals(userInfo.name); + } } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - XUserInfo userInfo = (XUserInfo) o; - if (name == null) { - if (userInfo.name != null) - return false; - } else if (!name.equals(userInfo.name)) - return false; - return true; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((name == null) ? 0 : name.hashCode()); - return result; - } + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("XUserInfo ["); + sb.append("id=").append(id).append(", "); + sb.append("name=").append(name).append(", "); + sb.append("firstName=").append(firstName).append(", "); + sb.append("description=").append(description).append(", "); + sb.append("groupNameList=").append(groupNameList).append(", "); + sb.append("userRoleList=").append(userRoleList); + sb.append("]"); + return sb.toString(); + } } diff --git a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java index f20bf91967..704e8c4be2 100644 --- a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java +++ b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java @@ -20,10 +20,11 @@ package org.apache.ranger.ugsyncutil.util; public class UgsyncCommonConstants { - public static final String ORIGINAL_NAME = "original_name"; - public static final String FULL_NAME = "full_name"; - public static final String SYNC_SOURCE = "sync_source"; - public static final String LDAP_URL = "ldap_url"; + public static final String FULL_NAME = "full_name"; + public static final String SYNC_SOURCE = "sync_source"; + public static final String LDAP_URL = "ldap_url"; + private UgsyncCommonConstants() { + } } diff --git a/ugsync/ldapconfigchecktool/ldapconfigcheck/pom.xml b/ugsync/ldapconfigchecktool/ldapconfigcheck/pom.xml index 95ebc76ffc..fca9c9d708 100644 --- a/ugsync/ldapconfigchecktool/ldapconfigcheck/pom.xml +++ b/ugsync/ldapconfigchecktool/ldapconfigcheck/pom.xml @@ -27,6 +27,9 @@ jar Ldap Config Check Tool Ldap configuration check tool + + true + commons-cli diff --git a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/AuthenticationCheck.java b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/AuthenticationCheck.java index b2b37dd59e..bc15f685bd 100644 --- a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/AuthenticationCheck.java +++ b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/AuthenticationCheck.java @@ -19,42 +19,39 @@ package org.apache.ranger.ldapconfigcheck; -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.List; import org.springframework.ldap.core.support.LdapContextSource; import org.springframework.security.authentication.BadCredentialsException; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; import org.springframework.security.core.Authentication; +import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.authority.SimpleGrantedAuthority; -import org.springframework.security.core.userdetails.UserDetails; import org.springframework.security.core.userdetails.User; -import org.springframework.security.ldap.userdetails.DefaultLdapAuthoritiesPopulator; -import org.springframework.security.core.GrantedAuthority; -import org.springframework.security.ldap.search.FilterBasedLdapUserSearch; +import org.springframework.security.core.userdetails.UserDetails; +import org.springframework.security.ldap.DefaultSpringSecurityContextSource; import org.springframework.security.ldap.authentication.BindAuthenticator; import org.springframework.security.ldap.authentication.LdapAuthenticationProvider; -import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; -import org.springframework.security.ldap.DefaultSpringSecurityContextSource; +import org.springframework.security.ldap.search.FilterBasedLdapUserSearch; +import org.springframework.security.ldap.userdetails.DefaultLdapAuthoritiesPopulator; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; public class AuthenticationCheck { - private String ldapUrl = null; + private String ldapUrl; + private String adDomain; + private String userDnPattern; + private String roleAttribute; + private String groupSearchBase; + private String groupSearchFilter; + private PrintStream logFile; + private PrintStream ambariProps; + private PrintStream installProps; private String authMethod = "NONE"; - private String adDomain = null; - private String userDnPattern = null; - private String roleAttribute = null; - private String groupSearchBase = null; - private String groupSearchFilter = null; - - private PrintStream logFile = null; - private PrintStream ambariProps = null; - private PrintStream installProps = null; - public AuthenticationCheck(String ldapUrl, UserSync userSyncObj, PrintStream logFile, - PrintStream ambariProps, PrintStream installProps) { - - this.logFile = logFile; - this.ambariProps = ambariProps; + public AuthenticationCheck(String ldapUrl, UserSync userSyncObj, PrintStream logFile, PrintStream ambariProps, PrintStream installProps) { + this.logFile = logFile; + this.ambariProps = ambariProps; this.installProps = installProps; if (userSyncObj.getUserNameAttribute().equalsIgnoreCase("sAMAccountName")) { @@ -62,17 +59,15 @@ public AuthenticationCheck(String ldapUrl, UserSync userSyncObj, PrintStream log } else { authMethod = "LDAP"; } - this.ldapUrl = ldapUrl; - adDomain = userSyncObj.getSearchBase(); - userDnPattern = userSyncObj.getUserNameAttribute() + "={0}," + userSyncObj.getUserSearchBase(); - roleAttribute = userSyncObj.getGroupNameAttrName(); - groupSearchBase = userSyncObj.getGroupSearchBase(); + this.ldapUrl = ldapUrl; + adDomain = userSyncObj.getSearchBase(); + userDnPattern = userSyncObj.getUserNameAttribute() + "={0}," + userSyncObj.getUserSearchBase(); + roleAttribute = userSyncObj.getGroupNameAttrName(); + groupSearchBase = userSyncObj.getGroupSearchBase(); groupSearchFilter = userSyncObj.getGroupMemberName() + "=" + userDnPattern; - } public void discoverAuthProperties() { - ambariProps.println("\n# Possible values for authentication properties:"); installProps.println("\n# Possible values for authentication properties:"); if (authMethod.equalsIgnoreCase("AD")) { @@ -99,8 +94,7 @@ public void discoverAuthProperties() { } } - public boolean isAuthenticated(String ldapUrl, String bindDn, String bindPassword, String userName, - String userPassword) { + public boolean isAuthenticated(String ldapUrl, String bindDn, String bindPassword, String userName, String userPassword) { boolean isAuthenticated = false; //Verify Authentication Authentication authentication; @@ -116,8 +110,7 @@ public boolean isAuthenticated(String ldapUrl, String bindDn, String bindPasswor return isAuthenticated; } - private Authentication getADBindAuthentication(String ldapUrl, String bindDn, String bindPassword, - String userName, String userPassword) { + private Authentication getADBindAuthentication(String ldapUrl, String bindDn, String bindPassword, String userName, String userPassword) { Authentication result = null; try { LdapContextSource ldapContextSource = new DefaultSpringSecurityContextSource(ldapUrl); @@ -129,8 +122,8 @@ private Authentication getADBindAuthentication(String ldapUrl, String bindDn, St ldapContextSource.setPooled(true); ldapContextSource.afterPropertiesSet(); - String searchFilter="(sAMAccountName={0})"; - FilterBasedLdapUserSearch userSearch=new FilterBasedLdapUserSearch(adDomain, searchFilter,ldapContextSource); + String searchFilter = "(sAMAccountName={0})"; + FilterBasedLdapUserSearch userSearch = new FilterBasedLdapUserSearch(adDomain, searchFilter, ldapContextSource); userSearch.setSearchSubtree(true); BindAuthenticator bindAuthenticator = new BindAuthenticator(ldapContextSource); @@ -142,23 +135,20 @@ private Authentication getADBindAuthentication(String ldapUrl, String bindDn, St if (userName != null && userPassword != null && !userName.trim().isEmpty() && !userPassword.trim().isEmpty()) { final List grantedAuths = new ArrayList<>(); grantedAuths.add(new SimpleGrantedAuthority("ROLE_USER")); - final UserDetails principal = new User(userName, userPassword, grantedAuths); + final UserDetails principal = new User(userName, userPassword, grantedAuths); final Authentication finalAuthentication = new UsernamePasswordAuthenticationToken(principal, userPassword, grantedAuths); result = ldapAuthenticationProvider.authenticate(finalAuthentication); } - } catch (BadCredentialsException bce) { - logFile.println("ERROR: LDAP Authentication Failed. Please verify values for ranger.admin.auth.sampleuser and " + - "ranger.admin.auth.samplepassword\n"); + logFile.println("ERROR: LDAP Authentication Failed. Please verify values for ranger.admin.auth.sampleuser and ranger.admin.auth.samplepassword\n"); } catch (Exception e) { logFile.println("ERROR: LDAP Authentication Failed: " + e); } return result; } - private Authentication getLdapBindAuthentication(String ldapUrl, String bindDn, String bindPassword, - String userName, String userPassword) { + private Authentication getLdapBindAuthentication(String ldapUrl, String bindDn, String bindPassword, String userName, String userPassword) { Authentication result = null; try { LdapContextSource ldapContextSource = new DefaultSpringSecurityContextSource(ldapUrl); @@ -175,34 +165,31 @@ private Authentication getLdapBindAuthentication(String ldapUrl, String bindDn, defaultLdapAuthoritiesPopulator.setGroupSearchFilter(groupSearchFilter); defaultLdapAuthoritiesPopulator.setIgnorePartialResultException(true); - String searchFilter="(uid={0})"; - FilterBasedLdapUserSearch userSearch=new FilterBasedLdapUserSearch(adDomain, searchFilter,ldapContextSource); + String searchFilter = "(uid={0})"; + FilterBasedLdapUserSearch userSearch = new FilterBasedLdapUserSearch(adDomain, searchFilter, ldapContextSource); userSearch.setSearchSubtree(true); BindAuthenticator bindAuthenticator = new BindAuthenticator(ldapContextSource); bindAuthenticator.setUserSearch(userSearch); - String[] userDnPatterns = new String[] { userDnPattern }; + String[] userDnPatterns = new String[] {userDnPattern}; bindAuthenticator.setUserDnPatterns(userDnPatterns); bindAuthenticator.afterPropertiesSet(); - LdapAuthenticationProvider ldapAuthenticationProvider = new LdapAuthenticationProvider(bindAuthenticator,defaultLdapAuthoritiesPopulator); + LdapAuthenticationProvider ldapAuthenticationProvider = new LdapAuthenticationProvider(bindAuthenticator, defaultLdapAuthoritiesPopulator); if (userName != null && userPassword != null && !userName.trim().isEmpty() && !userPassword.trim().isEmpty()) { final List grantedAuths = new ArrayList<>(); grantedAuths.add(new SimpleGrantedAuthority("ROLE_USER")); - final UserDetails principal = new User(userName, userPassword, grantedAuths); + final UserDetails principal = new User(userName, userPassword, grantedAuths); final Authentication finalAuthentication = new UsernamePasswordAuthenticationToken(principal, userPassword, grantedAuths); result = ldapAuthenticationProvider.authenticate(finalAuthentication); } } catch (BadCredentialsException bce) { - logFile.println("ERROR: LDAP Authentication Failed. Please verify values for ranger.admin.auth.sampleuser and " + - "ranger.admin.auth.samplepassword\n"); + logFile.println("ERROR: LDAP Authentication Failed. Please verify values for ranger.admin.auth.sampleuser and ranger.admin.auth.samplepassword\n"); } catch (Exception e) { logFile.println("ERROR: LDAP Authentication Failed: " + e); } return result; } } - - diff --git a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/CommandLineOptions.java b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/CommandLineOptions.java index 6e23a06127..b137aef0cc 100644 --- a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/CommandLineOptions.java +++ b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/CommandLineOptions.java @@ -19,25 +19,30 @@ package org.apache.ranger.ldapconfigcheck; -import org.apache.commons.cli.*; +import org.apache.commons.cli.BasicParser; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + import java.io.Console; public class CommandLineOptions { - - private String[] args = null; - private Options options = new Options(); - private String input = null; - private String output = null; - private String discoverProperties = null; - private String retrieveValues = null; - private boolean isAuthEnabled = true; - private String ldapUrl = ""; - private String bindDn = ""; - private String bindPassword = ""; - private String userSearchBase = ""; - private String userSearchFilter = ""; - private String authUser = ""; - private String authPass = ""; + private String[] args; + private String input; + private String output; + private String discoverProperties; + private String retrieveValues; + private String ldapUrl = ""; + private String bindDn = ""; + private String bindPassword = ""; + private String userSearchBase = ""; + private String userSearchFilter = ""; + private String authUser = ""; + private String authPass = ""; + private Options options = new Options(); + private boolean isAuthEnabled = true; public CommandLineOptions(String[] args) { this.args = args; @@ -52,17 +57,14 @@ public CommandLineOptions(String[] args) { } public void parse() { - CommandLineParser parser = new BasicParser(); + CommandLineParser parser = new BasicParser(); try { CommandLine cmd = parser.parse(options, args); - // if (cmd.hasOption("h")) { - //} - if (cmd.hasOption("p")) { - bindPassword = cmd.getOptionValue("p"); - if (bindPassword.trim().isEmpty()) { - System.out.println("Ldap Bind Password cannot be empty!"); - } + bindPassword = cmd.getOptionValue("p"); + if (bindPassword.trim().isEmpty()) { + System.out.println("Ldap Bind Password cannot be empty!"); + } } if (cmd.hasOption("u")) { @@ -88,18 +90,13 @@ public void parse() { } } else if (cmd.hasOption("r")) { retrieveValues = cmd.getOptionValue("r"); - if (retrieveValues == null || (!retrieveValues.equalsIgnoreCase("all") - && !retrieveValues.equalsIgnoreCase("users") && !retrieveValues.equalsIgnoreCase("groups"))) { + if (retrieveValues == null || (!retrieveValues.equalsIgnoreCase("all") && !retrieveValues.equalsIgnoreCase("users") && !retrieveValues.equalsIgnoreCase("groups"))) { System.out.println("Unsupported value for option r"); help(); } } else { - //if (discoverProperties == null || discoverProperties.isEmpty()) { - System.out.println("Default to discover all usersync properties"); - //help(); - // If "d" or "r" option is not specified, then default to discover all usersync properties - discoverProperties = "all"; - //} + System.out.println("Default to discover all usersync properties"); + discoverProperties = "all"; } if (cmd.hasOption("a") || discoverProperties == null || (discoverProperties != null && !discoverProperties.equalsIgnoreCase("all"))) { @@ -114,15 +111,13 @@ public void parse() { } if (bindPassword == null || bindPassword.trim().isEmpty()) { - System.out.println("Missing Ldap Bind Password!"); - } - + System.out.println("Missing Ldap Bind Password!"); + } } else { // Read the properties from CLI and write to the input properties file. input = LdapConfig.CONFIG_FILE; readCLI(); } - } catch (ParseException pe) { System.out.println("Failed to parse command line arguments " + pe); help(); @@ -130,7 +125,6 @@ public void parse() { } public void help() { - // This prints out some help HelpFormatter formater = new HelpFormatter(); formater.printHelp("ldapConfigCheck", options); System.exit(0); @@ -141,7 +135,6 @@ public String getInput() { } public String getOutput() { - return output; } @@ -157,6 +150,34 @@ public String getRetrieveValues() { return retrieveValues; } + public String getLdapUrl() { + return ldapUrl; + } + + public String getBindDn() { + return bindDn; + } + + public String getBindPassword() { + return bindPassword; + } + + public String getUserSearchBase() { + return userSearchBase; + } + + public String getUserSearchFilter() { + return userSearchFilter; + } + + public String getAuthUser() { + return authUser; + } + + public String getAuthPass() { + return authPass; + } + private void readCLI() { boolean repeat; Console console = System.console(); @@ -168,7 +189,8 @@ private void readCLI() { System.out.println("Please enter valid ldap url."); repeat = true; } - } while (repeat == true); + } + while (repeat == true); do { repeat = false; System.out.print("Bind DN [cn=admin,ou=users,dc=example,dc=com]: "); @@ -177,7 +199,8 @@ private void readCLI() { System.out.println("Please enter valid bindDn."); repeat = true; } - } while (repeat == true); + } + while (repeat == true); do { repeat = false; System.out.print("Bind Password: "); @@ -187,7 +210,8 @@ private void readCLI() { System.out.println("Bind Password can't be empty."); repeat = true; } - } while (repeat == true); + } + while (repeat == true); System.out.print("User Search Base [ou=users,dc=example,dc=com]: "); userSearchBase = console.readLine(); System.out.print("User Search Filter [cn=user1]: "); @@ -202,7 +226,8 @@ private void readCLI() { System.out.println("Sample Authentication user must not be empty!"); repeat = true; } - } while (repeat == true); + } + while (repeat == true); do { repeat = false; System.out.print("Sample Authentication Password: "); @@ -212,35 +237,8 @@ private void readCLI() { System.out.println("Sample Authentication password must not be empty!"); repeat = true; } - } while (repeat == true); + } + while (repeat == true); } } - - public String getLdapUrl() { - return ldapUrl; - } - - public String getBindDn() { - return bindDn; - } - - public String getBindPassword() { - return bindPassword; - } - - public String getUserSearchBase() { - return userSearchBase; - } - - public String getUserSearchFilter() { - return userSearchFilter; - } - - public String getAuthUser() { - return authUser; - } - - public String getAuthPass() { - return authPass; - } } diff --git a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfig.java b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfig.java index 46eafa9d25..9165434c18 100644 --- a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfig.java +++ b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfig.java @@ -19,13 +19,6 @@ package org.apache.ranger.ldapconfigcheck; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.Properties; - import org.apache.commons.configuration2.FileBasedConfiguration; import org.apache.commons.configuration2.PropertiesConfiguration; import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder; @@ -33,153 +26,61 @@ import org.apache.commons.configuration2.ex.ConfigurationException; import org.apache.commons.lang.NullArgumentException; -public class LdapConfig { - - public static final String CONFIG_FILE = "input.properties"; - - private static final String LGSYNC_LDAP_URL = "ranger.usersync.ldap.url"; - - private static final String LGSYNC_LDAP_BIND_DN = "ranger.usersync.ldap.binddn"; - - private static final String LGSYNC_LDAP_BIND_PASSWORD = "ranger.usersync.ldap.ldapbindpassword"; - - private static final String LGSYNC_LDAP_AUTHENTICATION_MECHANISM = "ranger.usersync.ldap.authentication.mechanism"; - private static final String DEFAULT_AUTHENTICATION_MECHANISM = "simple"; - - private static final String LGSYNC_SEARCH_BASE = "ranger.usersync.ldap.searchBase"; - - private static final String LGSYNC_USER_SEARCH_BASE = "ranger.usersync.ldap.user.searchbase"; - - private static final String LGSYNC_USER_SEARCH_SCOPE = "ranger.usersync.ldap.user.searchscope"; - - private static final String LGSYNC_USER_OBJECT_CLASS = "ranger.usersync.ldap.user.objectclass"; - - private static final String LGSYNC_USER_SEARCH_FILTER = "ranger.usersync.ldap.user.searchfilter"; - - private static final String LGSYNC_USER_NAME_ATTRIBUTE = "ranger.usersync.ldap.user.nameattribute"; - - private static final String LGSYNC_USER_GROUP_NAME_ATTRIBUTE = "ranger.usersync.ldap.user.groupnameattribute"; - - public static final String UGSYNC_NONE_CASE_CONVERSION_VALUE = "none"; - public static final String UGSYNC_LOWER_CASE_CONVERSION_VALUE = "lower"; - - private static final String UGSYNC_USERNAME_CASE_CONVERSION_PARAM = "ranger.usersync.ldap.username.caseconversion"; - private static final String DEFAULT_UGSYNC_USERNAME_CASE_CONVERSION_VALUE = UGSYNC_NONE_CASE_CONVERSION_VALUE; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; - private static final String UGSYNC_GROUPNAME_CASE_CONVERSION_PARAM = "ranger.usersync.ldap.groupname.caseconversion"; +public class LdapConfig { + public static final String CONFIG_FILE = "input.properties"; + public static final String UGSYNC_NONE_CASE_CONVERSION_VALUE = "none"; + public static final String UGSYNC_LOWER_CASE_CONVERSION_VALUE = "lower"; + private static final String LGSYNC_LDAP_URL = "ranger.usersync.ldap.url"; + private static final String LGSYNC_LDAP_BIND_DN = "ranger.usersync.ldap.binddn"; + private static final String LGSYNC_LDAP_BIND_PASSWORD = "ranger.usersync.ldap.ldapbindpassword"; + private static final String LGSYNC_LDAP_AUTHENTICATION_MECHANISM = "ranger.usersync.ldap.authentication.mechanism"; + private static final String DEFAULT_AUTHENTICATION_MECHANISM = "simple"; + private static final String LGSYNC_SEARCH_BASE = "ranger.usersync.ldap.searchBase"; + private static final String LGSYNC_USER_SEARCH_BASE = "ranger.usersync.ldap.user.searchbase"; + private static final String LGSYNC_USER_SEARCH_SCOPE = "ranger.usersync.ldap.user.searchscope"; + private static final String LGSYNC_USER_OBJECT_CLASS = "ranger.usersync.ldap.user.objectclass"; + private static final String LGSYNC_USER_SEARCH_FILTER = "ranger.usersync.ldap.user.searchfilter"; + private static final String LGSYNC_USER_NAME_ATTRIBUTE = "ranger.usersync.ldap.user.nameattribute"; + private static final String LGSYNC_USER_GROUP_NAME_ATTRIBUTE = "ranger.usersync.ldap.user.groupnameattribute"; + private static final String UGSYNC_USERNAME_CASE_CONVERSION_PARAM = "ranger.usersync.ldap.username.caseconversion"; + private static final String UGSYNC_GROUPNAME_CASE_CONVERSION_PARAM = "ranger.usersync.ldap.groupname.caseconversion"; + private static final String DEFAULT_UGSYNC_USERNAME_CASE_CONVERSION_VALUE = UGSYNC_NONE_CASE_CONVERSION_VALUE; private static final String DEFAULT_UGSYNC_GROUPNAME_CASE_CONVERSION_VALUE = UGSYNC_NONE_CASE_CONVERSION_VALUE; - - - private static final String LGSYNC_PAGED_RESULTS_ENABLED = "ranger.usersync.pagedresultsenabled"; + private static final String LGSYNC_PAGED_RESULTS_ENABLED = "ranger.usersync.pagedresultsenabled"; + private static final String LGSYNC_PAGED_RESULTS_SIZE = "ranger.usersync.pagedresultssize"; + private static final String LGSYNC_GROUP_SEARCH_ENABLED = "ranger.usersync.group.searchenabled"; + private static final String LGSYNC_GROUP_SEARCH_BASE = "ranger.usersync.group.searchbase"; + private static final String LGSYNC_GROUP_SEARCH_SCOPE = "ranger.usersync.group.searchscope"; + private static final String LGSYNC_GROUP_OBJECT_CLASS = "ranger.usersync.group.objectclass"; + private static final String LGSYNC_GROUP_SEARCH_FILTER = "ranger.usersync.group.searchfilter"; + private static final String LGSYNC_GROUP_NAME_ATTRIBUTE = "ranger.usersync.group.nameattribute"; + private static final String LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = "ranger.usersync.group.memberattributename"; + private static final int DEFAULT_LGSYNC_PAGED_RESULTS_SIZE = 500; private static final boolean DEFAULT_LGSYNC_PAGED_RESULTS_ENABLED = true; - - private static final String LGSYNC_PAGED_RESULTS_SIZE = "ranger.usersync.pagedresultssize"; - private static final int DEFAULT_LGSYNC_PAGED_RESULTS_SIZE = 500; - - private static final String LGSYNC_GROUP_SEARCH_ENABLED = "ranger.usersync.group.searchenabled"; - private static final boolean DEFAULT_LGSYNC_GROUP_SEARCH_ENABLED = false; - - private static final String LGSYNC_GROUP_SEARCH_BASE = "ranger.usersync.group.searchbase"; - - private static final String LGSYNC_GROUP_SEARCH_SCOPE = "ranger.usersync.group.searchscope"; - - private static final String LGSYNC_GROUP_OBJECT_CLASS = "ranger.usersync.group.objectclass"; - - private static final String LGSYNC_GROUP_SEARCH_FILTER = "ranger.usersync.group.searchfilter"; - - private static final String LGSYNC_GROUP_NAME_ATTRIBUTE = "ranger.usersync.group.nameattribute"; - - private static final String LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = "ranger.usersync.group.memberattributename"; + private static final boolean DEFAULT_LGSYNC_GROUP_SEARCH_ENABLED = false; //Authentication related properties - private static final String AUTHENTICATION_METHOD = "ranger.authentication.method"; - private static final String AD_DOMAIN = "ranger.ldap.ad.domain"; - private static final String USER_DN_PATTERN = "ranger.ldap.user.dnpattern"; - private static final String GROUP_ROLE_ATTRIBUTE = "ranger.ldap.group.roleattribute"; - private static final String GROUP_SEARCH_BASE = "ranger.ldap.group.searchbase"; - private static final String GROUP_SEARCH_FILTER = "ranger.ldap.group.searchfilter"; - private static final String AUTH_USERNAME = "ranger.admin.auth.sampleuser"; - private static final String AUTH_PASSWORD = "ranger.admin.auth.samplepassword"; - - - private Properties prop = new Properties(); - + private static final String AUTHENTICATION_METHOD = "ranger.authentication.method"; + private static final String AD_DOMAIN = "ranger.ldap.ad.domain"; + private static final String USER_DN_PATTERN = "ranger.ldap.user.dnpattern"; + private static final String GROUP_ROLE_ATTRIBUTE = "ranger.ldap.group.roleattribute"; + private static final String GROUP_SEARCH_BASE = "ranger.ldap.group.searchbase"; + private static final String GROUP_SEARCH_FILTER = "ranger.ldap.group.searchfilter"; + private static final String AUTH_USERNAME = "ranger.admin.auth.sampleuser"; + private static final String AUTH_PASSWORD = "ranger.admin.auth.samplepassword"; + private Properties prop = new Properties(); public LdapConfig(String configFile, String bindPasswd) { init(configFile, bindPasswd); } - private void init(String configFile, String bindPasswd) { - readConfigFile(configFile); - prop.setProperty(LGSYNC_LDAP_BIND_PASSWORD, bindPasswd); - } - - private void readConfigFile(String fileName) { - try { - InputStream in = getFileInputStream(fileName); - if (in != null) { - try { - System.out.println("Reading ldap properties from " + fileName); - prop.load(in); - - } finally { - try { - in.close(); - } catch (IOException ioe) { - // Ignore IOE when closing stream - System.out.println(ioe); - } - } - } - } catch (Throwable e) { - throw new RuntimeException("Unable to load configuration file [" + fileName + "]", e); - } - } - - - /*private InputStream getFileInputStream(String path) throws FileNotFoundException { - - InputStream ret = null; - - File f = new File(path); - - if (f.exists()) { - ret = new FileInputStream(f); - } - - return ret; - }*/ - - private InputStream getFileInputStream(String path) throws FileNotFoundException { - - InputStream ret = null; - - File f = new File(path); - - if (f.exists()) { - ret = new FileInputStream(f); - } else { - ret = getClass().getResourceAsStream(path); - - if (ret == null) { - if (! path.startsWith("/")) { - ret = getClass().getResourceAsStream("/" + path); - } - } - - if (ret == null) { - ret = ClassLoader.getSystemClassLoader().getResourceAsStream(path); - if (ret == null) { - if (! path.startsWith("/")) { - ret = ClassLoader.getSystemResourceAsStream("/" + path); - } - } - } - } - - return ret; - } - public String getLdapUrl() throws Throwable { String val = prop.getProperty(LGSYNC_LDAP_URL); if (val == null || val.trim().isEmpty()) { @@ -188,7 +89,6 @@ public String getLdapUrl() throws Throwable { return val; } - public String getLdapBindDn() throws Throwable { String val = prop.getProperty(LGSYNC_LDAP_BIND_DN); if (val == null || val.trim().isEmpty()) { @@ -197,7 +97,6 @@ public String getLdapBindDn() throws Throwable { return val; } - public String getLdapBindPassword() { //update credential from keystore if (prop == null) { @@ -206,7 +105,6 @@ public String getLdapBindPassword() { return prop.getProperty(LGSYNC_LDAP_BIND_PASSWORD); } - public String getLdapAuthenticationMechanism() { String val = prop.getProperty(LGSYNC_LDAP_AUTHENTICATION_MECHANISM); if (val == null || val.trim().isEmpty()) { @@ -215,7 +113,6 @@ public String getLdapAuthenticationMechanism() { return val; } - public String getUserSearchBase() { String val = prop.getProperty(LGSYNC_USER_SEARCH_BASE); if (val == null || val.trim().isEmpty()) { @@ -224,7 +121,6 @@ public String getUserSearchBase() { return val; } - public int getUserSearchScope() { String val = prop.getProperty(LGSYNC_USER_SEARCH_SCOPE); if (val == null || val.trim().isEmpty()) { @@ -241,25 +137,20 @@ public int getUserSearchScope() { } } - public String getUserObjectClass() { - String val = prop.getProperty(LGSYNC_USER_OBJECT_CLASS); - return val; + return prop.getProperty(LGSYNC_USER_OBJECT_CLASS); } public String getUserSearchFilter() { return prop.getProperty(LGSYNC_USER_SEARCH_FILTER); } - public String getUserNameAttribute() { - String val = prop.getProperty(LGSYNC_USER_NAME_ATTRIBUTE); - return val; + return prop.getProperty(LGSYNC_USER_NAME_ATTRIBUTE); } public String getUserGroupNameAttribute() { - String val = prop.getProperty(LGSYNC_USER_GROUP_NAME_ATTRIBUTE); - return val; + return prop.getProperty(LGSYNC_USER_GROUP_NAME_ATTRIBUTE); } public String getUserNameCaseConversion() { @@ -278,7 +169,7 @@ public String getSearchBase() { public boolean isPagedResultsEnabled() { boolean pagedResultsEnabled; - String val = prop.getProperty(LGSYNC_PAGED_RESULTS_ENABLED); + String val = prop.getProperty(LGSYNC_PAGED_RESULTS_ENABLED); if (val == null || val.trim().isEmpty()) { pagedResultsEnabled = DEFAULT_LGSYNC_PAGED_RESULTS_ENABLED; } else { @@ -288,7 +179,7 @@ public boolean isPagedResultsEnabled() { } public int getPagedResultsSize() { - int pagedResultsSize; + int pagedResultsSize; String val = prop.getProperty(LGSYNC_PAGED_RESULTS_SIZE); if (val == null || val.trim().isEmpty()) { pagedResultsSize = DEFAULT_LGSYNC_PAGED_RESULTS_SIZE; @@ -303,7 +194,7 @@ public int getPagedResultsSize() { public boolean isGroupSearchEnabled() { boolean groupSearchEnabled; - String val = prop.getProperty(LGSYNC_GROUP_SEARCH_ENABLED); + String val = prop.getProperty(LGSYNC_GROUP_SEARCH_ENABLED); if (val == null || val.trim().isEmpty()) { groupSearchEnabled = DEFAULT_LGSYNC_GROUP_SEARCH_ENABLED; } else { @@ -313,8 +204,7 @@ public boolean isGroupSearchEnabled() { } public String getGroupSearchBase() { - String val = prop.getProperty(LGSYNC_GROUP_SEARCH_BASE); - return val; + return prop.getProperty(LGSYNC_GROUP_SEARCH_BASE); } public int getGroupSearchScope() { @@ -334,8 +224,7 @@ public int getGroupSearchScope() { } public String getGroupObjectClass() { - String val = prop.getProperty(LGSYNC_GROUP_OBJECT_CLASS); - return val; + return prop.getProperty(LGSYNC_GROUP_OBJECT_CLASS); } public String getGroupSearchFilter() { @@ -343,43 +232,35 @@ public String getGroupSearchFilter() { } public String getUserGroupMemberAttributeName() { - String val = prop.getProperty(LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME); - return val; + return prop.getProperty(LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME); } public String getGroupNameAttribute() { - String val = prop.getProperty(LGSYNC_GROUP_NAME_ATTRIBUTE); - return val; + return prop.getProperty(LGSYNC_GROUP_NAME_ATTRIBUTE); } public String getAuthenticationMethod() { - String val = prop.getProperty(AUTHENTICATION_METHOD); - return val; + return prop.getProperty(AUTHENTICATION_METHOD); } public String getAdDomain() { - String val = prop.getProperty(AD_DOMAIN); - return val; + return prop.getProperty(AD_DOMAIN); } public String getUserDnPattern() { - String val = prop.getProperty(USER_DN_PATTERN); - return val; + return prop.getProperty(USER_DN_PATTERN); } public String getGroupRoleAttribute() { - String val = prop.getProperty(GROUP_ROLE_ATTRIBUTE); - return val; + return prop.getProperty(GROUP_ROLE_ATTRIBUTE); } public String getAuthGroupSearchBase() { - String val = prop.getProperty(GROUP_SEARCH_BASE); - return val; + return prop.getProperty(GROUP_SEARCH_BASE); } public String getAuthGroupSearchFilter() { - String val = prop.getProperty(GROUP_SEARCH_FILTER); - return val; + return prop.getProperty(GROUP_SEARCH_FILTER); } public String getAuthUsername() { @@ -390,14 +271,10 @@ public String getAuthPassword() { return prop.getProperty(AUTH_PASSWORD); } - public void updateInputPropFile(String ldapUrl, String bindDn, String bindPassword, - String userSearchBase, String userSearchFilter, - String authUser, String authPass) { + public void updateInputPropFile(String ldapUrl, String bindDn, String bindPassword, String userSearchBase, String userSearchFilter, String authUser, String authPass) { try { Parameters params = new Parameters(); - FileBasedConfigurationBuilder builder = - new FileBasedConfigurationBuilder(PropertiesConfiguration.class) - .configure(params.fileBased().setFileName(CONFIG_FILE)); + FileBasedConfigurationBuilder builder = new FileBasedConfigurationBuilder(PropertiesConfiguration.class).configure(params.fileBased().setFileName(CONFIG_FILE)); FileBasedConfiguration config = builder.getConfiguration(); // Update properties in memory and update the file as well prop.setProperty(LGSYNC_LDAP_URL, ldapUrl); @@ -409,19 +286,65 @@ public void updateInputPropFile(String ldapUrl, String bindDn, String bindPasswo prop.setProperty(AUTH_PASSWORD, authPass); config.setProperty(LGSYNC_LDAP_URL, ldapUrl); config.setProperty(LGSYNC_LDAP_BIND_DN, bindDn); - //config.setProperty(LGSYNC_LDAP_BIND_PASSWORD, bindPassword); config.setProperty(LGSYNC_USER_SEARCH_BASE, userSearchBase); config.setProperty(LGSYNC_USER_SEARCH_FILTER, userSearchFilter); config.setProperty(AUTH_USERNAME, authUser); - //config.setProperty(AUTH_PASSWORD, authPass); builder.save(); } catch (ConfigurationException e) { System.out.println("Failed to update " + CONFIG_FILE + ": " + e); } } -} + private void init(String configFile, String bindPasswd) { + readConfigFile(configFile); + prop.setProperty(LGSYNC_LDAP_BIND_PASSWORD, bindPasswd); + } + + private void readConfigFile(String fileName) { + try { + InputStream in = getFileInputStream(fileName); + if (in != null) { + try { + System.out.println("Reading ldap properties from " + fileName); + prop.load(in); + } finally { + try { + in.close(); + } catch (IOException ioe) { + System.out.println(ioe); + } + } + } + } catch (Throwable e) { + throw new RuntimeException("Unable to load configuration file [" + fileName + "]", e); + } + } + + private InputStream getFileInputStream(String path) throws FileNotFoundException { + InputStream ret = null; + File f = new File(path); + if (f.exists()) { + ret = new FileInputStream(f); + } else { + ret = getClass().getResourceAsStream(path); + if (ret == null) { + if (!path.startsWith("/")) { + ret = getClass().getResourceAsStream("/" + path); + } + } + if (ret == null) { + ret = ClassLoader.getSystemClassLoader().getResourceAsStream(path); + if (ret == null) { + if (!path.startsWith("/")) { + ret = ClassLoader.getSystemResourceAsStream("/" + path); + } + } + } + } + return ret; + } +} diff --git a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfigCheckMain.java b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfigCheckMain.java index 1e090cbb77..9cf69e9fa6 100644 --- a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfigCheckMain.java +++ b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfigCheckMain.java @@ -19,32 +19,34 @@ package org.apache.ranger.ldapconfigcheck; +import org.apache.commons.lang.NullArgumentException; + import javax.naming.Context; import javax.naming.NamingException; import javax.naming.ldap.Control; import javax.naming.ldap.InitialLdapContext; import javax.naming.ldap.LdapContext; import javax.naming.ldap.PagedResultsControl; + import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; import java.util.Properties; -import org.apache.commons.lang.NullArgumentException; - public class LdapConfigCheckMain { - - private static final String LOG_FILE = "ldapConfigCheck.log"; - private static final String AMBARI_PROPERTIES = "ambari.properties"; + private static final String LOG_FILE = "ldapConfigCheck.log"; + private static final String AMBARI_PROPERTIES = "ambari.properties"; private static final String INSTALL_PROPERTIES = "install.properties"; - public static void main(String[] args) { + private LdapConfigCheckMain() { + } + public static void main(String[] args) { CommandLineOptions cli = new CommandLineOptions(args); cli.parse(); String inFileName = cli.getInput(); - String outputDir = cli.getOutput(); + String outputDir = cli.getOutput(); if (!outputDir.endsWith("/")) { outputDir = outputDir.concat("/"); } @@ -55,14 +57,14 @@ public static void main(String[] args) { cli.getUserSearchBase(), cli.getUserSearchFilter(), cli.getAuthUser(), cli.getAuthPass()); } - PrintStream logFile = null; - PrintStream ambariProps = null; + PrintStream logFile = null; + PrintStream ambariProps = null; PrintStream installProps = null; - LdapContext ldapContext = null; + LdapContext ldapContext = null; try { - logFile = new PrintStream(new File(outputDir + LOG_FILE)); - ambariProps = new PrintStream(new File(outputDir + AMBARI_PROPERTIES)); + logFile = new PrintStream(new File(outputDir + LOG_FILE)); + ambariProps = new PrintStream(new File(outputDir + AMBARI_PROPERTIES)); installProps = new PrintStream(new File(outputDir + INSTALL_PROPERTIES)); UserSync userSyncObj = new UserSync(config, logFile, ambariProps, installProps); @@ -70,8 +72,7 @@ public static void main(String[] args) { String bindDn = config.getLdapBindDn(); Properties env = new Properties(); - env.put(Context.INITIAL_CONTEXT_FACTORY, - "com.sun.jndi.ldap.LdapCtxFactory"); + env.put(Context.INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory"); env.put(Context.PROVIDER_URL, config.getLdapUrl()); env.put(Context.SECURITY_PRINCIPAL, bindDn); env.put(Context.SECURITY_CREDENTIALS, cli.getBindPassword()); @@ -80,9 +81,8 @@ public static void main(String[] args) { ldapContext = new InitialLdapContext(env, null); - if (config.isPagedResultsEnabled()) { - ldapContext.setRequestControls(new Control[]{ - new PagedResultsControl(config.getPagedResultsSize(), Control.CRITICAL) }); + if (config.isPagedResultsEnabled()) { + ldapContext.setRequestControls(new Control[] {new PagedResultsControl(config.getPagedResultsSize(), Control.CRITICAL)}); } String retrieveValues = "all"; @@ -96,9 +96,8 @@ public static void main(String[] args) { } else { findAllUserSyncProperties(ldapContext, userSyncObj); } - }else if (cli.getRetrieveValues() != null){ + } else if (cli.getRetrieveValues() != null) { retrieveValues = cli.getRetrieveValues(); - } else { cli.help(); } @@ -112,14 +111,12 @@ public static void main(String[] args) { if (ldapContext != null) { ldapContext.close(); } - } catch (FileNotFoundException fe) { System.out.println(fe.getMessage()); } catch (IOException ioe) { logFile.println("ERROR: Failed while setting the paged results controls\n" + ioe); } catch (NamingException ne) { - System.out.println("ERROR: Failed to perfom ldap bind. Please verify values for " + - "ranger.usersync.ldap.binddn and ranger.usersync.ldap.ldapbindpassword\n" + ne); + System.out.println("ERROR: Failed to perfom ldap bind. Please verify values for ranger.usersync.ldap.binddn and ranger.usersync.ldap.ldapbindpassword\n" + ne); } catch (Throwable t) { if (logFile != null) { logFile.println("ERROR: Connection failed: " + t.getMessage()); @@ -137,24 +134,21 @@ public static void main(String[] args) { installProps.close(); } try { - if (ldapContext != null) { - ldapContext.close(); - } - } catch (NamingException ne){ - System.out.println("Failed to close LdapContext!"); + if (ldapContext != null) { + ldapContext.close(); + } + } catch (NamingException ne) { + System.out.println("Failed to close LdapContext!"); } } } private static void findAllUserSyncProperties(LdapContext ldapContext, UserSync userSyncObj) throws Throwable { - userSyncObj.findUserProperties(ldapContext); userSyncObj.findGroupProperties(ldapContext); } - private static void authenticate(UserSync userSyncObj, LdapConfig config, - PrintStream logFile, PrintStream ambariProps, - PrintStream installProps) throws Throwable{ + private static void authenticate(UserSync userSyncObj, LdapConfig config, PrintStream logFile, PrintStream ambariProps, PrintStream installProps) throws Throwable { AuthenticationCheck auth = new AuthenticationCheck(config.getLdapUrl(), userSyncObj, logFile, ambariProps, installProps); auth.discoverAuthProperties(); @@ -170,16 +164,14 @@ private static void authenticate(UserSync userSyncObj, LdapConfig config, throw new NullArgumentException(msg); } - if (auth.isAuthenticated(config.getLdapUrl(), config.getLdapBindDn(), config.getLdapBindPassword(), - config.getAuthUsername(), config.getAuthPassword())) { + if (auth.isAuthenticated(config.getLdapUrl(), config.getLdapBindDn(), config.getLdapBindPassword(), config.getAuthUsername(), config.getAuthPassword())) { logFile.println("INFO: Authentication verified successfully"); } else { logFile.println("ERROR: Failed to authenticate " + config.getAuthUsername()); } } - private static void retrieveUsersGroups(LdapContext ldapContext, UserSync userSyncObj, - String retrieve) throws Throwable { + private static void retrieveUsersGroups(LdapContext ldapContext, UserSync userSyncObj, String retrieve) throws Throwable { String msg; if (retrieve == null || userSyncObj == null || ldapContext == null) { msg = "Input validation failed while retrieving Users or Groups"; @@ -188,7 +180,7 @@ private static void retrieveUsersGroups(LdapContext ldapContext, UserSync userSy if (retrieve.equalsIgnoreCase("users")) { retrieveUsers(ldapContext, userSyncObj); - } else if (retrieve.equalsIgnoreCase("groups")){ + } else if (retrieve.equalsIgnoreCase("groups")) { retrieveGroups(ldapContext, userSyncObj); } else { // retrieve both @@ -207,10 +199,8 @@ private static void retrieveUsers(LdapContext ldapContext, UserSync userSyncObj) msg = "ranger.usersync.ldap.user.objectclass "; throw new NullArgumentException(msg); } - if ((userSyncObj.getUserSearchBase() == null || userSyncObj.getUserSearchBase().isEmpty()) && - (userSyncObj.getSearchBase() == null || userSyncObj.getSearchBase().isEmpty())) { - msg = "ranger.usersync.ldap.user.searchbase and " + - "ranger.usersync.ldap.searchBase "; + if ((userSyncObj.getUserSearchBase() == null || userSyncObj.getUserSearchBase().isEmpty()) && (userSyncObj.getSearchBase() == null || userSyncObj.getSearchBase().isEmpty())) { + msg = "ranger.usersync.ldap.user.searchbase and ranger.usersync.ldap.searchBase"; throw new NullArgumentException(msg); } userSyncObj.getAllUsers(ldapContext); @@ -230,15 +220,10 @@ private static void retrieveGroups(LdapContext ldapContext, UserSync userSyncObj msg = "ranger.usersync.group.memberattributename "; throw new NullArgumentException(msg); } - if ((userSyncObj.getGroupSearchBase() == null || userSyncObj.getGroupSearchBase().isEmpty()) && - (userSyncObj.getSearchBase() == null || userSyncObj.getSearchBase().isEmpty())) { - msg = "ranger.usersync.group.searchbase and " + - "ranger.usersync.ldap.searchBase "; + if ((userSyncObj.getGroupSearchBase() == null || userSyncObj.getGroupSearchBase().isEmpty()) && (userSyncObj.getSearchBase() == null || userSyncObj.getSearchBase().isEmpty())) { + msg = "ranger.usersync.group.searchbase and ranger.usersync.ldap.searchBase"; throw new NullArgumentException(msg); } userSyncObj.getAllGroups(ldapContext); } - - } - diff --git a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/UserSync.java b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/UserSync.java index cab40726fe..01b2f008f5 100644 --- a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/UserSync.java +++ b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/UserSync.java @@ -25,41 +25,63 @@ import javax.naming.directory.Attributes; import javax.naming.directory.SearchControls; import javax.naming.directory.SearchResult; -import javax.naming.ldap.LdapContext; import javax.naming.ldap.Control; -import javax.naming.ldap.PagedResultsResponseControl; +import javax.naming.ldap.LdapContext; import javax.naming.ldap.PagedResultsControl; +import javax.naming.ldap.PagedResultsResponseControl; + import java.io.PrintStream; import java.util.HashMap; import java.util.HashSet; import java.util.Set; public class UserSync { - private static String[] userNameAttrValues = { "sAMAccountName", "uid", "cn" }; - private static String[] userObjClassValues = { "person", "posixAccount" }; //Not needed as this is read from the second occurence of objectClass attribute from user entry - private static String[] userGroupMemAttrValues = { "memberOf", "ismemberOf"}; - - private static String[] groupObjectClassValues = { "group", "groupOfNames", "posixGroup" }; - private static String[] groupNameAttrValues = { "distinguishedName", "cn" }; - private static String[] groupMemAttrValues = { "member", "memberUid" }; - - private String userNameAttribute = null; - private String userObjClassName = null; - private String userGroupMemberName = null; - private String groupMemberName = null; - private String groupNameAttrName = null; - private String groupObjClassName = null; - private String groupSearchBase = null; - private String groupSearchFilter = null; - private String userSearchBase = null; - private String userSearchFilter = null; - private String searchBase = null; - private String groupName = null; - private PrintStream logFile = null; - private PrintStream ambariProps = null; - private PrintStream installProps = null; - - private LdapConfig config = null; + private static String[] userNameAttrValues = { + "sAMAccountName", "uid", "cn" + }; + + /* Not needed as this is read from the second occurrence of objectClass attribute from user entry */ + private static String[] userObjClassValues = { + "person", "posixAccount" + }; + private static String[] userGroupMemAttrValues = { + "memberOf", "ismemberOf" + }; + + private static String[] groupObjectClassValues = { + "group", "groupOfNames", "posixGroup" + }; + private static String[] groupNameAttrValues = { + "distinguishedName", "cn" + }; + private static String[] groupMemAttrValues = { + "member", "memberUid" + }; + + private String userNameAttribute; + private String userObjClassName; + private String userGroupMemberName; + private String groupMemberName; + private String groupNameAttrName; + private String groupObjClassName; + private String groupSearchBase; + private String groupSearchFilter; + private String userSearchBase; + private String userSearchFilter; + private String searchBase; + private String groupName; + private PrintStream logFile; + private PrintStream ambariProps; + private PrintStream installProps; + private LdapConfig config; + + public UserSync(LdapConfig config, PrintStream logFile, PrintStream ambariProps, PrintStream installProps) { + this.config = config; + this.logFile = logFile; + this.ambariProps = ambariProps; + this.installProps = installProps; + initUserSync(); + } public String getUserNameAttribute() { return userNameAttribute; @@ -85,35 +107,285 @@ public String getGroupObjClassName() { return groupObjClassName; } - public String getGroupSearchBase() { return groupSearchBase; } + public String getGroupSearchBase() { + return groupSearchBase; + } - public String getUserSearchBase() { return userSearchBase; } + public String getUserSearchBase() { + return userSearchBase; + } public String getSearchBase() { return searchBase; } - public UserSync(LdapConfig config, PrintStream logFile, PrintStream ambariProps, PrintStream installProps) { - this.config = config; - this.logFile = logFile; - this.ambariProps = ambariProps; - this.installProps = installProps; - initUserSync(); + public void findUserProperties(LdapContext ldapContext) throws Throwable { + // 1. find basic user properties + // 2. find user search base and user search filter by passing basic attributes + findBasicUserProperties(ldapContext, true); + + findAdvUserProperties(ldapContext, true); + } + + public void getAllUsers(LdapContext ldapContext) throws Throwable { + int noOfUsers = 0; + Attribute userNameAttr = null; + //String groupName = null; + Attribute groupMemberAttr = null; + NamingEnumeration userSearchResultEnum = null; + SearchControls userSearchControls = new SearchControls(); + userSearchControls.setSearchScope(config.getUserSearchScope()); + Set userSearchAttributes = new HashSet<>(); + if (userNameAttribute != null) { + userSearchAttributes.add(userNameAttribute); + } + if (userGroupMemberName != null) { + userSearchAttributes.add(userGroupMemberName); + } + + if (userSearchAttributes.size() > 0) { + userSearchControls.setReturningAttributes(userSearchAttributes.toArray( + new String[userSearchAttributes.size()])); + } else { + userSearchControls.setReturningAttributes(new java.lang.String[] {"*", "+"}); + } + + String extendedUserSearchFilter = "(objectclass=" + userObjClassName + ")"; + if (userSearchFilter != null && !userSearchFilter.trim().isEmpty()) { + String customFilter = userSearchFilter.trim(); + if (!customFilter.startsWith("(")) { + customFilter = "(" + customFilter + ")"; + } + extendedUserSearchFilter = "(&" + extendedUserSearchFilter + customFilter + ")"; + } + + byte[] cookie = null; + logFile.println(); + logFile.println("INFO: First 20 Users and associated groups are:"); + + try { + do { + userSearchResultEnum = ldapContext.search(userSearchBase, extendedUserSearchFilter, userSearchControls); + + while (userSearchResultEnum.hasMore()) { + final SearchResult userEntry = userSearchResultEnum.next(); + + if (userEntry == null) { + logFile.println("WARN: userEntry null"); + continue; + } + + Attributes attributes = userEntry.getAttributes(); + if (attributes == null) { + logFile.println("WARN: Attributes missing for entry " + userEntry.getNameInNamespace()); + continue; + } + + if (userNameAttribute == null || userNameAttribute.isEmpty()) { + for (int i = 0; i < userNameAttrValues.length; i++) { + userNameAttr = attributes.get(userNameAttrValues[i]); + if (userNameAttr != null) { + userNameAttribute = userNameAttrValues[i]; + break; + } + } + if (userNameAttr == null) { + logFile.print("WARN: Failed to find any of ( "); + for (int i = 0; i < userNameAttrValues.length; i++) { + logFile.print(userNameAttrValues[i] + " "); + } + logFile.println(") for entry " + userEntry.getNameInNamespace()); + continue; + } + } else { + userNameAttr = attributes.get(userNameAttribute); + if (userNameAttr == null) { + logFile.println("WARN: Failed to find " + userNameAttribute + " for entry " + userEntry.getNameInNamespace()); + continue; + } + } + + String userName = userNameAttr.get().toString(); + + if (userName == null || userName.trim().isEmpty()) { + logFile.println("WARN: " + userNameAttribute + " empty for entry " + userEntry.getNameInNamespace()); + continue; + } + userName = userName.toLowerCase(); + + Set groups = new HashSet<>(); + groupMemberAttr = attributes.get(userGroupMemberName); + + if (groupMemberAttr != null) { + NamingEnumeration groupEnum = groupMemberAttr.getAll(); + while (groupEnum.hasMore()) { + String groupRes = groupEnum.next().toString(); + groups.add(groupRes); + if (groupName == null || groupName.isEmpty()) { + groupName = groupRes; + } + } + } + + if (noOfUsers < 20) { + logFile.println("Username: " + userName + ", Groups: " + groups); + } + noOfUsers++; + } + // Examine the paged results control response + Control[] controls = ldapContext.getResponseControls(); + if (controls != null) { + for (Control control : controls) { + if (control instanceof PagedResultsResponseControl) { + PagedResultsResponseControl prrc = (PagedResultsResponseControl) control; + cookie = prrc.getCookie(); + } + } + } else { + logFile.println("WARN: No controls were sent from the server"); + } + // Re-activate paged results + if (config.isPagedResultsEnabled()) { + ldapContext.setRequestControls(new Control[] {new PagedResultsControl(config.getPagedResultsSize(), cookie, Control.CRITICAL)}); + } + } + while (cookie != null); + logFile.println("\nINFO: Total no. of users = " + noOfUsers); + } catch (NamingException ne) { + String msg = "Exception occured while retreiving users\n"; + if ((config.getUserNameAttribute() != null && !config.getUserNameAttribute().isEmpty()) || + (config.getUserObjectClass() != null && !config.getUserObjectClass().isEmpty()) || + (config.getGroupNameAttribute() != null && !config.getGroupNameAttribute().isEmpty()) || + (config.getUserSearchBase() != null && !config.getUserSearchBase().isEmpty()) || + (config.getUserSearchFilter() != null && !config.getUserSearchFilter().isEmpty())) { + throw new Exception("Please verify values for:\n ranger.usersync.ldap.user.nameattribute\n " + + "ranger.usersync.ldap.user.objectclass\n" + + "ranger.usersync.ldap.user.groupnameattribute\n" + + "ranger.usersync.ldap.user.searchbase\n" + + "ranger.usersync.ldap.user.searchfilter\n"); + } else { + throw new Exception(msg + ne); + } + } finally { + if (userSearchResultEnum != null) { + userSearchResultEnum.close(); + } + } + } + + public void findGroupProperties(LdapContext ldapContext) throws Throwable { + // find basic group attributes/properties + // find group search base and group search filter + // Get all groups + + if (groupName == null || groupName.isEmpty()) { + // Perform basic user search and get the group name from the user's group attribute name. + findBasicUserProperties(ldapContext, false); + } + + if (groupName == null || groupName.isEmpty()) { + // Perform adv user search and get the group name from the user's group attribute name. + findAdvUserProperties(ldapContext, false); + } + + findBasicGroupProperties(ldapContext); + findAdvGroupProperties(ldapContext); + } + + public void getAllGroups(LdapContext ldapContext) throws Throwable { + int noOfGroups = 0; + Attribute groupNameAttr; + Attribute groupMemberAttr; + NamingEnumeration groupSearchResultEnum = null; + SearchControls groupSearchControls = new SearchControls(); + groupSearchControls.setSearchScope(config.getGroupSearchScope()); + Set groupSearchAttributes = new HashSet<>(); + groupSearchAttributes.add(groupNameAttrName); + groupSearchAttributes.add(groupMemberName); + groupSearchAttributes.add("distinguishedName"); + groupSearchControls.setReturningAttributes(groupSearchAttributes.toArray( + new String[groupSearchAttributes.size()])); + + String extendedGroupSearchFilter = "(objectclass=" + groupObjClassName + ")"; + if (groupSearchFilter != null && !groupSearchFilter.trim().isEmpty()) { + String customFilter = groupSearchFilter.trim(); + if (!customFilter.startsWith("(")) { + customFilter = "(" + customFilter + ")"; + } + extendedGroupSearchFilter = "(&" + extendedGroupSearchFilter + customFilter + ")"; + } + + try { + groupSearchResultEnum = ldapContext.search(groupSearchBase, extendedGroupSearchFilter, + groupSearchControls); + + logFile.println("\nINFO: First 20 Groups and associated Users are:"); + + while (groupSearchResultEnum.hasMore()) { + final SearchResult groupEntry = groupSearchResultEnum.next(); + if (groupEntry == null) { + continue; + } + Attributes groupAttributes = groupEntry.getAttributes(); + if (groupAttributes == null) { + logFile.println("WARN: Attributes missing for entry " + groupEntry.getNameInNamespace()); + continue; + } + + groupMemberAttr = groupAttributes.get(groupMemberName); + + Set users = new HashSet<>(); + if (groupMemberAttr != null) { + NamingEnumeration userEnum = groupMemberAttr.getAll(); + while (userEnum.hasMore()) { + String userRes = userEnum.next().toString(); + users.add(userRes); + } + } + + groupNameAttr = groupAttributes.get(groupNameAttrName); + if (noOfGroups < 20) { + logFile.println("Group name: " + groupNameAttr.get().toString() + ", Users: " + users); + } + noOfGroups++; + } + + logFile.println("\nINFO: Total no. of groups = " + noOfGroups); + } catch (NamingException ne) { + String msg = "Exception occured while retreiving groups\n"; + if ((config.getGroupNameAttribute() != null && !config.getGroupNameAttribute().isEmpty()) || + (config.getGroupObjectClass() != null && !config.getGroupObjectClass().isEmpty()) || + (config.getUserGroupMemberAttributeName() != null && !config.getUserGroupMemberAttributeName().isEmpty()) || + (config.getGroupSearchBase() != null && !config.getGroupSearchBase().isEmpty()) || + (config.getGroupSearchFilter() != null && !config.getGroupSearchFilter().isEmpty())) { + throw new Exception("Please verify values for:\n ranger.usersync.group.memberattributename\n " + + "ranger.usersync.group.nameattribute\n" + + "ranger.usersync.group.objectclass\n" + + "ranger.usersync.group.searchbase\n" + + "ranger.usersync.group.searchfilter\n"); + } else { + throw new Exception(msg + ne); + } + } finally { + if (groupSearchResultEnum != null) { + groupSearchResultEnum.close(); + } + } } private void initUserSync() { try { String bindDn = config.getLdapBindDn(); - userObjClassName = config.getUserObjectClass(); - userNameAttribute = config.getUserNameAttribute(); + userObjClassName = config.getUserObjectClass(); + userNameAttribute = config.getUserNameAttribute(); userGroupMemberName = config.getUserGroupNameAttribute(); - userSearchBase = config.getUserSearchBase(); - userSearchFilter = config.getUserSearchFilter(); - groupObjClassName = config.getGroupObjectClass(); - groupNameAttrName = config.getGroupNameAttribute(); - groupMemberName = config.getUserGroupMemberAttributeName(); - groupSearchBase = config.getGroupSearchBase(); - groupSearchFilter = config.getGroupSearchFilter(); + userSearchBase = config.getUserSearchBase(); + userSearchFilter = config.getUserSearchFilter(); + groupObjClassName = config.getGroupObjectClass(); + groupNameAttrName = config.getGroupNameAttribute(); + groupMemberName = config.getUserGroupMemberAttributeName(); + groupSearchBase = config.getGroupSearchBase(); + groupSearchFilter = config.getGroupSearchFilter(); //String userName = null; if (bindDn.contains("@")) { @@ -131,26 +403,17 @@ private void initUserSync() { } } - public void findUserProperties(LdapContext ldapContext) throws Throwable { - // 1. find basic user properties - // 2. find user search base and user search filter by passing basic attributes - - findBasicUserProperties(ldapContext, true); - - findAdvUserProperties(ldapContext, true); - } - /* Use the provided bind dn or the user search base and user search filter for sample user and determine the basic user attribute. */ - private void findBasicUserProperties(LdapContext ldapContext, boolean isOutputNeeded) throws Throwable{ - String bindDn = config.getLdapBindDn(); - String userSFilter = config.getUserSearchFilter(); - String userSBase = config.getUserSearchBase(); - Attribute userNameAttr = null; - Attribute groupMemberAttr; + private void findBasicUserProperties(LdapContext ldapContext, boolean isOutputNeeded) throws Throwable { + String bindDn = config.getLdapBindDn(); + String userSFilter = config.getUserSearchFilter(); + String userSBase = config.getUserSearchBase(); + Attribute userNameAttr = null; + Attribute groupMemberAttr; SearchControls userSearchControls = new SearchControls(); userSearchControls.setSearchScope(config.getUserSearchScope()); - userSearchControls.setReturningAttributes(new java.lang.String[]{"*", "+"}); + userSearchControls.setReturningAttributes(new java.lang.String[] {"*", "+"}); int noOfUsers = 0; NamingEnumeration userSearchResultEnum = null; @@ -173,8 +436,7 @@ private void findBasicUserProperties(LdapContext ldapContext, boolean isOutputNe userSFilter = "userPrincipalName=" + bindDn; } else { int cnEndIndex = bindDn.indexOf(","); - userSFilter = bindDn.substring(0,cnEndIndex); - + userSFilter = bindDn.substring(0, cnEndIndex); } //System.out.println("Derived user search filter = " + userSearchFilter); } @@ -230,9 +492,9 @@ private void findBasicUserProperties(LdapContext ldapContext, boolean isOutputNe continue; } userName = userName.toLowerCase(); - Attribute userObjClassAttr = attributes.get("objectClass"); + Attribute userObjClassAttr = attributes.get("objectClass"); NamingEnumeration userObjClassEnum = userObjClassAttr.getAll(); - String userObjClass = null; + String userObjClass = null; while (userObjClassEnum.hasMore()) { userObjClass = userObjClassEnum.next().toString(); if (userObjClassName == null || userObjClassName.isEmpty()) { @@ -258,7 +520,7 @@ private void findBasicUserProperties(LdapContext ldapContext, boolean isOutputNe groupMemberAttr = attributes.get(userGroupMemAttrValues[i]); if (groupMemberAttr != null) { userGroupMemberName = userGroupMemAttrValues[i]; - groupName = groupMemberAttr.get(0).toString(); + groupName = groupMemberAttr.get(0).toString(); break; } } @@ -300,34 +562,30 @@ private void findBasicUserProperties(LdapContext ldapContext, boolean isOutputNe } } - private void findAdvUserProperties(LdapContext ldapContext, boolean isOutputNeeded) throws Throwable{ - int noOfUsers; + private void findAdvUserProperties(LdapContext ldapContext, boolean isOutputNeeded) throws Throwable { + int noOfUsers; NamingEnumeration userSearchResultEnum = null; - SearchControls userSearchControls = new SearchControls(); + SearchControls userSearchControls = new SearchControls(); userSearchControls.setSearchScope(config.getUserSearchScope()); if (userNameAttribute != null && !userNameAttribute.isEmpty()) { Set userSearchAttributes = new HashSet<>(); userSearchAttributes.add(userNameAttribute); userSearchAttributes.add(userGroupMemberName); userSearchAttributes.add("distinguishedName"); - userSearchControls.setReturningAttributes(userSearchAttributes.toArray( - new String[userSearchAttributes.size()])); + userSearchControls.setReturningAttributes(userSearchAttributes.toArray(new String[userSearchAttributes.size()])); } else { - userSearchControls.setReturningAttributes(new java.lang.String[]{"*", "+"}); + userSearchControls.setReturningAttributes(new java.lang.String[] {"*", "+"}); } String extendedUserSearchFilter = "(objectclass=" + userObjClassName + ")"; try { - HashMap ouOccurences = new HashMap<>(); if (userSearchBase == null || userSearchBase.isEmpty()) { - userSearchResultEnum = ldapContext.search(searchBase, - extendedUserSearchFilter, userSearchControls); + userSearchResultEnum = ldapContext.search(searchBase, extendedUserSearchFilter, userSearchControls); } else { - userSearchResultEnum = ldapContext.search(userSearchBase, - extendedUserSearchFilter, userSearchControls); + userSearchResultEnum = ldapContext.search(userSearchBase, extendedUserSearchFilter, userSearchControls); } noOfUsers = 0; @@ -353,26 +611,20 @@ private void findAdvUserProperties(LdapContext ldapContext, boolean isOutputNeed Attribute dnAttr = attributes.get("distinguishedName"); if (dnAttr != null) { dnValue = dnAttr.get().toString(); - String ouStr = "OU="; - int indexOfOU = dnValue.indexOf(ouStr); + String ouStr = "OU="; + int indexOfOU = dnValue.indexOf(ouStr); if (indexOfOU > 0) { dnValue = dnValue.substring(indexOfOU); - } else { dnValue = dnValue.substring(dnValue.indexOf(",") + 1); } - } else { - // If distinguishedName is not found, - // strip off the userName from the long name for OU or sub domain + // If distinguishedName is not found, strip off the userName from the long name for OU or sub domain dnValue = userEntry.getNameInNamespace(); dnValue = dnValue.substring(dnValue.indexOf(",") + 1); - } - //System.out.println("OU from dn = " + dnValue); Integer ouOccrs = ouOccurences.get(dnValue); if (ouOccrs == null) { - //System.out.println("value = 0"); ouOccrs = Integer.valueOf(0); } int val = ouOccrs.intValue(); @@ -382,20 +634,20 @@ private void findAdvUserProperties(LdapContext ldapContext, boolean isOutputNeed } if (!ouOccurences.isEmpty()) { - Set keys = ouOccurences.keySet(); - int maxOUOccr = 0; + Set keys = ouOccurences.keySet(); + int maxOUOccr = 0; for (String key : keys) { int ouOccurVal = ouOccurences.get(key).intValue(); logFile.println("INFO: No. of users from " + key + " = " + ouOccurVal); if (ouOccurVal > maxOUOccr) { - maxOUOccr = ouOccurVal; + maxOUOccr = ouOccurVal; userSearchBase = key; } } } if (userSearchFilter == null || userSearchFilter.isEmpty()) { - userSearchFilter = userNameAttribute + "=*"; + userSearchFilter = userNameAttribute + "=*"; } if (isOutputNeeded) { @@ -405,7 +657,6 @@ private void findAdvUserProperties(LdapContext ldapContext, boolean isOutputNeed ambariProps.println("ranger.usersync.ldap.user.searchbase=" + userSearchBase); ambariProps.println("ranger.usersync.ldap.user.searchfilter=" + userSearchFilter); } - } catch (NamingException ne) { String msg = "Exception occured while discovering user properties:\n" + "ranger.usersync.ldap.user.searchbase\n" + @@ -426,196 +677,26 @@ private void findAdvUserProperties(LdapContext ldapContext, boolean isOutputNeed } } - public void getAllUsers(LdapContext ldapContext) throws Throwable { - int noOfUsers = 0; - Attribute userNameAttr = null; - //String groupName = null; - Attribute groupMemberAttr = null; - NamingEnumeration userSearchResultEnum = null; - SearchControls userSearchControls = new SearchControls(); - userSearchControls.setSearchScope(config.getUserSearchScope()); - Set userSearchAttributes = new HashSet<>(); - if (userNameAttribute != null) { - userSearchAttributes.add(userNameAttribute); - } - if (userGroupMemberName != null) { - userSearchAttributes.add(userGroupMemberName); - } - - if (userSearchAttributes.size() > 0) { - userSearchControls.setReturningAttributes(userSearchAttributes.toArray( - new String[userSearchAttributes.size()])); - } else { - userSearchControls.setReturningAttributes(new java.lang.String[]{"*", "+"}); - } - - String extendedUserSearchFilter = "(objectclass=" + userObjClassName + ")"; - if (userSearchFilter != null && !userSearchFilter.trim().isEmpty()) { - String customFilter = userSearchFilter.trim(); - if (!customFilter.startsWith("(")) { - customFilter = "(" + customFilter + ")"; - } - extendedUserSearchFilter = "(&" + extendedUserSearchFilter + customFilter + ")"; - } - - byte[] cookie = null; - logFile.println(); - logFile.println("INFO: First 20 Users and associated groups are:"); - - try { - do { - - userSearchResultEnum = ldapContext.search(userSearchBase, - extendedUserSearchFilter, userSearchControls); - - while (userSearchResultEnum.hasMore()) { - final SearchResult userEntry = userSearchResultEnum.next(); - - if (userEntry == null) { - logFile.println("WARN: userEntry null"); - continue; - } - - Attributes attributes = userEntry.getAttributes(); - if (attributes == null) { - logFile.println("WARN: Attributes missing for entry " + userEntry.getNameInNamespace()); - continue; - } - - if (userNameAttribute == null || userNameAttribute.isEmpty()) { - for (int i = 0; i < userNameAttrValues.length; i++) { - userNameAttr = attributes.get(userNameAttrValues[i]); - if (userNameAttr != null) { - userNameAttribute = userNameAttrValues[i]; - break; - } - } - if (userNameAttr == null) { - logFile.print("WARN: Failed to find any of ( "); - for (int i = 0; i < userNameAttrValues.length; i++) { - logFile.print(userNameAttrValues[i] + " "); - } - logFile.println(") for entry " + userEntry.getNameInNamespace()); - continue; - } - } else { - userNameAttr = attributes.get(userNameAttribute); - if (userNameAttr == null) { - logFile.println("WARN: Failed to find " + userNameAttribute + " for entry " + userEntry.getNameInNamespace()); - continue; - } - } - - String userName = userNameAttr.get().toString(); - - if (userName == null || userName.trim().isEmpty()) { - logFile.println("WARN: " + userNameAttribute + " empty for entry " + userEntry.getNameInNamespace()); - continue; - } - userName = userName.toLowerCase(); - - Set groups = new HashSet<>(); - groupMemberAttr = attributes.get(userGroupMemberName); - - if (groupMemberAttr != null) { - NamingEnumeration groupEnum = groupMemberAttr.getAll(); - while (groupEnum.hasMore()) { - String groupRes = groupEnum.next().toString(); - groups.add(groupRes); - if (groupName == null || groupName.isEmpty()) { - groupName = groupRes; - } - } - } - - if (noOfUsers < 20) { - logFile.println("Username: " + userName + ", Groups: " + groups); - } - noOfUsers++; - } - // Examine the paged results control response - Control[] controls = ldapContext.getResponseControls(); - if (controls != null) { - for (int i = 0; i < controls.length; i++) { - if (controls[i] instanceof PagedResultsResponseControl) { - PagedResultsResponseControl prrc = - (PagedResultsResponseControl)controls[i]; - cookie = prrc.getCookie(); - } - } - } else { - logFile.println("WARN: No controls were sent from the server"); - } - // Re-activate paged results - if (config.isPagedResultsEnabled()) { - ldapContext.setRequestControls(new Control[]{ - new PagedResultsControl(config.getPagedResultsSize(), cookie, Control.CRITICAL)}); - } - } while (cookie != null); - logFile.println("\nINFO: Total no. of users = " + noOfUsers); - - } catch (NamingException ne) { - String msg = "Exception occured while retreiving users\n"; - if ((config.getUserNameAttribute() != null && !config.getUserNameAttribute().isEmpty()) || - (config.getUserObjectClass() != null && !config.getUserObjectClass().isEmpty()) || - (config.getGroupNameAttribute() != null && !config.getGroupNameAttribute().isEmpty()) || - (config.getUserSearchBase() != null && !config.getUserSearchBase().isEmpty()) || - (config.getUserSearchFilter() != null && !config.getUserSearchFilter().isEmpty())) { - throw new Exception("Please verify values for:\n ranger.usersync.ldap.user.nameattribute\n " + - "ranger.usersync.ldap.user.objectclass\n" + - "ranger.usersync.ldap.user.groupnameattribute\n" + - "ranger.usersync.ldap.user.searchbase\n" + - "ranger.usersync.ldap.user.searchfilter\n"); - } else { - throw new Exception(msg + ne); - } - } finally { - if (userSearchResultEnum != null) { - userSearchResultEnum.close(); - } - } - } - - public void findGroupProperties(LdapContext ldapContext) throws Throwable { - // find basic group attributes/properties - // find group search base and group search filter - // Get all groups - - if (groupName == null || groupName.isEmpty()) { - // Perform basic user search and get the group name from the user's group attribute name. - findBasicUserProperties(ldapContext, false); - } - - if (groupName == null || groupName.isEmpty()) { - // Perform adv user search and get the group name from the user's group attribute name. - findAdvUserProperties(ldapContext, false); - } - - findBasicGroupProperties(ldapContext); - - findAdvGroupProperties(ldapContext); - } - private void findBasicGroupProperties(LdapContext ldapContext) throws Throwable { - int noOfGroups; - Attribute groupNameAttr; - String groupBase; - String groupFilter; - Attribute groupMemberAttr; + int noOfGroups; + Attribute groupNameAttr; + String groupBase; + String groupFilter; + Attribute groupMemberAttr; NamingEnumeration groupSearchResultEnum = null; - SearchControls groupSearchControls = new SearchControls(); + SearchControls groupSearchControls = new SearchControls(); groupSearchControls.setSearchScope(config.getGroupSearchScope()); try { - if (groupName == null || groupName.isEmpty()) { - groupSearchResultEnum = ldapContext.search(searchBase, null); - } else { + if (groupName == null || groupName.isEmpty()) { + groupSearchResultEnum = ldapContext.search(searchBase, null); + } else { int baseIndex = groupName.indexOf(","); - groupBase = groupName.substring(baseIndex + 1); - groupFilter = groupName.substring(0, baseIndex); - groupSearchResultEnum = ldapContext.search(groupBase, groupFilter, - groupSearchControls); - } + groupBase = groupName.substring(baseIndex + 1); + groupFilter = groupName.substring(0, baseIndex); + groupSearchResultEnum = ldapContext.search(groupBase, groupFilter, + groupSearchControls); + } noOfGroups = 0; while (groupSearchResultEnum.hasMore()) { if (noOfGroups >= 1) { @@ -650,7 +731,6 @@ private void findBasicGroupProperties(LdapContext ldapContext) throws Throwable } if (groupNameAttrName == null || groupNameAttrName.isEmpty()) { - for (int i = 0; i < groupNameAttrValues.length; i++) { groupNameAttr = groupAttributes.get(groupNameAttrValues[i]); if (groupNameAttr != null) { @@ -679,9 +759,7 @@ private void findBasicGroupProperties(LdapContext ldapContext) throws Throwable ambariProps.println("ranger.usersync.group.memberattributename=" + groupMemberName); ambariProps.println("ranger.usersync.group.nameattribute=" + groupNameAttrName); ambariProps.println("ranger.usersync.group.objectclass=" + groupObjClassName); - } finally { - if (groupSearchResultEnum != null) { groupSearchResultEnum.close(); } @@ -689,26 +767,23 @@ private void findBasicGroupProperties(LdapContext ldapContext) throws Throwable } private void findAdvGroupProperties(LdapContext ldapContext) throws Throwable { - int noOfGroups = 0; + int noOfGroups = 0; NamingEnumeration groupSearchResultEnum = null; - SearchControls groupSearchControls = new SearchControls(); + SearchControls groupSearchControls = new SearchControls(); groupSearchControls.setSearchScope(config.getGroupSearchScope()); Set groupSearchAttributes = new HashSet<>(); groupSearchAttributes.add(groupNameAttrName); groupSearchAttributes.add(groupMemberName); groupSearchAttributes.add("distinguishedName"); - groupSearchControls.setReturningAttributes(groupSearchAttributes.toArray( - new String[groupSearchAttributes.size()])); + groupSearchControls.setReturningAttributes(groupSearchAttributes.toArray(new String[groupSearchAttributes.size()])); String extendedGroupSearchFilter = "(objectclass=" + groupObjClassName + ")"; try { HashMap ouOccurences = new HashMap<>(); if (groupSearchBase == null || groupSearchBase.isEmpty()) { - groupSearchResultEnum = ldapContext.search(searchBase, extendedGroupSearchFilter, - groupSearchControls); + groupSearchResultEnum = ldapContext.search(searchBase, extendedGroupSearchFilter, groupSearchControls); } else { - groupSearchResultEnum = ldapContext.search(groupSearchBase, extendedGroupSearchFilter, - groupSearchControls); + groupSearchResultEnum = ldapContext.search(groupSearchBase, extendedGroupSearchFilter, groupSearchControls); } while (groupSearchResultEnum.hasMore()) { @@ -731,25 +806,21 @@ private void findAdvGroupProperties(LdapContext ldapContext) throws Throwable { Attribute dnAttr = groupAttributes.get("distinguishedName"); if (dnAttr != null) { dnValue = dnAttr.get().toString(); - String ouStr = "OU="; - int indexOfOU = dnValue.indexOf(ouStr); + String ouStr = "OU="; + int indexOfOU = dnValue.indexOf(ouStr); if (indexOfOU > 0) { dnValue = dnValue.substring(indexOfOU); - } else { dnValue = dnValue.substring(dnValue.indexOf(",") + 1); } - } else { - // If distinguishedName is not found, - // strip off the userName from the long name for OU or sub domain + // If distinguishedName is not found, strip off the userName from the long name for OU or sub domain dnValue = groupEntry.getNameInNamespace(); dnValue = dnValue.substring(dnValue.indexOf(",") + 1); } - //System.out.println("OU from dn = " + dnValue); + Integer ouOccrs = ouOccurences.get(dnValue); if (ouOccrs == null) { - //System.out.println("value = 0"); ouOccrs = Integer.valueOf(0); } int val = ouOccrs.intValue(); @@ -760,20 +831,20 @@ private void findAdvGroupProperties(LdapContext ldapContext) throws Throwable { } if (!ouOccurences.isEmpty()) { - Set keys = ouOccurences.keySet(); - int maxOUOccr = 0; + Set keys = ouOccurences.keySet(); + int maxOUOccr = 0; for (String key : keys) { int ouOccurVal = ouOccurences.get(key).intValue(); logFile.println("INFO: No. of groups from " + key + " = " + ouOccurVal); if (ouOccurVal > maxOUOccr) { - maxOUOccr = ouOccurVal; + maxOUOccr = ouOccurVal; groupSearchBase = key; } } } if (groupSearchFilter == null || groupSearchFilter.isEmpty()) { - groupSearchFilter = groupNameAttrName + "=*"; + groupSearchFilter = groupNameAttrName + "=*"; } installProps.println("SYNC_GROUP_SEARCH_BASE=" + groupSearchBase); @@ -781,98 +852,10 @@ private void findAdvGroupProperties(LdapContext ldapContext) throws Throwable { ambariProps.println("ranger.usersync.group.searchbase=" + groupSearchBase); ambariProps.println("ranger.usersync.group.searchfilter=" + groupSearchFilter); - - } finally { - - if (groupSearchResultEnum != null) { - groupSearchResultEnum.close(); - } - } - } - - public void getAllGroups(LdapContext ldapContext) throws Throwable { - int noOfGroups = 0; - Attribute groupNameAttr; - Attribute groupMemberAttr; - NamingEnumeration groupSearchResultEnum = null; - SearchControls groupSearchControls = new SearchControls(); - groupSearchControls.setSearchScope(config.getGroupSearchScope()); - Set groupSearchAttributes = new HashSet<>(); - groupSearchAttributes.add(groupNameAttrName); - groupSearchAttributes.add(groupMemberName); - groupSearchAttributes.add("distinguishedName"); - groupSearchControls.setReturningAttributes(groupSearchAttributes.toArray( - new String[groupSearchAttributes.size()])); - - String extendedGroupSearchFilter= "(objectclass=" + groupObjClassName + ")"; - if (groupSearchFilter != null && !groupSearchFilter.trim().isEmpty()) { - String customFilter = groupSearchFilter.trim(); - if (!customFilter.startsWith("(")) { - customFilter = "(" + customFilter + ")"; - } - extendedGroupSearchFilter = "(&" + extendedGroupSearchFilter + customFilter + ")"; - } - - try { - - groupSearchResultEnum = ldapContext.search(groupSearchBase, extendedGroupSearchFilter, - groupSearchControls); - - logFile.println("\nINFO: First 20 Groups and associated Users are:"); - - while (groupSearchResultEnum.hasMore()) { - final SearchResult groupEntry = groupSearchResultEnum.next(); - if (groupEntry == null) { - continue; - } - Attributes groupAttributes = groupEntry.getAttributes(); - if (groupAttributes == null) { - logFile.println("WARN: Attributes missing for entry " + groupEntry.getNameInNamespace()); - continue; - } - - groupMemberAttr = groupAttributes.get(groupMemberName); - - Set users = new HashSet<>(); - if (groupMemberAttr != null) { - NamingEnumeration userEnum = groupMemberAttr.getAll(); - while (userEnum.hasMore()) { - String userRes = userEnum.next().toString(); - users.add(userRes); - } - } - - groupNameAttr = groupAttributes.get(groupNameAttrName); - if (noOfGroups < 20) { - logFile.println("Group name: " + groupNameAttr.get().toString() + ", Users: " + users); - } - noOfGroups++; - } - - logFile.println("\nINFO: Total no. of groups = " + noOfGroups); - - } catch (NamingException ne) { - String msg = "Exception occured while retreiving groups\n"; - if ((config.getGroupNameAttribute() != null && !config.getGroupNameAttribute().isEmpty()) || - (config.getGroupObjectClass() != null && !config.getGroupObjectClass().isEmpty()) || - (config.getUserGroupMemberAttributeName() != null && !config.getUserGroupMemberAttributeName().isEmpty()) || - (config.getGroupSearchBase() != null && !config.getGroupSearchBase().isEmpty()) || - (config.getGroupSearchFilter() != null && !config.getGroupSearchFilter().isEmpty())) { - throw new Exception("Please verify values for:\n ranger.usersync.group.memberattributename\n " + - "ranger.usersync.group.nameattribute\n" + - "ranger.usersync.group.objectclass\n" + - "ranger.usersync.group.searchbase\n" + - "ranger.usersync.group.searchfilter\n"); - } else { - throw new Exception(msg + ne); - } } finally { - if (groupSearchResultEnum != null) { groupSearchResultEnum.close(); } } } } - - diff --git a/ugsync/pom.xml b/ugsync/pom.xml index 824132d670..eaef820d5f 100644 --- a/ugsync/pom.xml +++ b/ugsync/pom.xml @@ -27,6 +27,9 @@ jar Unix User Group Synchronizer User, Group sync from sources such as Unix and LDAP + + true + ch.qos.logback diff --git a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/CustomSSLSocketFactory.java b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/CustomSSLSocketFactory.java index e564aa1d68..89035a3435 100644 --- a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/CustomSSLSocketFactory.java +++ b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/CustomSSLSocketFactory.java @@ -19,6 +19,18 @@ package org.apache.ranger.ldapusersync.process; +import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; +import org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; + import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -30,89 +42,70 @@ import java.security.KeyStore; import java.security.SecureRandom; -import javax.net.ssl.KeyManager; -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSocketFactory; -import javax.net.ssl.TrustManager; -import javax.net.ssl.TrustManagerFactory; - -import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; -import org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class CustomSSLSocketFactory extends SSLSocketFactory{ - private static final Logger LOG = LoggerFactory.getLogger(CustomSSLSocketFactory.class); - private SSLSocketFactory sockFactory; - private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); +public class CustomSSLSocketFactory extends SSLSocketFactory { + private static final Logger LOG = LoggerFactory.getLogger(CustomSSLSocketFactory.class); + private SSLSocketFactory sockFactory; + private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); public CustomSSLSocketFactory() { - SSLContext sslContext = null; - String keyStoreFile = config.getSSLKeyStorePath(); - String keyStoreFilepwd = config.getSSLKeyStorePathPassword(); - String trustStoreFile = config.getSSLTrustStorePath(); - String trustStoreFilepwd = config.getSSLTrustStorePathPassword(); - String keyStoreType = config.getSSLKeyStoreType(); - String trustStoreType = config.getSSLTrustStoreType(); - try { - - KeyManager[] kmList = null; - TrustManager[] tmList = null; - - if (keyStoreFile != null && keyStoreFilepwd != null) { - - KeyStore keyStore = KeyStore.getInstance(keyStoreType); - InputStream in = null; - try { - in = getFileInputStream(keyStoreFile); - if (in == null) { - LOG.error("Unable to obtain keystore from file [" + keyStoreFile + "]"); - return; - } - keyStore.load(in, keyStoreFilepwd.toCharArray()); - KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray()); - kmList = keyManagerFactory.getKeyManagers(); - } - finally { - if (in != null) { - in.close(); - } - } - - } - - if (trustStoreFile != null && trustStoreFilepwd != null) { - - KeyStore trustStore = KeyStore.getInstance(trustStoreType); - InputStream in = null; - try { - in = getFileInputStream(trustStoreFile); - if (in == null) { - LOG.error("Unable to obtain keystore from file [" + trustStoreFile + "]"); - return; - } - trustStore.load(in, trustStoreFilepwd.toCharArray()); - TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - trustManagerFactory.init(trustStore); - tmList = trustManagerFactory.getTrustManagers(); - } - finally { - if (in != null) { - in.close(); - } - } - } - - sslContext = SSLContext.getInstance("TLSv1.2"); - - sslContext.init(kmList, tmList, new SecureRandom()); - sockFactory = sslContext.getSocketFactory(); - } - catch(Throwable t) { - throw new RuntimeException("Unable to create SSLConext for communication to policy manager", t); - } + SSLContext sslContext = null; + String keyStoreFile = config.getSSLKeyStorePath(); + String keyStoreFilepwd = config.getSSLKeyStorePathPassword(); + String trustStoreFile = config.getSSLTrustStorePath(); + String trustStoreFilepwd = config.getSSLTrustStorePathPassword(); + String keyStoreType = config.getSSLKeyStoreType(); + String trustStoreType = config.getSSLTrustStoreType(); + try { + KeyManager[] kmList = null; + TrustManager[] tmList = null; + + if (keyStoreFile != null && keyStoreFilepwd != null) { + KeyStore keyStore = KeyStore.getInstance(keyStoreType); + InputStream in = null; + try { + in = getFileInputStream(keyStoreFile); + if (in == null) { + LOG.error("Unable to obtain keystore from file [{}]", keyStoreFile); + return; + } + keyStore.load(in, keyStoreFilepwd.toCharArray()); + KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray()); + kmList = keyManagerFactory.getKeyManagers(); + } finally { + if (in != null) { + in.close(); + } + } + } + + if (trustStoreFile != null && trustStoreFilepwd != null) { + KeyStore trustStore = KeyStore.getInstance(trustStoreType); + InputStream in = null; + try { + in = getFileInputStream(trustStoreFile); + if (in == null) { + LOG.error("Unable to obtain keystore from file [{}]", trustStoreFile); + return; + } + trustStore.load(in, trustStoreFilepwd.toCharArray()); + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(trustStore); + tmList = trustManagerFactory.getTrustManagers(); + } finally { + if (in != null) { + in.close(); + } + } + } + + sslContext = SSLContext.getInstance("TLSv1.2"); + + sslContext.init(kmList, tmList, new SecureRandom()); + sockFactory = sslContext.getSocketFactory(); + } catch (Throwable t) { + throw new RuntimeException("Unable to create SSLConext for communication to policy manager", t); + } } public static SSLSocketFactory getDefault() { @@ -155,32 +148,27 @@ public Socket createSocket(InetAddress address, int port, InetAddress localHost, } private InputStream getFileInputStream(String path) throws FileNotFoundException { + InputStream ret = null; + + File f = new File(path); + + if (f.exists()) { + ret = new FileInputStream(f); + } else { + ret = PolicyMgrUserGroupBuilder.class.getResourceAsStream(path); - InputStream ret = null; - - File f = new File(path); - - if (f.exists()) { - ret = new FileInputStream(f); - } else { - ret = PolicyMgrUserGroupBuilder.class.getResourceAsStream(path); - - if (ret == null) { - if (! path.startsWith("/")) { - ret = getClass().getResourceAsStream("/" + path); - } - } - - if (ret == null) { - ret = ClassLoader.getSystemClassLoader().getResourceAsStream(path); - if (ret == null) { - if (! path.startsWith("/")) { - ret = ClassLoader.getSystemResourceAsStream("/" + path); - } - } - } - } - - return ret; - } + if (ret == null && !path.startsWith("/")) { + ret = getClass().getResourceAsStream("/" + path); + } + + if (ret == null) { + ret = ClassLoader.getSystemClassLoader().getResourceAsStream(path); + if (ret == null && !path.startsWith("/")) { + ret = ClassLoader.getSystemResourceAsStream("/" + path); + } + } + } + + return ret; + } } diff --git a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java index cce3064721..17992d728b 100644 --- a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java @@ -19,20 +19,18 @@ package org.apache.ranger.ldapusersync.process; - -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Properties; -import java.util.Set; -import java.util.HashMap; -import java.util.UUID; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.thirdparty.com.google.common.collect.HashBasedTable; +import org.apache.hadoop.thirdparty.com.google.common.collect.Table; +import org.apache.ranger.ugsyncutil.model.LdapSyncSourceInfo; +import org.apache.ranger.ugsyncutil.model.UgsyncAuditInfo; +import org.apache.ranger.ugsyncutil.util.UgsyncCommonConstants; +import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; +import org.apache.ranger.usergroupsync.UserGroupSink; +import org.apache.ranger.usergroupsync.UserGroupSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.naming.Context; import javax.naming.InvalidNameException; @@ -52,1240 +50,1161 @@ import javax.naming.ldap.StartTlsRequest; import javax.naming.ldap.StartTlsResponse; -import org.apache.commons.collections.MapUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.thirdparty.com.google.common.collect.HashBasedTable; -import org.apache.hadoop.thirdparty.com.google.common.collect.Table; -import org.apache.ranger.ugsyncutil.util.UgsyncCommonConstants; -import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; -import org.apache.ranger.ugsyncutil.model.LdapSyncSourceInfo; -import org.apache.ranger.ugsyncutil.model.UgsyncAuditInfo; -import org.apache.ranger.usergroupsync.UserGroupSink; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.ranger.usergroupsync.UserGroupSource; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Properties; +import java.util.Set; +import java.util.UUID; public class LdapUserGroupBuilder implements UserGroupSource { - - private static final Logger LOG = LoggerFactory.getLogger(LdapUserGroupBuilder.class); - - private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); - - private static final String DATA_TYPE_BYTEARRAY = "byte[]"; - private static final String DATE_FORMAT = "yyyyMMddHHmmss"; - private static final int PAGE_SIZE = 500; - private static final String MEMBER_OF_ATTR = "memberof="; - private static final String GROUP_NAME_ATTRIBUTE = "cn="; - private static long deltaSyncUserTime = 0; // Used for AD uSNChanged - private static long deltaSyncGroupTime = 0; // Used for AD uSNChanged - private String deltaSyncUserTimeStamp; // Used for OpenLdap modifyTimestamp - private String deltaSyncGroupTimeStamp; // Used for OpenLdap modifyTimestamp - - private String ldapUrl; - private String ldapBindDn; - private String ldapBindPassword; - private String ldapAuthenticationMechanism; - private String ldapReferral; - private String searchBase; - - private String[] userSearchBase; - private String userNameAttribute; - private String userCloudIdAttribute; - private int userSearchScope; - private String userObjectClass; - private String userSearchFilter; - private Set groupNameSet; - private String extendedUserSearchFilter; - private SearchControls userSearchControls; - private Set userGroupNameAttributeSet; - private Set otherUserAttributes; - - private boolean pagedResultsEnabled = true; - private int pagedResultsSize = PAGE_SIZE; - - private boolean groupSearchFirstEnabled = true; - private boolean userSearchEnabled = true; - private boolean groupSearchEnabled = true; - private String[] groupSearchBase; - private int groupSearchScope; - private String groupObjectClass; - private String groupSearchFilter; - private String extendedGroupSearchFilter; - private String extendedAllGroupsSearchFilter; - private SearchControls groupSearchControls; - private String groupMemberAttributeName; - private String groupNameAttribute; - private String groupCloudIdAttribute; - private Set otherGroupAttributes; - private int groupHierarchyLevels; - private int deleteCycles; - private String currentSyncSource; - - private LdapContext ldapContext; - StartTlsResponse tls; - - private Table groupUserTable; - UgsyncAuditInfo ugsyncAuditInfo; - LdapSyncSourceInfo ldapSyncSourceInfo; - - private Map> sourceUsers; // key is user DN and value is map of user attributes containing original name, DN, etc... - private Map> sourceGroups; // key is group DN and value is map of group attributes containing original name, DN, etc... - private Map> sourceGroupUsers; // key is group DN and value is set of user DNs (members) - - public static void main(String[] args) throws Throwable { - LdapUserGroupBuilder ugBuilder = new LdapUserGroupBuilder(); - ugBuilder.init(); - } - - @Override - public void init() throws Throwable{ - deltaSyncUserTime = 0; - deltaSyncGroupTime = 0; - deleteCycles = 1; - DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); - deltaSyncUserTimeStamp = dateFormat.format(new Date(0)); - deltaSyncGroupTimeStamp = dateFormat.format(new Date(0)); - setConfig(); - ugsyncAuditInfo = new UgsyncAuditInfo(); - ldapSyncSourceInfo = new LdapSyncSourceInfo(); - ldapSyncSourceInfo.setLdapUrl(ldapUrl); - ldapSyncSourceInfo.setIncrementalSycn(Boolean.toString(config.isDeltaSyncEnabled())); - ldapSyncSourceInfo.setUserSearchEnabled(Boolean.toString(userSearchEnabled)); - ldapSyncSourceInfo.setGroupSearchEnabled(Boolean.toString(groupSearchEnabled)); - ldapSyncSourceInfo.setGroupSearchFirstEnabled(Boolean.toString(groupSearchFirstEnabled)); - ldapSyncSourceInfo.setGroupHierarchyLevel(Integer.toString(groupHierarchyLevels)); - ugsyncAuditInfo.setSyncSource(currentSyncSource); - ugsyncAuditInfo.setLdapSyncSourceInfo(ldapSyncSourceInfo); - } - - private void createLdapContext() throws Throwable { - Properties env = new Properties(); - env.put(Context.INITIAL_CONTEXT_FACTORY, - "com.sun.jndi.ldap.LdapCtxFactory"); - env.put(Context.PROVIDER_URL, ldapUrl); - if (ldapUrl.startsWith("ldaps") && (config.getSSLTrustStorePath() != null && !config.getSSLTrustStorePath().trim().isEmpty())) { - env.put("java.naming.ldap.factory.socket", "org.apache.ranger.ldapusersync.process.CustomSSLSocketFactory"); - } - - if (StringUtils.isNotEmpty(userCloudIdAttribute)) { - if (config.getUserCloudIdAttributeDataType().equals(DATA_TYPE_BYTEARRAY)) { - env.put("java.naming.ldap.attributes.binary", userCloudIdAttribute); - } - } - - if (StringUtils.isNotEmpty(groupCloudIdAttribute)) { - if (config.getGroupCloudIdAttributeDataType().equals(DATA_TYPE_BYTEARRAY)) { - env.put("java.naming.ldap.attributes.binary", groupCloudIdAttribute); - } - } - - for (String otherUserAttribute : otherUserAttributes) { - String attrType = config.getOtherUserAttributeDataType(otherUserAttribute); - if (attrType.equals(DATA_TYPE_BYTEARRAY)) { - env.put("java.naming.ldap.attributes.binary", otherUserAttribute); - } - } - - for (String otherGroupAttribute : otherGroupAttributes) { - String attrType = config.getOtherGroupAttributeDataType(otherGroupAttribute); - if (attrType.equals(DATA_TYPE_BYTEARRAY)) { - env.put("java.naming.ldap.attributes.binary", otherGroupAttribute); - } - } - ldapContext = new InitialLdapContext(env, null); - if (!ldapUrl.startsWith("ldaps")) { - if (config.isStartTlsEnabled()) { - tls = (StartTlsResponse) ldapContext.extendedOperation(new StartTlsRequest()); - if (config.getSSLTrustStorePath() != null && !config.getSSLTrustStorePath().trim().isEmpty()) { - tls.negotiate(CustomSSLSocketFactory.getDefault()); - } else { - tls.negotiate(); - } - LOG.info("Starting TLS session..."); - } - } - - ldapContext.addToEnvironment(Context.SECURITY_PRINCIPAL, ldapBindDn); - ldapContext.addToEnvironment(Context.SECURITY_CREDENTIALS, ldapBindPassword); - ldapContext.addToEnvironment(Context.SECURITY_AUTHENTICATION, ldapAuthenticationMechanism); - ldapContext.addToEnvironment(Context.REFERRAL, ldapReferral); - } - - private void setConfig() throws Throwable { - LOG.info("LdapUserGroupBuilder initialization started"); - - currentSyncSource = config.getCurrentSyncSource(); - groupSearchFirstEnabled = true; - userSearchEnabled = config.isUserSearchEnabled(); - groupSearchEnabled = config.isGroupSearchEnabled(); - ldapUrl = config.getLdapUrl(); - ldapBindDn = config.getLdapBindDn(); - ldapBindPassword = config.getLdapBindPassword(); - //ldapBindPassword = "admin-password"; - ldapAuthenticationMechanism = config.getLdapAuthenticationMechanism(); - ldapReferral = config.getContextReferral(); - searchBase = config.getSearchBase(); - - userSearchBase = config.getUserSearchBase().split(";"); - userSearchScope = config.getUserSearchScope(); - userObjectClass = config.getUserObjectClass(); - userSearchFilter = config.getUserSearchFilter(); - userNameAttribute = config.getUserNameAttribute(); - userCloudIdAttribute = config.getUserCloudIdAttribute(); - - Set userSearchAttributes = new HashSet(); - userSearchAttributes.add(userNameAttribute); - userGroupNameAttributeSet = config.getUserGroupNameAttributeSet(); - for (String useGroupNameAttribute : userGroupNameAttributeSet) { - userSearchAttributes.add(useGroupNameAttribute); - } - userSearchAttributes.add(userCloudIdAttribute); - otherUserAttributes = config.getOtherUserAttributes(); - for (String otherUserAttribute : otherUserAttributes) { - userSearchAttributes.add(otherUserAttribute); - } - userSearchAttributes.add("uSNChanged"); - userSearchAttributes.add("modifytimestamp"); - userSearchControls = new SearchControls(); - userSearchControls.setSearchScope(userSearchScope); - userSearchControls.setReturningAttributes(userSearchAttributes.toArray( - new String[userSearchAttributes.size()])); - - pagedResultsEnabled = config.isPagedResultsEnabled(); - pagedResultsSize = config.getPagedResultsSize(); - - groupSearchBase = config.getGroupSearchBase().split(";"); - groupSearchScope = config.getGroupSearchScope(); - groupObjectClass = config.getGroupObjectClass(); - groupSearchFilter = config.getGroupSearchFilter(); - groupMemberAttributeName = config.getUserGroupMemberAttributeName(); - groupNameAttribute = config.getGroupNameAttribute(); - groupCloudIdAttribute = config.getGroupCloudIdAttribute(); - groupHierarchyLevels = config.getGroupHierarchyLevels(); - - extendedGroupSearchFilter = "(&" + extendedGroupSearchFilter + "(|(" + groupMemberAttributeName + "={0})(" + groupMemberAttributeName + "={1})))"; - - groupSearchControls = new SearchControls(); - groupSearchControls.setSearchScope(groupSearchScope); - - Set groupSearchAttributes = new HashSet(); - groupSearchAttributes.add(groupNameAttribute); - groupSearchAttributes.add(groupCloudIdAttribute); - groupSearchAttributes.add(groupMemberAttributeName); - groupSearchAttributes.add("uSNChanged"); - groupSearchAttributes.add("modifytimestamp"); - otherGroupAttributes = config.getOtherGroupAttributes(); - for (String otherGroupAttribute : otherGroupAttributes) { - groupSearchAttributes.add(otherGroupAttribute); - } - groupSearchControls.setReturningAttributes(groupSearchAttributes.toArray( - new String[groupSearchAttributes.size()])); - - if (StringUtils.isEmpty(userSearchFilter)) { - groupNameSet = config.getGroupNameSet(); - String computedSearchFilter = ""; - for (String groupName : groupNameSet) { - if (LOG.isDebugEnabled()) { - LOG.debug("groupName = " + groupName); - } - if (!groupName.startsWith(MEMBER_OF_ATTR) && !groupName.startsWith(GROUP_NAME_ATTRIBUTE)){ - LOG.info("Ignoring unsupported format for " + groupName ); - continue; - } - String searchFilter = groupName; - if (groupName.startsWith(MEMBER_OF_ATTR)) { - searchFilter = groupName.substring(MEMBER_OF_ATTR.length()); - } - searchFilter = getFirstRDN(searchFilter); - computedSearchFilter += getDNForMemberOf(searchFilter); - } - if (StringUtils.isNotEmpty(computedSearchFilter)) { - computedSearchFilter = "(|" + computedSearchFilter + ")"; - } - LOG.info("Final computedSearchFilter = " + computedSearchFilter); - userSearchFilter = computedSearchFilter; - } - - LOG.info("LdapUserGroupBuilder initialization completed with -- " - + "ldapUrl: " + ldapUrl - + ", ldapBindDn: " + ldapBindDn - + ", ldapBindPassword: ***** " - + ", ldapAuthenticationMechanism: " + ldapAuthenticationMechanism - + ", searchBase: " + searchBase - + ", userSearchBase: " + Arrays.toString(userSearchBase) - + ", userSearchScope: " + userSearchScope - + ", userObjectClass: " + userObjectClass - + ", userSearchFilter: " + userSearchFilter - + ", extendedUserSearchFilter: " + extendedUserSearchFilter - + ", userNameAttribute: " + userNameAttribute - + ", userSearchAttributes: " + userSearchAttributes - + ", userGroupNameAttributeSet: " + userGroupNameAttributeSet - + ", otherUserAttributes: " + otherUserAttributes - + ", pagedResultsEnabled: " + pagedResultsEnabled - + ", pagedResultsSize: " + pagedResultsSize - + ", groupSearchEnabled: " + groupSearchEnabled - + ", groupSearchBase: " + Arrays.toString(groupSearchBase) - + ", groupSearchScope: " + groupSearchScope - + ", groupObjectClass: " + groupObjectClass - + ", groupSearchFilter: " + groupSearchFilter - + ", extendedGroupSearchFilter: " + extendedGroupSearchFilter - + ", extendedAllGroupsSearchFilter: " + extendedAllGroupsSearchFilter - + ", groupMemberAttributeName: " + groupMemberAttributeName - + ", groupNameAttribute: " + groupNameAttribute - + ", groupSearchAttributes: " + groupSearchAttributes - + ", groupSearchFirstEnabled: " + groupSearchFirstEnabled - + ", userSearchEnabled: " + userSearchEnabled - + ", ldapReferral: " + ldapReferral - ); - } - - private void closeLdapContext() throws Throwable { - if (tls != null) { - tls.close(); - } - if (ldapContext != null) { - ldapContext.close(); - } - } - - @Override - public boolean isChanged() { - // we do not want to get the full ldap dit and check whether anything has changed - return true; - } - - @Override - public void updateSink(UserGroupSink sink) throws Throwable { - LOG.info("LdapUserGroupBuilder updateSink started"); + private static final Logger LOG = LoggerFactory.getLogger(LdapUserGroupBuilder.class); + private static final String DATA_TYPE_BYTEARRAY = "byte[]"; + private static final String DATE_FORMAT = "yyyyMMddHHmmss"; + private static final String MEMBER_OF_ATTR = "memberof="; + private static final String GROUP_NAME_ATTRIBUTE = "cn="; + private static final int PAGE_SIZE = 500; + /* for AD uSNChanged */ + private static long deltaSyncUserTime; + private static long deltaSyncGroupTime; + /* ***************** */ + private boolean pagedResultsEnabled = true; + private boolean groupSearchFirstEnabled = true; + private boolean userSearchEnabled = true; + private boolean groupSearchEnabled = true; + private int pagedResultsSize = PAGE_SIZE; + private int groupHierarchyLevels; + private int deleteCycles; + private int userSearchScope; + private int groupSearchScope; + + private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); + /* for OpenLdap modifyTimestamp */ + private String deltaSyncUserTimeStamp; + private String deltaSyncGroupTimeStamp; + /* ******************************** */ + private String ldapUrl; + private String ldapBindDn; + private String ldapBindPassword; + private String ldapAuthenticationMechanism; + private String ldapReferral; + private String searchBase; + private String userNameAttribute; + private String userCloudIdAttribute; + private String userObjectClass; + private String userSearchFilter; + private String extendedUserSearchFilter; + private String groupObjectClass; + private String groupSearchFilter; + private String extendedGroupSearchFilter; + private String extendedAllGroupsSearchFilter; + private String groupMemberAttributeName; + private String groupNameAttribute; + private String groupCloudIdAttribute; + private String currentSyncSource; + private String[] userSearchBase; + private String[] groupSearchBase; + private Set groupNameSet; + private Set userGroupNameAttributeSet; + private Set otherUserAttributes; + private Set otherGroupAttributes; + private LdapContext ldapContext; + private SearchControls userSearchControls; + private SearchControls groupSearchControls; + private Table groupUserTable; + /* { key = user DN, value = map of user attributes {original name, DN, etc.}} */ + private Map> sourceUsers; + /* { key = group DN, value = map of group attributes {original name, DN, etc.}} */ + private Map> sourceGroups; + /* { key = group DN, value = set of user DNs (members) } */ + private Map> sourceGroupUsers; + + StartTlsResponse tls; + UgsyncAuditInfo ugsyncAuditInfo; + LdapSyncSourceInfo ldapSyncSourceInfo; + + public static void main(String[] args) throws Throwable { + LdapUserGroupBuilder ugBuilder = new LdapUserGroupBuilder(); + ugBuilder.init(); + } + + @Override + public void init() throws Throwable { + deltaSyncUserTime = 0; + deltaSyncGroupTime = 0; + deleteCycles = 1; + DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); + deltaSyncUserTimeStamp = dateFormat.format(new Date(0)); + deltaSyncGroupTimeStamp = dateFormat.format(new Date(0)); + setConfig(); + ugsyncAuditInfo = new UgsyncAuditInfo(); + ldapSyncSourceInfo = new LdapSyncSourceInfo(); + ldapSyncSourceInfo.setLdapUrl(ldapUrl); + ldapSyncSourceInfo.setIncrementalSycn(Boolean.toString(config.isDeltaSyncEnabled())); + ldapSyncSourceInfo.setUserSearchEnabled(Boolean.toString(userSearchEnabled)); + ldapSyncSourceInfo.setGroupSearchEnabled(Boolean.toString(groupSearchEnabled)); + ldapSyncSourceInfo.setGroupSearchFirstEnabled(Boolean.toString(groupSearchFirstEnabled)); + ldapSyncSourceInfo.setGroupHierarchyLevel(Integer.toString(groupHierarchyLevels)); + ugsyncAuditInfo.setSyncSource(currentSyncSource); + ugsyncAuditInfo.setLdapSyncSourceInfo(ldapSyncSourceInfo); + } + + @Override + public boolean isChanged() { + // we do not want to get the full ldap dit and check whether anything has changed + return true; + } + + @Override + public void updateSink(UserGroupSink sink) throws Throwable { + LOG.info("LdapUserGroupBuilder updateSink started"); boolean computeDeletes = false; - groupUserTable = HashBasedTable.create(); - sourceGroups = new HashMap<>(); - sourceUsers = new HashMap<>(); - sourceGroupUsers = new HashMap<>(); - long highestdeltaSyncUserTime = 0; - long highestdeltaSyncGroupTime = 0; - - if (config.isUserSyncDeletesEnabled() && deleteCycles >= config.getUserSyncDeletesFrequency()) { - deleteCycles = 1; - computeDeletes = true; - if (LOG.isDebugEnabled()) { - LOG.debug("Compute deleted users/groups is enabled for this sync cycle"); - } - } - if (config.isUserSyncDeletesEnabled()) { - deleteCycles++; - } - if (groupSearchEnabled) { - highestdeltaSyncGroupTime = getGroups(computeDeletes); - } - if (userSearchEnabled) { - LOG.info("Performing user search to retrieve users from AD/LDAP"); - highestdeltaSyncUserTime = getUsers(computeDeletes); - } - - if (groupHierarchyLevels > 0) { - LOG.info("Going through group hierarchy for nested group evaluation"); + groupUserTable = HashBasedTable.create(); + sourceGroups = new HashMap<>(); + sourceUsers = new HashMap<>(); + sourceGroupUsers = new HashMap<>(); + long highestdeltaSyncUserTime = 0; + long highestdeltaSyncGroupTime = 0; + + if (config.isUserSyncDeletesEnabled() && deleteCycles >= config.getUserSyncDeletesFrequency()) { + deleteCycles = 1; + computeDeletes = true; + LOG.debug("Compute deleted users/groups is enabled for this sync cycle"); + } + if (config.isUserSyncDeletesEnabled()) { + deleteCycles++; + } + if (groupSearchEnabled) { + highestdeltaSyncGroupTime = getGroups(computeDeletes); + } + if (userSearchEnabled) { + LOG.info("Performing user search to retrieve users from AD/LDAP"); + highestdeltaSyncUserTime = getUsers(computeDeletes); + } + + if (groupHierarchyLevels > 0) { + LOG.info("Going through group hierarchy for nested group evaluation"); Set groupFullNames = sourceGroups.keySet(); - for(String group : groupFullNames) { - Set nextLevelGroups = groupUserTable.column(group).keySet(); - goUpGroupHierarchy(nextLevelGroups, groupHierarchyLevels-1, group); - } - LOG.info("Completed group hierarchy computation"); - } - - Iterator groupUserTableIterator = groupUserTable.rowKeySet().iterator(); - while (groupUserTableIterator.hasNext()) { - String groupName = groupUserTableIterator.next(); - Map groupUsersMap = groupUserTable.row(groupName); - Set userSet = new HashSet(); - for(Map.Entry entry : groupUsersMap.entrySet()){ - if (sourceUsers.containsKey(entry.getValue())) { - userSet.add(entry.getValue()); - } - } - sourceGroupUsers.put(groupName, userSet); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Users = " + sourceUsers.keySet()); - LOG.debug("Groups = " + sourceGroups.keySet()); - LOG.debug("GroupUsers = " + sourceGroupUsers.keySet()); - } - - try { - sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers, sourceGroupUsers, computeDeletes); - DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); - LOG.info("deltaSyncUserTime = " + deltaSyncUserTime + " and highestdeltaSyncUserTime = " + highestdeltaSyncUserTime); - if (deltaSyncUserTime < highestdeltaSyncUserTime) { - // Incrementing highestdeltaSyncUserTime (for AD) in order to avoid search record repetition for next sync cycle. - deltaSyncUserTime = highestdeltaSyncUserTime + 1; - // Incrementing the highest timestamp value (for Openldap) with 1sec in order to avoid search record repetition for next sync cycle. - deltaSyncUserTimeStamp = dateFormat.format(new Date(highestdeltaSyncUserTime + 60l)); - } - - LOG.info("deltaSyncGroupTime = " + deltaSyncGroupTime + " and highestdeltaSyncGroupTime = " + highestdeltaSyncGroupTime); - // Update deltaSyncUserTime/deltaSyncUserTimeStamp here so that in case of failures, we get updates in next cycle - if (deltaSyncGroupTime < highestdeltaSyncGroupTime) { - // Incrementing highestdeltaSyncGroupTime (for AD) in order to avoid search record repetition for next sync cycle. - deltaSyncGroupTime = highestdeltaSyncGroupTime+1; - // Incrementing the highest timestamp value (for OpenLdap) with 1min in order to avoid search record repetition for next sync cycle. - deltaSyncGroupTimeStamp = dateFormat.format(new Date(highestdeltaSyncGroupTime + 60l)); - } - } catch (Throwable t) { - LOG.error("Failed to update ranger admin. Will retry in next sync cycle!!", t); - } - - ldapSyncSourceInfo.setUserSearchFilter(extendedUserSearchFilter); - ldapSyncSourceInfo.setGroupSearchFilter(extendedAllGroupsSearchFilter); - - try { - sink.postUserGroupAuditInfo(ugsyncAuditInfo); - } catch (Throwable t) { - LOG.error("sink.postUserGroupAuditInfo failed with exception: " + t.getMessage()); - } - } - - private long getUsers(boolean computeDeletes) throws Throwable { - NamingEnumeration userSearchResultEnum = null; - NamingEnumeration groupSearchResultEnum = null; - long highestdeltaSyncUserTime; - try { - createLdapContext(); - int total; - // Activate paged results - if (pagedResultsEnabled) { - ldapContext.setRequestControls(new Control[]{ - new PagedResultsControl(pagedResultsSize, Control.NONCRITICAL) }); - } - DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); - if (groupUserTable.rowKeySet().size() != 0 || !config.isDeltaSyncEnabled() || (computeDeletes)) { - // Fix RANGER-1957: Perform full sync when there are updates to the groups or when incremental sync is not enabled - deltaSyncUserTime = 0; - deltaSyncUserTimeStamp = dateFormat.format(new Date(0)); - } - - extendedUserSearchFilter = "(objectclass=" + userObjectClass + ")(|(uSNChanged>=" + deltaSyncUserTime + ")(modifyTimestamp>=" + deltaSyncUserTimeStamp + "Z))"; - - if (userSearchFilter != null && !userSearchFilter.trim().isEmpty()) { - String customFilter = userSearchFilter.trim(); - if (!customFilter.startsWith("(")) { - customFilter = "(" + customFilter + ")"; - } - - extendedUserSearchFilter = "(&" + extendedUserSearchFilter + customFilter + ")"; - } else { - extendedUserSearchFilter = "(&" + extendedUserSearchFilter + ")"; - } - LOG.info("extendedUserSearchFilter = " + extendedUserSearchFilter); - - highestdeltaSyncUserTime = deltaSyncUserTime; - - // When multiple OUs are configured, go through each OU as the user search base to search for users. - for (int ou=0; ou groupAttrMap = new HashMap<>(); - String groupName = getShortName(groupDN); - groupAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, groupName); - groupAttrMap.put(UgsyncCommonConstants.FULL_NAME, groupDN); - groupAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); - groupAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl()); - sourceGroups.put(groupDN, groupAttrMap); - if (LOG.isDebugEnabled()) { - LOG.debug("As groupsearch is disabled, adding group " + groupName + " from user memberof attribute for user " + userName); - } - groupUserTable.put(groupDN, userFullName, userFullName); - } - } - } - } - - Map userAttrMap = new HashMap<>(); - userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName); - userAttrMap.put(UgsyncCommonConstants.FULL_NAME, userFullName); - userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); - userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl()); - Attribute userCloudIdAttr = attributes.get(userCloudIdAttribute); - if (userCloudIdAttr != null) { - addToAttrMap(userAttrMap, "cloud_id", userCloudIdAttr, config.getUserCloudIdAttributeDataType()); - } - for (String otherUserAttribute : otherUserAttributes) { - if (attributes.get(otherUserAttribute) != null) { - String attrType = config.getOtherUserAttributeDataType(otherUserAttribute); - addToAttrMap(userAttrMap, otherUserAttribute, attributes.get(otherUserAttribute), attrType); - } - } - - sourceUsers.put(userFullName, userAttrMap); - if ((groupUserTable.containsColumn(userFullName) || groupUserTable.containsColumn(userName))) { - //Update the username in the groupUserTable with the one from username attribute. - Map userMap = groupUserTable.column(userFullName); - if (MapUtils.isEmpty(userMap)) { - userMap = groupUserTable.column(userName); - } - for (Map.Entry entry : userMap.entrySet()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Updating groupUserTable " + entry.getValue() + " with: " + userName + " for " + entry.getKey()); - } - groupUserTable.put(entry.getKey(), userFullName, userFullName); - } - } - counter++; - - if (counter <= 2000) { - LOG.info("Updating user count: " + counter + ", userName: " + userName); - if ( counter == 2000 ) { - LOG.info("===> 2000 user records have been synchronized so far. From now on, only a summary progress log will be written for every 100 users. To continue to see detailed log for every user, please enable Trace level logging. <==="); + for (String group : groupFullNames) { + Set nextLevelGroups = groupUserTable.column(group).keySet(); + goUpGroupHierarchy(nextLevelGroups, groupHierarchyLevels - 1, group); + } + LOG.info("Completed group hierarchy computation"); + } + + Iterator groupUserTableIterator = groupUserTable.rowKeySet().iterator(); + while (groupUserTableIterator.hasNext()) { + String groupName = groupUserTableIterator.next(); + Map groupUsersMap = groupUserTable.row(groupName); + Set userSet = new HashSet(); + for (Map.Entry entry : groupUsersMap.entrySet()) { + if (sourceUsers.containsKey(entry.getValue())) { + userSet.add(entry.getValue()); + } + } + sourceGroupUsers.put(groupName, userSet); + } + + LOG.debug("Users = {}", sourceUsers.keySet()); + LOG.debug("Groups = {}", sourceGroups.keySet()); + LOG.debug("GroupUsers = {}", sourceGroupUsers.keySet()); + + try { + sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers, sourceGroupUsers, computeDeletes); + DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); + LOG.info("deltaSyncUserTime = {} and highestdeltaSyncUserTime = {}", deltaSyncUserTime, highestdeltaSyncUserTime); + if (deltaSyncUserTime < highestdeltaSyncUserTime) { + // Incrementing highestdeltaSyncUserTime (for AD) in order to avoid search record repetition for next sync cycle. + deltaSyncUserTime = highestdeltaSyncUserTime + 1; + // Incrementing the highest timestamp value (for Openldap) with 1sec in order to avoid search record repetition for next sync cycle. + deltaSyncUserTimeStamp = dateFormat.format(new Date(highestdeltaSyncUserTime + 60L)); + } + + LOG.info("deltaSyncGroupTime = {} and highestdeltaSyncGroupTime = {} ", deltaSyncGroupTime, highestdeltaSyncGroupTime); + // Update deltaSyncUserTime/deltaSyncUserTimeStamp here so that in case of failures, we get updates in next cycle + if (deltaSyncGroupTime < highestdeltaSyncGroupTime) { + // Incrementing highestdeltaSyncGroupTime (for AD) in order to avoid search record repetition for next sync cycle. + deltaSyncGroupTime = highestdeltaSyncGroupTime + 1; + // Incrementing the highest timestamp value (for OpenLdap) with 1min in order to avoid search record repetition for next sync cycle. + deltaSyncGroupTimeStamp = dateFormat.format(new Date(highestdeltaSyncGroupTime + 60L)); + } + } catch (Throwable t) { + LOG.error("Failed to update ranger admin. Will retry in next sync cycle!!", t); + } + + ldapSyncSourceInfo.setUserSearchFilter(extendedUserSearchFilter); + ldapSyncSourceInfo.setGroupSearchFilter(extendedAllGroupsSearchFilter); + + try { + sink.postUserGroupAuditInfo(ugsyncAuditInfo); + } catch (Throwable t) { + LOG.error("sink.postUserGroupAuditInfo failed with exception: " + t.getMessage()); + } + } + + private void createLdapContext() throws Throwable { + Properties env = new Properties(); + env.put(Context.INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory"); + env.put(Context.PROVIDER_URL, ldapUrl); + if (ldapUrl.startsWith("ldaps") && (config.getSSLTrustStorePath() != null && !config.getSSLTrustStorePath().trim().isEmpty())) { + env.put("java.naming.ldap.factory.socket", "org.apache.ranger.ldapusersync.process.CustomSSLSocketFactory"); + } + + if (StringUtils.isNotEmpty(userCloudIdAttribute)) { + if (config.getUserCloudIdAttributeDataType().equals(DATA_TYPE_BYTEARRAY)) { + env.put("java.naming.ldap.attributes.binary", userCloudIdAttribute); + } + } + + if (StringUtils.isNotEmpty(groupCloudIdAttribute)) { + if (config.getGroupCloudIdAttributeDataType().equals(DATA_TYPE_BYTEARRAY)) { + env.put("java.naming.ldap.attributes.binary", groupCloudIdAttribute); + } + } + + for (String otherUserAttribute : otherUserAttributes) { + String attrType = config.getOtherUserAttributeDataType(otherUserAttribute); + if (attrType.equals(DATA_TYPE_BYTEARRAY)) { + env.put("java.naming.ldap.attributes.binary", otherUserAttribute); + } + } + + for (String otherGroupAttribute : otherGroupAttributes) { + String attrType = config.getOtherGroupAttributeDataType(otherGroupAttribute); + if (attrType.equals(DATA_TYPE_BYTEARRAY)) { + env.put("java.naming.ldap.attributes.binary", otherGroupAttribute); + } + } + ldapContext = new InitialLdapContext(env, null); + if (!ldapUrl.startsWith("ldaps")) { + if (config.isStartTlsEnabled()) { + tls = (StartTlsResponse) ldapContext.extendedOperation(new StartTlsRequest()); + if (config.getSSLTrustStorePath() != null && !config.getSSLTrustStorePath().trim().isEmpty()) { + tls.negotiate(CustomSSLSocketFactory.getDefault()); + } else { + tls.negotiate(); + } + LOG.info("Starting TLS session..."); + } + } + + ldapContext.addToEnvironment(Context.SECURITY_PRINCIPAL, ldapBindDn); + ldapContext.addToEnvironment(Context.SECURITY_CREDENTIALS, ldapBindPassword); + ldapContext.addToEnvironment(Context.SECURITY_AUTHENTICATION, ldapAuthenticationMechanism); + ldapContext.addToEnvironment(Context.REFERRAL, ldapReferral); + } + + private void setConfig() throws Throwable { + LOG.info("LdapUserGroupBuilder initialization started"); + groupSearchFirstEnabled = true; + currentSyncSource = config.getCurrentSyncSource(); + userSearchEnabled = config.isUserSearchEnabled(); + groupSearchEnabled = config.isGroupSearchEnabled(); + ldapUrl = config.getLdapUrl(); + ldapBindDn = config.getLdapBindDn(); + ldapBindPassword = config.getLdapBindPassword(); + ldapAuthenticationMechanism = config.getLdapAuthenticationMechanism(); + ldapReferral = config.getContextReferral(); + searchBase = config.getSearchBase(); + userSearchBase = config.getUserSearchBase().split(";"); + userSearchScope = config.getUserSearchScope(); + userObjectClass = config.getUserObjectClass(); + userSearchFilter = config.getUserSearchFilter(); + userNameAttribute = config.getUserNameAttribute(); + userCloudIdAttribute = config.getUserCloudIdAttribute(); + + Set userSearchAttributes = new HashSet(); + userSearchAttributes.add(userNameAttribute); + userGroupNameAttributeSet = config.getUserGroupNameAttributeSet(); + for (String useGroupNameAttribute : userGroupNameAttributeSet) { + userSearchAttributes.add(useGroupNameAttribute); + } + userSearchAttributes.add(userCloudIdAttribute); + otherUserAttributes = config.getOtherUserAttributes(); + for (String otherUserAttribute : otherUserAttributes) { + userSearchAttributes.add(otherUserAttribute); + } + userSearchAttributes.add("uSNChanged"); + userSearchAttributes.add("modifytimestamp"); + userSearchControls = new SearchControls(); + userSearchControls.setSearchScope(userSearchScope); + userSearchControls.setReturningAttributes(userSearchAttributes.toArray(new String[userSearchAttributes.size()])); + + pagedResultsEnabled = config.isPagedResultsEnabled(); + pagedResultsSize = config.getPagedResultsSize(); + + groupSearchBase = config.getGroupSearchBase().split(";"); + groupSearchScope = config.getGroupSearchScope(); + groupObjectClass = config.getGroupObjectClass(); + groupSearchFilter = config.getGroupSearchFilter(); + groupMemberAttributeName = config.getUserGroupMemberAttributeName(); + groupNameAttribute = config.getGroupNameAttribute(); + groupCloudIdAttribute = config.getGroupCloudIdAttribute(); + groupHierarchyLevels = config.getGroupHierarchyLevels(); + + extendedGroupSearchFilter = "(&" + extendedGroupSearchFilter + "(|(" + groupMemberAttributeName + "={0})(" + groupMemberAttributeName + "={1})))"; + + groupSearchControls = new SearchControls(); + groupSearchControls.setSearchScope(groupSearchScope); + + Set groupSearchAttributes = new HashSet(); + groupSearchAttributes.add(groupNameAttribute); + groupSearchAttributes.add(groupCloudIdAttribute); + groupSearchAttributes.add(groupMemberAttributeName); + groupSearchAttributes.add("uSNChanged"); + groupSearchAttributes.add("modifytimestamp"); + otherGroupAttributes = config.getOtherGroupAttributes(); + for (String otherGroupAttribute : otherGroupAttributes) { + groupSearchAttributes.add(otherGroupAttribute); + } + groupSearchControls.setReturningAttributes(groupSearchAttributes.toArray(new String[groupSearchAttributes.size()])); + + if (StringUtils.isEmpty(userSearchFilter)) { + groupNameSet = config.getGroupNameSet(); + String computedSearchFilter = ""; + for (String groupName : groupNameSet) { + if (LOG.isDebugEnabled()) { + LOG.debug("groupName = {}", groupName); + } + if (!groupName.startsWith(MEMBER_OF_ATTR) && !groupName.startsWith(GROUP_NAME_ATTRIBUTE)) { + LOG.info("Ignoring unsupported format for {}", groupName); + continue; + } + String searchFilter = groupName; + if (groupName.startsWith(MEMBER_OF_ATTR)) { + searchFilter = groupName.substring(MEMBER_OF_ATTR.length()); + } + searchFilter = getFirstRDN(searchFilter); + computedSearchFilter += getDNForMemberOf(searchFilter); + } + if (StringUtils.isNotEmpty(computedSearchFilter)) { + computedSearchFilter = "(|" + computedSearchFilter + ")"; + } + LOG.info("Final computedSearchFilter = {}", computedSearchFilter); + userSearchFilter = computedSearchFilter; + } + + LOG.info("LdapUserGroupBuilder initialization completed with -- " + + "ldapUrl: " + ldapUrl + + ", ldapBindDn: " + ldapBindDn + + ", ldapBindPassword: ***** " + + ", ldapAuthenticationMechanism: " + ldapAuthenticationMechanism + + ", searchBase: " + searchBase + + ", userSearchBase: " + Arrays.toString(userSearchBase) + + ", userSearchScope: " + userSearchScope + + ", userObjectClass: " + userObjectClass + + ", userSearchFilter: " + userSearchFilter + + ", extendedUserSearchFilter: " + extendedUserSearchFilter + + ", userNameAttribute: " + userNameAttribute + + ", userSearchAttributes: " + userSearchAttributes + + ", userGroupNameAttributeSet: " + userGroupNameAttributeSet + + ", otherUserAttributes: " + otherUserAttributes + + ", pagedResultsEnabled: " + pagedResultsEnabled + + ", pagedResultsSize: " + pagedResultsSize + + ", groupSearchEnabled: " + groupSearchEnabled + + ", groupSearchBase: " + Arrays.toString(groupSearchBase) + + ", groupSearchScope: " + groupSearchScope + + ", groupObjectClass: " + groupObjectClass + + ", groupSearchFilter: " + groupSearchFilter + + ", extendedGroupSearchFilter: " + extendedGroupSearchFilter + + ", extendedAllGroupsSearchFilter: " + extendedAllGroupsSearchFilter + + ", groupMemberAttributeName: " + groupMemberAttributeName + + ", groupNameAttribute: " + groupNameAttribute + + ", groupSearchAttributes: " + groupSearchAttributes + + ", groupSearchFirstEnabled: " + groupSearchFirstEnabled + + ", userSearchEnabled: " + userSearchEnabled + + ", ldapReferral: " + ldapReferral); + } + + private void closeLdapContext() throws Throwable { + if (tls != null) { + tls.close(); + } + if (ldapContext != null) { + ldapContext.close(); + } + } + + private long getUsers(boolean computeDeletes) throws Throwable { + NamingEnumeration userSearchResultEnum = null; + NamingEnumeration groupSearchResultEnum = null; + long highestdeltaSyncUserTime; + try { + createLdapContext(); + int total; + // Activate paged results + if (pagedResultsEnabled) { + ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, Control.NONCRITICAL)}); + } + DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); + if (groupUserTable.rowKeySet().size() != 0 || !config.isDeltaSyncEnabled() || (computeDeletes)) { + // Fix RANGER-1957: Perform full sync when there are updates to the groups or when incremental sync is not enabled + deltaSyncUserTime = 0; + deltaSyncUserTimeStamp = dateFormat.format(new Date(0)); + } + + extendedUserSearchFilter = "(objectclass=" + userObjectClass + ")(|(uSNChanged>=" + deltaSyncUserTime + ")(modifyTimestamp>=" + deltaSyncUserTimeStamp + "Z))"; + + if (userSearchFilter != null && !userSearchFilter.trim().isEmpty()) { + String customFilter = userSearchFilter.trim(); + if (!customFilter.startsWith("(")) { + customFilter = "(" + customFilter + ")"; + } + + extendedUserSearchFilter = "(&" + extendedUserSearchFilter + customFilter + ")"; + } else { + extendedUserSearchFilter = "(&" + extendedUserSearchFilter + ")"; + } + LOG.info("extendedUserSearchFilter = {}", extendedUserSearchFilter); + + highestdeltaSyncUserTime = deltaSyncUserTime; + + // When multiple OUs are configured, go through each OU as the user search base to search for users. + for (String s : userSearchBase) { + byte[] cookie = null; + int counter = 0; + try { + int paged = 0; + do { + userSearchResultEnum = ldapContext.search(s, extendedUserSearchFilter, userSearchControls); + + while (userSearchResultEnum.hasMore()) { + // searchResults contains all the user entries + final SearchResult userEntry = userSearchResultEnum.next(); + + if (userEntry == null) { + LOG.info("userEntry null, skipping sync for the entry"); + continue; } - } else { - if (LOG.isTraceEnabled()) { - LOG.trace("Updating user count: " + counter - + ", userName: " + userName); - } else { - if ( counter % 100 == 0) { - LOG.info("Synced " + counter + " users till now"); + + Attributes attributes = userEntry.getAttributes(); + if (attributes == null) { + LOG.info("attributes missing for entry {}, skipping sync", userEntry.getNameInNamespace()); + continue; + } + + Attribute userNameAttr = attributes.get(userNameAttribute); + if (userNameAttr == null) { + LOG.info("{} missing for entry {}, skipping sync", userNameAttribute, userEntry.getNameInNamespace()); + continue; + } + + String userFullName = userEntry.getNameInNamespace(); + String userName = (String) userNameAttr.get(); + + if (userName == null || userName.trim().isEmpty()) { + LOG.info("{} empty for entry {}, skipping sync", userNameAttribute, userEntry.getNameInNamespace()); + continue; + } + + Attribute timeStampAttr = attributes.get("uSNChanged"); + if (timeStampAttr != null) { + String uSNChangedVal = (String) timeStampAttr.get(); + long currentDeltaSyncTime = Long.parseLong(uSNChangedVal); + LOG.info("uSNChangedVal = {} and currentDeltaSyncTime = {}", uSNChangedVal, currentDeltaSyncTime); + if (currentDeltaSyncTime > highestdeltaSyncUserTime) { + highestdeltaSyncUserTime = currentDeltaSyncTime; + } + } else { + timeStampAttr = attributes.get("modifytimestamp"); + if (timeStampAttr != null) { + String timeStampVal = (String) timeStampAttr.get(); + Date parseDate = dateFormat.parse(timeStampVal); + long currentDeltaSyncTime = parseDate.getTime(); + LOG.info("timeStampVal = {} and currentDeltaSyncTime = {}", timeStampVal, currentDeltaSyncTime); + if (currentDeltaSyncTime > highestdeltaSyncUserTime) { + highestdeltaSyncUserTime = currentDeltaSyncTime; + deltaSyncUserTimeStamp = timeStampVal; + } + } + } + + // Get all the groups from the group name attribute of the user only when group search is not enabled. + if (!groupSearchEnabled) { + for (String useGroupNameAttribute : userGroupNameAttributeSet) { + Attribute userGroupfAttribute = userEntry.getAttributes().get(useGroupNameAttribute); + if (userGroupfAttribute != null) { + NamingEnumeration groupEnum = userGroupfAttribute.getAll(); + while (groupEnum.hasMore()) { + String groupDN = (String) groupEnum.next(); + if (LOG.isDebugEnabled()) { + LOG.debug("Adding {} to {}", groupDN, userName); + } + Map groupAttrMap = new HashMap<>(); + String groupName = getShortName(groupDN); + groupAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, groupName); + groupAttrMap.put(UgsyncCommonConstants.FULL_NAME, groupDN); + groupAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); + groupAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl()); + sourceGroups.put(groupDN, groupAttrMap); + if (LOG.isDebugEnabled()) { + LOG.debug("As groupsearch is disabled, adding group {} from user memberof attribute for user {}", groupName, userName); + } + groupUserTable.put(groupDN, userFullName, userFullName); + } + } + } + } + + Map userAttrMap = new HashMap<>(); + userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName); + userAttrMap.put(UgsyncCommonConstants.FULL_NAME, userFullName); + userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); + userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl()); + Attribute userCloudIdAttr = attributes.get(userCloudIdAttribute); + if (userCloudIdAttr != null) { + addToAttrMap(userAttrMap, "cloud_id", userCloudIdAttr, config.getUserCloudIdAttributeDataType()); + } + for (String otherUserAttribute : otherUserAttributes) { + if (attributes.get(otherUserAttribute) != null) { + String attrType = config.getOtherUserAttributeDataType(otherUserAttribute); + addToAttrMap(userAttrMap, otherUserAttribute, attributes.get(otherUserAttribute), attrType); + } + } + + sourceUsers.put(userFullName, userAttrMap); + if ((groupUserTable.containsColumn(userFullName) || groupUserTable.containsColumn(userName))) { + Map userMap = groupUserTable.column(userFullName); + if (MapUtils.isEmpty(userMap)) { + userMap = groupUserTable.column(userName); + } + for (Map.Entry entry : userMap.entrySet()) { + LOG.debug("Updating groupUserTable {} with: {} for {}", entry.getValue(), userName, entry.getKey()); + groupUserTable.put(entry.getKey(), userFullName, userFullName); + } + } + counter++; + + if (counter <= 2000) { + LOG.info("Updating user count: " + counter + ", userName: " + userName); + if (counter == 2000) { + LOG.info("===> 2000 user records have been synchronized so far. From now on, only a summary progress log will be written for every 100 users. To continue to see detailed log for every user, please enable Trace level logging. <==="); + } + } else { + if (LOG.isTraceEnabled()) { + LOG.trace("Updating user count: {}, userName: {}", counter, userName); + } else if (counter % 100 == 0) { + LOG.info("Synced {} users till now", counter); } } } - } - - // Examine the paged results control response - Control[] controls = ldapContext.getResponseControls(); - if (controls != null) { - for (int i = 0; i < controls.length; i++) { - if (controls[i] instanceof PagedResultsResponseControl) { - PagedResultsResponseControl prrc = - (PagedResultsResponseControl)controls[i]; - total = prrc.getResultSize(); - if (total != 0) { - if (LOG.isDebugEnabled()) { - LOG.debug("END-OF-PAGE total : " + total); - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("END-OF-PAGE total : unknown"); - } - } - cookie = prrc.getCookie(); - } - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("No controls were sent from the server"); - } - } - // Re-activate paged results - if (pagedResultsEnabled) { - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("Fetched paged results round: %s", ++paged)); - } - ldapContext.setRequestControls(new Control[]{ - new PagedResultsControl(pagedResultsSize, cookie, Control.CRITICAL) }); - } - } while (cookie != null); - LOG.info("LdapUserGroupBuilder.getUsers() completed with user count: " - + counter); - } catch (Exception t) { - LOG.error("LdapUserGroupBuilder.getUsers() failed with exception: ", t); - LOG.info("LdapUserGroupBuilder.getUsers() user count: " - + counter); - } - } - } finally { - if (userSearchResultEnum != null) { - userSearchResultEnum.close(); - } - if (groupSearchResultEnum != null) { - groupSearchResultEnum.close(); - } - closeLdapContext(); - } - if (LOG.isDebugEnabled()) { - LOG.debug("highestdeltaSyncUserTime = " + highestdeltaSyncUserTime); - } - return highestdeltaSyncUserTime; - } - - private long getGroups(boolean computeDeletes) throws Throwable { - NamingEnumeration groupSearchResultEnum = null; - DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); - long highestdeltaSyncGroupTime = deltaSyncGroupTime; - try { - createLdapContext(); - int total; - // Activate paged results - if (pagedResultsEnabled) { - ldapContext.setRequestControls(new Control[]{ - new PagedResultsControl(pagedResultsSize, Control.NONCRITICAL) }); - } - extendedGroupSearchFilter = "(objectclass=" + groupObjectClass + ")"; - if (groupSearchFilter != null && !groupSearchFilter.trim().isEmpty()) { - String customFilter = groupSearchFilter.trim(); - if (!customFilter.startsWith("(")) { - customFilter = "(" + customFilter + ")"; - } - extendedGroupSearchFilter = extendedGroupSearchFilter + customFilter; - } - - if (!config.isDeltaSyncEnabled() || (computeDeletes)) { - // Perform full sync when incremental sync is not enabled - deltaSyncGroupTime = 0; - deltaSyncGroupTimeStamp = dateFormat.format(new Date(0)); - } - - extendedAllGroupsSearchFilter = "(&" + extendedGroupSearchFilter + "(|(uSNChanged>=" + deltaSyncGroupTime + ")(modifyTimestamp>=" + deltaSyncGroupTimeStamp + "Z)))"; - - LOG.info("extendedAllGroupsSearchFilter = " + extendedAllGroupsSearchFilter); - for (int ou=0; ou groupAttrMap = new HashMap<>(); + // Examine the paged results control response + Control[] controls = ldapContext.getResponseControls(); + if (controls != null) { + for (int i = 0; i < controls.length; i++) { + if (controls[i] instanceof PagedResultsResponseControl) { + PagedResultsResponseControl prrc = + (PagedResultsResponseControl) controls[i]; + total = prrc.getResultSize(); + if (total != 0) { + LOG.debug("END-OF-PAGE total : " + total); + } else { + LOG.debug("END-OF-PAGE total : unknown"); + } + cookie = prrc.getCookie(); + } + } + } else { + LOG.debug("No controls were sent from the server"); + } + // Re-activate paged results + if (pagedResultsEnabled) { + LOG.debug(String.format("Fetched paged results round: %s", ++paged)); + ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, cookie, Control.CRITICAL)}); + } + } + while (cookie != null); + LOG.info("LdapUserGroupBuilder.getUsers() completed with user count: {}", counter); + } catch (Exception t) { + LOG.error("LdapUserGroupBuilder.getUsers() failed with exception: ", t); + LOG.info("LdapUserGroupBuilder.getUsers() user count: {}", counter); + } + } + } finally { + if (userSearchResultEnum != null) { + userSearchResultEnum.close(); + } + if (groupSearchResultEnum != null) { + groupSearchResultEnum.close(); + } + closeLdapContext(); + } + + LOG.debug("highestdeltaSyncUserTime = {}", highestdeltaSyncUserTime); + return highestdeltaSyncUserTime; + } + + private long getGroups(boolean computeDeletes) throws Throwable { + NamingEnumeration groupSearchResultEnum = null; + DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); + long highestdeltaSyncGroupTime = deltaSyncGroupTime; + try { + createLdapContext(); + int total; + // Activate paged results + if (pagedResultsEnabled) { + ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, Control.NONCRITICAL)}); + } + extendedGroupSearchFilter = "(objectclass=" + groupObjectClass + ")"; + if (groupSearchFilter != null && !groupSearchFilter.trim().isEmpty()) { + String customFilter = groupSearchFilter.trim(); + if (!customFilter.startsWith("(")) { + customFilter = "(" + customFilter + ")"; + } + extendedGroupSearchFilter = extendedGroupSearchFilter + customFilter; + } + + if (!config.isDeltaSyncEnabled() || (computeDeletes)) { + // Perform full sync when incremental sync is not enabled + deltaSyncGroupTime = 0; + deltaSyncGroupTimeStamp = dateFormat.format(new Date(0)); + } + + extendedAllGroupsSearchFilter = "(&" + extendedGroupSearchFilter + "(|(uSNChanged>=" + deltaSyncGroupTime + ")(modifyTimestamp>=" + deltaSyncGroupTimeStamp + "Z)))"; + + LOG.info("extendedAllGroupsSearchFilter = {}", extendedAllGroupsSearchFilter); + for (String s : groupSearchBase) { + byte[] cookie = null; + int counter = 0; + try { + int paged = 0; + do { + groupSearchResultEnum = ldapContext.search(s, extendedAllGroupsSearchFilter, groupSearchControls); + while (groupSearchResultEnum.hasMore()) { + final SearchResult groupEntry = groupSearchResultEnum.next(); + if (groupEntry == null) { + LOG.info("groupEntry null, skipping sync for the entry"); + continue; + } + counter++; + Attributes attributes = groupEntry.getAttributes(); + Attribute groupNameAttr = attributes.get(groupNameAttribute); + if (groupNameAttr == null) { + LOG.info("{} empty for entry {}, skipping sync", groupNameAttribute, groupEntry.getNameInNamespace()); + continue; + } + String groupFullName = (groupEntry.getNameInNamespace()); + String gName = (String) groupNameAttr.get(); + Map groupAttrMap = new HashMap<>(); groupAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, gName); groupAttrMap.put(UgsyncCommonConstants.FULL_NAME, groupFullName); groupAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); groupAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl()); - Attribute groupCloudIdAttr = attributes.get(groupCloudIdAttribute); - if (groupCloudIdAttr != null) { - addToAttrMap(groupAttrMap, "cloud_id", groupCloudIdAttr, config.getGroupCloudIdAttributeDataType()); - } - for (String otherGroupAttribute : otherGroupAttributes) { - if (attributes.get(otherGroupAttribute) != null) { - String attrType = config.getOtherGroupAttributeDataType(otherGroupAttribute); - addToAttrMap(groupAttrMap, otherGroupAttribute, attributes.get(otherGroupAttribute), attrType); - } - } - sourceGroups.put(groupFullName, groupAttrMap); - - Attribute timeStampAttr = attributes.get("uSNChanged"); - if (timeStampAttr != null) { - String uSNChangedVal = (String) timeStampAttr.get(); - long currentDeltaSyncTime = Long.parseLong(uSNChangedVal); - if (currentDeltaSyncTime > highestdeltaSyncGroupTime) { - highestdeltaSyncGroupTime = currentDeltaSyncTime; - } - } else { - timeStampAttr = attributes.get("modifytimestamp"); - if (timeStampAttr != null) { - String timeStampVal = (String) timeStampAttr.get(); - Date parseDate = dateFormat.parse(timeStampVal); - long currentDeltaSyncTime = parseDate.getTime(); - LOG.info("timeStampVal = " + timeStampVal + "and currentDeltaSyncTime = " + currentDeltaSyncTime); - if (currentDeltaSyncTime > highestdeltaSyncGroupTime) { - highestdeltaSyncGroupTime = currentDeltaSyncTime; - deltaSyncGroupTimeStamp = timeStampVal; - } - } - } - Attribute groupMemberAttr = attributes.get(groupMemberAttributeName); - int userCount = 0; - if (groupMemberAttr == null || groupMemberAttr.size() <= 0) { - LOG.info("No members available for " + gName); - sourceGroupUsers.put(groupFullName, new HashSet<>()); - continue; - } - - NamingEnumeration userEnum = groupMemberAttr.getAll(); - while (userEnum.hasMore()) { - String originalUserFullName = (String) userEnum.next(); - if (originalUserFullName == null || originalUserFullName.trim().isEmpty()) { - sourceGroupUsers.put(groupFullName, new HashSet<>()); - continue; - } - userCount++; - - if (!userSearchEnabled) { - Map userAttrMap = new HashMap<>(); - String userName = getShortName(originalUserFullName); + Attribute groupCloudIdAttr = attributes.get(groupCloudIdAttribute); + if (groupCloudIdAttr != null) { + addToAttrMap(groupAttrMap, "cloud_id", groupCloudIdAttr, config.getGroupCloudIdAttributeDataType()); + } + for (String otherGroupAttribute : otherGroupAttributes) { + if (attributes.get(otherGroupAttribute) != null) { + String attrType = config.getOtherGroupAttributeDataType(otherGroupAttribute); + addToAttrMap(groupAttrMap, otherGroupAttribute, attributes.get(otherGroupAttribute), attrType); + } + } + sourceGroups.put(groupFullName, groupAttrMap); + + Attribute timeStampAttr = attributes.get("uSNChanged"); + if (timeStampAttr != null) { + String uSNChangedVal = (String) timeStampAttr.get(); + long currentDeltaSyncTime = Long.parseLong(uSNChangedVal); + if (currentDeltaSyncTime > highestdeltaSyncGroupTime) { + highestdeltaSyncGroupTime = currentDeltaSyncTime; + } + } else { + timeStampAttr = attributes.get("modifytimestamp"); + if (timeStampAttr != null) { + String timeStampVal = (String) timeStampAttr.get(); + Date parseDate = dateFormat.parse(timeStampVal); + long currentDeltaSyncTime = parseDate.getTime(); + LOG.info("timeStampVal = {} and currentDeltaSyncTime = {}", timeStampVal, currentDeltaSyncTime); + if (currentDeltaSyncTime > highestdeltaSyncGroupTime) { + highestdeltaSyncGroupTime = currentDeltaSyncTime; + deltaSyncGroupTimeStamp = timeStampVal; + } + } + } + Attribute groupMemberAttr = attributes.get(groupMemberAttributeName); + int userCount = 0; + if (groupMemberAttr == null || groupMemberAttr.size() <= 0) { + LOG.info("No members available for {}", gName); + sourceGroupUsers.put(groupFullName, new HashSet<>()); + continue; + } + + NamingEnumeration userEnum = groupMemberAttr.getAll(); + while (userEnum.hasMore()) { + String originalUserFullName = (String) userEnum.next(); + if (originalUserFullName == null || originalUserFullName.trim().isEmpty()) { + sourceGroupUsers.put(groupFullName, new HashSet<>()); + continue; + } + userCount++; + + if (!userSearchEnabled) { + Map userAttrMap = new HashMap<>(); + String userName = getShortName(originalUserFullName); userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName); userAttrMap.put(UgsyncCommonConstants.FULL_NAME, originalUserFullName); userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl()); - sourceUsers.put(originalUserFullName, userAttrMap); - if (LOG.isDebugEnabled()) { - LOG.debug("As usersearch is disabled, adding user " + userName + " from group member attribute for group " + gName); - } - } - - groupUserTable.put(groupFullName, originalUserFullName, originalUserFullName); - } - - LOG.info("No. of members in the group " + gName + " = " + userCount); - } - // Examine the paged results control response - Control[] controls = ldapContext.getResponseControls(); - if (controls != null) { - for (int i = 0; i < controls.length; i++) { - if (controls[i] instanceof PagedResultsResponseControl) { - PagedResultsResponseControl prrc = - (PagedResultsResponseControl)controls[i]; - total = prrc.getResultSize(); - if (total != 0) { - if (LOG.isDebugEnabled()) { - LOG.debug("END-OF-PAGE total : " + total); - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("END-OF-PAGE total : unknown"); - } - } - cookie = prrc.getCookie(); - } - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("No controls were sent from the server"); - } - } - // Re-activate paged results - if (pagedResultsEnabled) { - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("Fetched paged results round: %s", ++paged)); - } - ldapContext.setRequestControls(new Control[]{ - new PagedResultsControl(pagedResultsSize, cookie, Control.CRITICAL) }); - } - } while (cookie != null); - LOG.info("LdapUserGroupBuilder.getGroups() completed with group count: " - + counter); - } catch (Exception t) { - LOG.error("LdapUserGroupBuilder.getGroups() failed with exception: " + t); - LOG.info("LdapUserGroupBuilder.getGroups() group count: " - + counter); - } - } - - } finally { - if (groupSearchResultEnum != null) { - groupSearchResultEnum.close(); - } - closeLdapContext(); - } + sourceUsers.put(originalUserFullName, userAttrMap); + if (LOG.isDebugEnabled()) { + LOG.debug("As usersearch is disabled, adding user {} from group member attribute for group {}", userName, gName); + } + } + + groupUserTable.put(groupFullName, originalUserFullName, originalUserFullName); + } + + LOG.info("No. of members in the group {} = {}", gName, userCount); + } + // Examine the paged results control response + Control[] controls = ldapContext.getResponseControls(); + if (controls != null) { + for (Control control : controls) { + if (control instanceof PagedResultsResponseControl) { + PagedResultsResponseControl prrc = (PagedResultsResponseControl) control; + total = prrc.getResultSize(); + if (total != 0) { + if (LOG.isDebugEnabled()) { + LOG.debug("END-OF-PAGE total: {}", total); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("END-OF-PAGE total: unknown"); + } + } + cookie = prrc.getCookie(); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No controls were sent from the server"); + } + } + // Re-activate paged results + if (pagedResultsEnabled) { + LOG.debug(String.format("Fetched paged results round: %s", ++paged)); + ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, cookie, Control.CRITICAL)}); + } + } + while (cookie != null); + LOG.info("LdapUserGroupBuilder.getGroups() completed with group count: {}", counter); + } catch (Exception t) { + LOG.error("LdapUserGroupBuilder.getGroups() failed with exception: ", t); + LOG.info("LdapUserGroupBuilder.getGroups() group count: {}", counter); + } + } + } finally { + if (groupSearchResultEnum != null) { + groupSearchResultEnum.close(); + } + closeLdapContext(); + } if (groupHierarchyLevels > 0) { - if (LOG.isDebugEnabled()) { - LOG.debug("deltaSyncGroupTime = " + deltaSyncGroupTime); - } + LOG.debug("deltaSyncGroupTime = {}", deltaSyncGroupTime); if (deltaSyncGroupTime > 0) { - LOG.info("LdapUserGroupBuilder.getGroups(): Going through group hierarchy for nested group evaluation for deltasync"); - goUpGroupHierarchyLdap(sourceGroups.keySet(), groupHierarchyLevels-1); + LOG.info("LdapUserGroupBuilder.getGroups(): Going through group hierarchy for nested group evaluation for deltasync"); + goUpGroupHierarchyLdap(sourceGroups.keySet(), groupHierarchyLevels - 1); } } - if (LOG.isDebugEnabled()) { - LOG.debug("highestdeltaSyncGroupTime = " + highestdeltaSyncGroupTime); - } - + LOG.debug("highestdeltaSyncGroupTime = {}", highestdeltaSyncGroupTime); return highestdeltaSyncGroupTime; - } - - private void goUpGroupHierarchy(Set groups, int groupHierarchyLevels, String groupSName) throws InvalidNameException { - if (groupHierarchyLevels <= 0 || groups.isEmpty()) { - return; - } - LOG.info("nextLevelGroups = " + groups + " for group = " + groupSName); - Set nextLevelGroups; - - for (String group : groups) { - - // Add all members of sub group to the parent groups if the member is not a group in turn - Set allMembers = groupUserTable.row(groupSName).keySet(); - LOG.info("members of " + groupSName + " = " + allMembers); - for(String member : allMembers) { - if (!groupUserTable.containsRow(member)) { //Check if the member of a group is in turn a group - LOG.info("Adding " + member + " to " + group); - String userSName = groupUserTable.get(groupSName, member); - LOG.info("Short name of " + member + " = " + userSName); - if (userSName != null) { - groupUserTable.put(group, member, userSName); //Add users from the nested group to parent group - } - } - } - nextLevelGroups = groupUserTable.column(group).keySet(); - goUpGroupHierarchy(nextLevelGroups, groupHierarchyLevels - 1, group); - } - } - - private void goUpGroupHierarchyLdap(Set groupDNs, int groupHierarchyLevels) throws Throwable { - if (groupHierarchyLevels <= 0 || groupDNs.isEmpty()) { - return; - } - Set nextLevelGroups = new HashSet(); - - NamingEnumeration groupSearchResultEnum = null; - try { - createLdapContext(); - int total; - // Activate paged results - if (pagedResultsEnabled) { - ldapContext.setRequestControls(new Control[]{ - new PagedResultsControl(pagedResultsSize, Control.NONCRITICAL) }); - } - String groupFilter = "(&(objectclass=" + groupObjectClass + ")"; - if (groupSearchFilter != null && !groupSearchFilter.trim().isEmpty()) { - String customFilter = groupSearchFilter.trim(); - if (!customFilter.startsWith("(")) { - customFilter = "(" + customFilter + ")"; - } - groupFilter += customFilter + "(|"; - } - StringBuilder filter = new StringBuilder(); - - for (String groupDN : groupDNs) { - filter.append("(").append(groupMemberAttributeName).append("=") - .append(groupDN).append(")"); - } - filter.append("))"); - groupFilter += filter; - - LOG.info("extendedAllGroupsSearchFilter = " + groupFilter); - for (int ou=0; ou groupAttrMap = new HashMap<>(); + } + + private void goUpGroupHierarchy(Set groups, int groupHierarchyLevels, String groupSName) throws InvalidNameException { + if (groupHierarchyLevels <= 0 || groups.isEmpty()) { + return; + } + LOG.info("nextLevelGroups = {} for group = {}", groups, groupSName); + Set nextLevelGroups; + + for (String group : groups) { + // Add all members of subgroup to the parent groups if the member is not a group in turn + Set allMembers = groupUserTable.row(groupSName).keySet(); + LOG.info("members of {} = {}", groupSName, allMembers); + for (String member : allMembers) { + if (!groupUserTable.containsRow(member)) { //Check if the member of a group is in turn a group + LOG.info("Adding {} to {}", member, group); + String userSName = groupUserTable.get(groupSName, member); + LOG.info("Short name of {} = {}", member, userSName); + if (userSName != null) { + groupUserTable.put(group, member, userSName); //Add users from the nested group to parent group + } + } + } + nextLevelGroups = groupUserTable.column(group).keySet(); + goUpGroupHierarchy(nextLevelGroups, groupHierarchyLevels - 1, group); + } + } + + private void goUpGroupHierarchyLdap(Set groupDNs, int groupHierarchyLevels) throws Throwable { + if (groupHierarchyLevels <= 0 || groupDNs.isEmpty()) { + return; + } + Set nextLevelGroups = new HashSet(); + + NamingEnumeration groupSearchResultEnum = null; + try { + createLdapContext(); + int total; + // Activate paged results + if (pagedResultsEnabled) { + ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, Control.NONCRITICAL)}); + } + String groupFilter = "(&(objectclass=" + groupObjectClass + ")"; + if (groupSearchFilter != null && !groupSearchFilter.trim().isEmpty()) { + String customFilter = groupSearchFilter.trim(); + if (!customFilter.startsWith("(")) { + customFilter = "(" + customFilter + ")"; + } + groupFilter += customFilter + "(|"; + } + StringBuilder filter = new StringBuilder(); + + for (String groupDN : groupDNs) { + filter.append("(").append(groupMemberAttributeName).append("=").append(groupDN).append(")"); + } + filter.append("))"); + groupFilter += filter; + + LOG.info("extendedAllGroupsSearchFilter = {}", groupFilter); + for (String s : groupSearchBase) { + byte[] cookie = null; + int counter = 0; + try { + do { + groupSearchResultEnum = ldapContext.search(s, groupFilter, groupSearchControls); + while (groupSearchResultEnum.hasMore()) { + final SearchResult groupEntry = groupSearchResultEnum.next(); + if (groupEntry == null) { + LOG.info("groupEntry null, skipping sync for the entry"); + continue; + } + counter++; + Attribute groupNameAttr = groupEntry.getAttributes().get(groupNameAttribute); + if (groupNameAttr == null) { + LOG.info("{} empty for entry {}, skipping sync", groupNameAttribute, groupEntry.getNameInNamespace()); + continue; + } + String groupFullName = (groupEntry.getNameInNamespace()); + nextLevelGroups.add(groupFullName); + String gName = (String) groupNameAttr.get(); + + Attribute groupMemberAttr = groupEntry.getAttributes().get(groupMemberAttributeName); + int userCount = 0; + if (groupMemberAttr == null || groupMemberAttr.size() <= 0) { + LOG.info("No members available for {}", gName); + continue; + } + + Map groupAttrMap = new HashMap<>(); groupAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, gName); groupAttrMap.put(UgsyncCommonConstants.FULL_NAME, groupFullName); groupAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); groupAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl()); - for (String otherGroupAttribute : otherGroupAttributes) { - Attribute otherGroupAttr = groupEntry.getAttributes().get(otherGroupAttribute); - if (otherGroupAttr != null) { - groupAttrMap.put(otherGroupAttribute, (String) otherGroupAttr.get()); - } - } - sourceGroups.put(groupFullName, groupAttrMap); - - NamingEnumeration userEnum = groupMemberAttr.getAll(); - while (userEnum.hasMore()) { - String originalUserFullName = (String) userEnum.next(); - if (originalUserFullName == null || originalUserFullName.trim().isEmpty()) { - continue; - } - userCount++; - if (!userSearchEnabled && !sourceGroups.containsKey(originalUserFullName)) { - Map userAttrMap = new HashMap<>(); - String userName = getShortName(originalUserFullName); + for (String otherGroupAttribute : otherGroupAttributes) { + Attribute otherGroupAttr = groupEntry.getAttributes().get(otherGroupAttribute); + if (otherGroupAttr != null) { + groupAttrMap.put(otherGroupAttribute, (String) otherGroupAttr.get()); + } + } + sourceGroups.put(groupFullName, groupAttrMap); + + NamingEnumeration userEnum = groupMemberAttr.getAll(); + while (userEnum.hasMore()) { + String originalUserFullName = (String) userEnum.next(); + if (originalUserFullName == null || originalUserFullName.trim().isEmpty()) { + continue; + } + userCount++; + if (!userSearchEnabled && !sourceGroups.containsKey(originalUserFullName)) { + Map userAttrMap = new HashMap<>(); + String userName = getShortName(originalUserFullName); userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName); userAttrMap.put(UgsyncCommonConstants.FULL_NAME, originalUserFullName); userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl()); - sourceUsers.put(originalUserFullName, userAttrMap); - } - groupUserTable.put(groupFullName, originalUserFullName, originalUserFullName); - - } - LOG.info("No. of members in the group " + gName + " = " + userCount); - } - // Examine the paged results control response - Control[] controls = ldapContext.getResponseControls(); - if (controls != null) { - for (int i = 0; i < controls.length; i++) { - if (controls[i] instanceof PagedResultsResponseControl) { - PagedResultsResponseControl prrc = - (PagedResultsResponseControl)controls[i]; - total = prrc.getResultSize(); - if (total != 0) { - if (LOG.isDebugEnabled()) { - LOG.debug("END-OF-PAGE total : " + total); - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("END-OF-PAGE total : unknown"); - } - } - cookie = prrc.getCookie(); - } - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("No controls were sent from the server"); - } - } - // Re-activate paged results - if (pagedResultsEnabled) { - ldapContext.setRequestControls(new Control[]{ - new PagedResultsControl(pagedResultsSize, cookie, Control.CRITICAL) }); - } - } while (cookie != null); - LOG.info("LdapUserGroupBuilder.goUpGroupHierarchyLdap() completed with group count: " - + counter); - } catch (RuntimeException re) { - LOG.error("LdapUserGroupBuilder.goUpGroupHierarchyLdap() failed with runtime exception: ", re); - throw re; - } catch (Exception t) { - LOG.error("LdapUserGroupBuilder.goUpGroupHierarchyLdap() failed with exception: ", t); - LOG.info("LdapUserGroupBuilder.goUpGroupHierarchyLdap() group count: " - + counter); - } - } - - } catch (RuntimeException re) { - LOG.error("LdapUserGroupBuilder.goUpGroupHierarchyLdap() failed with exception: ", re); - throw re; - } finally { - if (groupSearchResultEnum != null) { - groupSearchResultEnum.close(); - } - closeLdapContext(); - } - goUpGroupHierarchyLdap(nextLevelGroups, groupHierarchyLevels-1); - } - - private void addToAttrMap(Map userAttrMap, String attrName, Attribute attr, String attrType) throws Throwable{ - if (attrType.equals(DATA_TYPE_BYTEARRAY)) { - try { - byte[] otherUserAttrBytes = (byte[]) attr.get(); - //Convert objectGUID into string and add to userAttrMap - String attrVal = UUID.nameUUIDFromBytes(otherUserAttrBytes).toString(); - userAttrMap.put(attrName, attrVal); - } catch (ClassCastException e) { - LOG.error(attrName + " type is not set properly " + e.getMessage()); - } - } else if (attrType.equals("String")) { - userAttrMap.put(attrName, (String) attr.get()); - } else { - // This should not be reached. - LOG.warn("Attribute Type " + attrType + " not supported for " + attrName); - } - } - - private static String getShortName(String longName) { - if (StringUtils.isEmpty(longName)) { - return null; - } - String shortName = ""; - try { - LdapName subjectDN = new LdapName(longName); - List rdns = subjectDN.getRdns(); - for (int i = rdns.size() - 1; i >= 0; i--) { - if (StringUtils.isNotEmpty(shortName)) { - break; - } - Rdn rdn = rdns.get(i); - Attributes attributes = rdn.toAttributes(); - try { - Attribute uid = attributes.get("uid"); - if (uid != null) { - Object value = uid.get(); - if (value != null) { - shortName = value.toString(); - } - } else { - Attribute cn = attributes.get("cn"); - if (cn != null) { - Object value = cn.get(); - if (value != null) { - shortName = value.toString(); - } - } - } - } catch (NoSuchElementException ignore) { - shortName = longName; - } catch (NamingException ignore) { - shortName = longName; - } - } - } catch (InvalidNameException ex) { - shortName = longName; - } - LOG.info("longName: " + longName + ", userName: " + shortName); - return shortName; - } - - private String getFirstRDN(String name) { - if (StringUtils.isEmpty(name)) { - return null; - } - String shortName = ""; - try { - LdapName subjectDN = new LdapName(name); - List rdns = subjectDN.getRdns(); - for (int i = rdns.size() - 1; i >= 0; i--) { - if (StringUtils.isNotEmpty(shortName)) { - break; - } - Rdn rdn = rdns.get(i); - Attributes attributes = rdn.toAttributes(); - try { - Attribute cn = attributes.get("cn"); - if (cn != null) { - Object value = cn.get(); - if (value != null) { - shortName = GROUP_NAME_ATTRIBUTE + value.toString(); - } - } - } catch (NoSuchElementException ignore) { - LOG.warn("NoSuchElementException while retrieving first RDN for " + name); - } catch (NamingException ignore) { - LOG.warn("NamingException while retrieving first RDN for " + name); - } - } - } catch (InvalidNameException ex) { - LOG.warn("InvalidNameException while retrieving first RDN for " + name); - } - if (LOG.isDebugEnabled()) { - LOG.debug("Input group name: " + name + ", first RDN: " + shortName); - } - return shortName; - } - - private String getDNForMemberOf(String searchFilter) throws Throwable { - NamingEnumeration userSearchResultEnum = null; - if (LOG.isDebugEnabled()) { - LOG.debug("getDNForMemberOf(" + searchFilter + ")"); - } - String computedSearchFilter = ""; - try { - createLdapContext(); - int total; - // Activate paged results - if (pagedResultsEnabled) { - ldapContext.setRequestControls(new Control[]{ - new PagedResultsControl(pagedResultsSize, Control.NONCRITICAL) }); - } - SearchControls searchControls = new SearchControls(); - searchControls.setSearchScope(groupSearchScope); - - Set searchAttributes = new HashSet(); - searchAttributes.add(groupNameAttribute); - searchControls.setReturningAttributes(searchAttributes.toArray( - new String[searchAttributes.size()])); - - - // When multiple OUs are configured, go through each OU as the user search base to search for users. - for (int ou=0; ou 2000 group records have been synchronized so far. From now on, only a summary progress log will be written for every 100 users. To continue to see detailed log for every user, please enable Trace level logging. <==="); - } - } else { - if (LOG.isTraceEnabled()) { - LOG.trace("Updating group count: " + counter - + ", groupName: " + groupFullName); - } else { - if ( counter % 100 == 0) { - LOG.info("Synced " + counter + " groups till now"); - } - } - } - - } - - // Examine the paged results control response - Control[] controls = ldapContext.getResponseControls(); - if (controls != null) { - for (int i = 0; i < controls.length; i++) { - if (controls[i] instanceof PagedResultsResponseControl) { - PagedResultsResponseControl prrc = - (PagedResultsResponseControl)controls[i]; - total = prrc.getResultSize(); - if (total != 0) { - if (LOG.isDebugEnabled()) { - LOG.debug("END-OF-PAGE total : " + total); - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("END-OF-PAGE total : unknown"); - } - } - cookie = prrc.getCookie(); - } - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("No controls were sent from the server"); - } - } - // Re-activate paged results - if (pagedResultsEnabled) { - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("Fetched paged results round: %s", ++paged)); - } - ldapContext.setRequestControls(new Control[]{ - new PagedResultsControl(pagedResultsSize, cookie, Control.CRITICAL) }); - } - } while (cookie != null); - LOG.info("LdapUserGroupBuilder.getDNForMemberOf() completed with group count: " - + counter); - } catch (Exception t) { - LOG.error("LdapUserGroupBuilder.getDNForMemberOf() failed with exception: ", t); - LOG.info("LdapUserGroupBuilder.getDNForMemberOf() group count: " - + counter); - } - } - } finally { - if (userSearchResultEnum != null) { - userSearchResultEnum.close(); - } - closeLdapContext(); - } - if (LOG.isDebugEnabled()) { - LOG.debug("computedSearchFilter = " + computedSearchFilter); - } - return computedSearchFilter; - } + sourceUsers.put(originalUserFullName, userAttrMap); + } + groupUserTable.put(groupFullName, originalUserFullName, originalUserFullName); + } + LOG.info("No. of members in the group {} = {}", gName, userCount); + } + // Examine the paged results control response + Control[] controls = ldapContext.getResponseControls(); + if (controls != null) { + for (Control control : controls) { + if (control instanceof PagedResultsResponseControl) { + PagedResultsResponseControl prrc = (PagedResultsResponseControl) control; + total = prrc.getResultSize(); + if (total != 0) { + if (LOG.isDebugEnabled()) { + LOG.debug("END-OF-PAGE total : {}", total); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("END-OF-PAGE total : unknown"); + } + } + cookie = prrc.getCookie(); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No controls were sent from the server"); + } + } + // Re-activate paged results + if (pagedResultsEnabled) { + ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, cookie, Control.CRITICAL)}); + } + } + while (cookie != null); + LOG.info("LdapUserGroupBuilder.goUpGroupHierarchyLdap() completed with group count: {}", counter); + } catch (RuntimeException re) { + LOG.error("LdapUserGroupBuilder.goUpGroupHierarchyLdap() failed with runtime exception: ", re); + throw re; + } catch (Exception t) { + LOG.error("LdapUserGroupBuilder.goUpGroupHierarchyLdap() failed with exception: ", t); + LOG.info("LdapUserGroupBuilder.goUpGroupHierarchyLdap() group count: {}", counter); + } + } + } catch (RuntimeException re) { + LOG.error("LdapUserGroupBuilder.goUpGroupHierarchyLdap() failed with exception: ", re); + throw re; + } finally { + if (groupSearchResultEnum != null) { + groupSearchResultEnum.close(); + } + closeLdapContext(); + } + goUpGroupHierarchyLdap(nextLevelGroups, groupHierarchyLevels - 1); + } + + private void addToAttrMap(Map userAttrMap, String attrName, Attribute attr, String attrType) throws Throwable { + if (attrType.equals(DATA_TYPE_BYTEARRAY)) { + try { + byte[] otherUserAttrBytes = (byte[]) attr.get(); + String attrVal = UUID.nameUUIDFromBytes(otherUserAttrBytes).toString(); + userAttrMap.put(attrName, attrVal); + } catch (ClassCastException e) { + LOG.error("{} type is not set properly ", attrName, e.getMessage()); + } + } else if (attrType.equals("String")) { + userAttrMap.put(attrName, (String) attr.get()); + } else { + LOG.warn("Attribute Type {} not supported for {}", attrType, attrName); + } + } + + private static String getShortName(String longName) { + if (StringUtils.isEmpty(longName)) { + return null; + } + String shortName = ""; + try { + LdapName subjectDN = new LdapName(longName); + List rdns = subjectDN.getRdns(); + for (int i = rdns.size() - 1; i >= 0; i--) { + if (StringUtils.isNotEmpty(shortName)) { + break; + } + Rdn rdn = rdns.get(i); + Attributes attributes = rdn.toAttributes(); + try { + Attribute uid = attributes.get("uid"); + if (uid != null) { + Object value = uid.get(); + if (value != null) { + shortName = value.toString(); + } + } else { + Attribute cn = attributes.get("cn"); + if (cn != null) { + Object value = cn.get(); + if (value != null) { + shortName = value.toString(); + } + } + } + } catch (NoSuchElementException ignore) { + shortName = longName; + } catch (NamingException ignore) { + shortName = longName; + } + } + } catch (InvalidNameException ex) { + shortName = longName; + } + LOG.info("longName: {}, userName: ", longName, shortName); + return shortName; + } + + private String getFirstRDN(String name) { + if (StringUtils.isEmpty(name)) { + return null; + } + String shortName = ""; + try { + LdapName subjectDN = new LdapName(name); + List rdns = subjectDN.getRdns(); + for (int i = rdns.size() - 1; i >= 0; i--) { + if (StringUtils.isNotEmpty(shortName)) { + break; + } + Rdn rdn = rdns.get(i); + Attributes attributes = rdn.toAttributes(); + try { + Attribute cn = attributes.get("cn"); + if (cn != null) { + Object value = cn.get(); + if (value != null) { + shortName = GROUP_NAME_ATTRIBUTE + value; + } + } + } catch (NoSuchElementException ignore) { + LOG.warn("NoSuchElementException while retrieving first RDN for {}", name); + } catch (NamingException ignore) { + LOG.warn("NamingException while retrieving first RDN for {}", name); + } + } + } catch (InvalidNameException ex) { + LOG.warn("InvalidNameException while retrieving first RDN for " + name); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Input group name: {}, first RDN: {}", name, shortName); + } + return shortName; + } + + private String getDNForMemberOf(String searchFilter) throws Throwable { + NamingEnumeration userSearchResultEnum = null; + + LOG.debug("getDNForMemberOf({})", searchFilter); + + String computedSearchFilter = ""; + try { + createLdapContext(); + int total; + // Activate paged results + if (pagedResultsEnabled) { + ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, Control.NONCRITICAL)}); + } + SearchControls searchControls = new SearchControls(); + searchControls.setSearchScope(groupSearchScope); + + Set searchAttributes = new HashSet(); + searchAttributes.add(groupNameAttribute); + searchControls.setReturningAttributes(searchAttributes.toArray(new String[searchAttributes.size()])); + + // When multiple OUs are configured, go through each OU as the user search base to search for users. + for (String s : groupSearchBase) { + byte[] cookie = null; + int counter = 0; + try { + int paged = 0; + do { + String filter = String.format("(&(objectclass=%s)(%s))", groupObjectClass, searchFilter); + userSearchResultEnum = ldapContext.search(s, filter, searchControls); + + while (userSearchResultEnum.hasMore()) { + // searchResults contains all the user entries + final SearchResult userEntry = userSearchResultEnum.next(); + + if (userEntry == null) { + LOG.info("userEntry null, skipping sync for the entry"); + continue; + } + + Attributes attributes = userEntry.getAttributes(); + if (attributes == null) { + LOG.info("attributes missing for entry {}, skipping sync", userEntry.getNameInNamespace()); + continue; + } + + Attribute groupNameAttr = attributes.get(groupNameAttribute); + if (groupNameAttr == null) { + LOG.info("{} missing for entry {}, skipping sync", groupNameAttribute, userEntry.getNameInNamespace()); + continue; + } + + String groupFullName = userEntry.getNameInNamespace(); + LOG.info("groupFullName = {}", groupFullName); + computedSearchFilter += "(" + MEMBER_OF_ATTR + groupFullName + ")"; + counter++; + + if (counter <= 2000) { + LOG.info("Updating group count: {}, groupName: {}", counter, groupFullName); + if (counter == 2000) { + LOG.info("===> 2000 group records have been synchronized so far. From now on, only a summary progress log will be written for every 100 users. To continue to see detailed log for every user, please enable Trace level logging. <==="); + } + } else { + if (LOG.isTraceEnabled()) { + LOG.trace("Updating group count: {}, groupName: {}", counter, groupFullName); + } else if (counter % 100 == 0) { + LOG.info("Synced {} groups till now", counter); + } + } + } + + // Examine the paged results control response + Control[] controls = ldapContext.getResponseControls(); + if (controls != null) { + for (Control control : controls) { + if (control instanceof PagedResultsResponseControl) { + PagedResultsResponseControl prrc = (PagedResultsResponseControl) control; + total = prrc.getResultSize(); + if (total != 0) { + if (LOG.isDebugEnabled()) { + LOG.debug("END-OF-PAGE total : {}", total); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("END-OF-PAGE total : unknown"); + } + } + cookie = prrc.getCookie(); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No controls were sent from the server"); + } + } + // Re-activate paged results + if (pagedResultsEnabled) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Fetched paged results round: %s", ++paged)); + } + ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, cookie, Control.CRITICAL)}); + } + } + while (cookie != null); + LOG.info("LdapUserGroupBuilder.getDNForMemberOf() completed with group count: {}", counter); + } catch (Exception t) { + LOG.error("LdapUserGroupBuilder.getDNForMemberOf() failed with exception: ", t); + LOG.info("LdapUserGroupBuilder.getDNForMemberOf() group count: {}", counter); + } + } + } finally { + if (userSearchResultEnum != null) { + userSearchResultEnum.close(); + } + closeLdapContext(); + } + if (LOG.isDebugEnabled()) { + LOG.debug("computedSearchFilter = {}", computedSearchFilter); + } + return computedSearchFilter; + } } diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java index a10b8233e3..d0b9004cd6 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java @@ -19,6 +19,17 @@ package org.apache.ranger.unixusersync.config; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.credentialapi.CredentialReader; +import org.apache.ranger.plugin.util.RangerCommonConstants; +import org.apache.ranger.plugin.util.XMLUtils; +import org.apache.ranger.unixusersync.ha.UserSyncHAInitializerImpl; +import org.apache.ranger.usergroupsync.UserGroupSink; +import org.apache.ranger.usergroupsync.UserGroupSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -31,1120 +42,1012 @@ import java.util.Set; import java.util.StringTokenizer; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -//import org.apache.hadoop.security.alias.BouncyCastleFipsKeyStoreProvider; -import org.apache.ranger.credentialapi.CredentialReader; -import org.apache.ranger.plugin.util.RangerCommonConstants; -import org.apache.ranger.plugin.util.XMLUtils; -import org.apache.ranger.unixusersync.ha.UserSyncHAInitializerImpl; -import org.apache.ranger.usergroupsync.UserGroupSink; -import org.apache.ranger.usergroupsync.UserGroupSource; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +public class UserGroupSyncConfig { + private static final Logger LOG = LoggerFactory.getLogger(UserGroupSyncConfig.class); + public static final String CONFIG_FILE = "ranger-ugsync-site.xml"; + public static final String DEFAULT_CONFIG_FILE = "ranger-ugsync-default.xml"; + private static final String CORE_SITE_CONFIG_FILE = "core-site.xml"; + public static final String UGSYNC_ENABLED_PROP = "ranger.usersync.enabled"; + public static final String UGSYNC_PM_URL_PROP = "ranger.usersync.policymanager.baseURL"; + public static final String UGSYNC_UNIX_PASSWORD_FILE = "ranger.usersync.unix.password.file"; + public static final String DEFAULT_UGSYNC_UNIX_PASSWORD_FILE = "/etc/passwd"; + public static final String UGSYNC_UNIX_GROUP_FILE = "ranger.usersync.unix.group.file"; + public static final String DEFAULT_UGSYNC_UNIX_GROUP_FILE = "/etc/group"; + public static final String UGSYNC_MIN_USERID_PROP = "ranger.usersync.unix.minUserId"; + public static final String UGSYNC_MIN_GROUPID_PROP = "ranger.usersync.unix.minGroupId"; + public static final String DEFAULT_UGSYNC_MIN_GROUPID = "0"; + public static final String UGSYNC_MAX_RECORDS_PER_API_CALL_PROP = "ranger.usersync.policymanager.maxrecordsperapicall"; + public static final String UGSYNC_MOCK_RUN_PROP = "ranger.usersync.policymanager.mockrun"; + public static final String UGSYNC_TEST_RUN_PROP = "ranger.usersync.policymanager.testrun"; + public static final String UGSYNC_SOURCE_FILE_PROC = "ranger.usersync.filesource.file"; + public static final String UGSYNC_SOURCE_FILE_DELIMITER = "ranger.usersync.filesource.text.delimiter"; + public static final String UGSYNC_SOURCE_FILE_DELIMITERER = "ranger.usersync.filesource.text.delimiterer"; + public static final String UGSYNC_NONE_CASE_CONVERSION_VALUE = "none"; + public static final String UGSYNC_LOWER_CASE_CONVERSION_VALUE = "lower"; + public static final String UGSYNC_UPPER_CASE_CONVERSION_VALUE = "upper"; + public static final String SYNC_MAPPING_USERNAME = "ranger.usersync.mapping.username.regex"; + public static final String SYNC_MAPPING_GROUPNAME = "ranger.usersync.mapping.groupname.regex"; + public static final String UGSYNC_METRICS_ENABLED_PROP = "ranger.usersync.metrics.enabled"; + public static final String UGSYNC_NAME_VALIDATION_ENABLED = "ranger.usersync.name.validation.enabled"; + public static final String UGSYNC_SERVER_HA_ENABLED_PARAM = "ranger-ugsync.server.ha.enabled"; + public static final String UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED = "ranger.usersync.syncsource.validation.enabled"; + + private static final String SSL_KEYSTORE_FILE_TYPE_PARAM = "ranger.keystore.file.type"; + private static final String SSL_TRUSTSTORE_FILE_TYPE_PARAM = "ranger.truststore.file.type"; + private static final String SSL_KEYSTORE_PATH_PARAM = "ranger.usersync.keystore.file"; + private static final String SSL_KEYSTORE_PATH_PASSWORD_PARAM = "ranger.usersync.keystore.password"; + private static final String SSL_KEYSTORE_PATH_PASSWORD_ALIAS = "usersync.ssl.key.password"; + private static final String SSL_TRUSTSTORE_PATH_PARAM = "ranger.usersync.truststore.file"; + private static final String SSL_TRUSTSTORE_PATH_PASSWORD_PARAM = "ranger.usersync.truststore.password"; + private static final String SSL_TRUSTSTORE_PATH_PASSWORD_ALIAS = "usersync.ssl.truststore.password"; + private static final String UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM = "ranger.usersync.sleeptimeinmillisbetweensynccycle"; + private static final String UGSYNC_SLEEP_LDAP_FORCE_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM_ENABLED = "ranger.usersync.ldap.force.sleeptimeinmillisbetweensynccycle.enabled"; + private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE = 60000L; + private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_UNIX_DEFAULT_VALUE = 60000L; + private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_LDAP_DEFAULT_VALUE = 3600000L; + private static final String UGSYNC_SOURCE_CLASS_PARAM = "ranger.usersync.source.impl.class"; + private static final String UGSYNC_SINK_CLASS_PARAM = "ranger.usersync.sink.impl.class"; + private static final String UGSYNC_SOURCE_CLASS = "org.apache.ranger.unixusersync.process.UnixUserGroupBuilder"; + private static final String UGSYNC_SINK_CLASS = "org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder"; + private static final String LGSYNC_SOURCE_CLASS = "org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder"; + private static final String LGSYNC_LDAP_URL = "ranger.usersync.ldap.url"; + private static final String LGSYNC_LDAP_DELTASYNC_ENABLED = "ranger.usersync.ldap.deltasync"; + private static final boolean DEFAULT_LGSYNC_LDAP_DELTASYNC_ENABLED = false; + private static final String LGSYNC_LDAP_STARTTLS_ENABLED = "ranger.usersync.ldap.starttls"; + private static final boolean DEFAULT_LGSYNC_LDAP_STARTTLS_ENABLED = false; + private static final String LGSYNC_LDAP_BIND_DN = "ranger.usersync.ldap.binddn"; + private static final String LGSYNC_LDAP_BIND_KEYSTORE = "ranger.usersync.credstore.filename"; + private static final String LGSYNC_LDAP_BIND_ALIAS = "ranger.usersync.ldap.bindalias"; + private static final String LGSYNC_LDAP_BIND_PASSWORD = "ranger.usersync.ldap.ldapbindpassword"; + private static final String LGSYNC_LDAP_AUTHENTICATION_MECHANISM = "ranger.usersync.ldap.authentication.mechanism"; + private static final String DEFAULT_AUTHENTICATION_MECHANISM = "simple"; + private static final String LGSYNC_SEARCH_BASE = "ranger.usersync.ldap.searchBase"; + private static final String LGSYNC_USER_SEARCH_BASE = "ranger.usersync.ldap.user.searchbase"; + private static final String LGSYNC_USER_SEARCH_SCOPE = "ranger.usersync.ldap.user.searchscope"; + private static final String LGSYNC_USER_OBJECT_CLASS = "ranger.usersync.ldap.user.objectclass"; + private static final String DEFAULT_USER_OBJECT_CLASS = "person"; + private static final String LGSYNC_GROUPNAMES = "ranger.usersync.ldap.groupnames"; + private static final String LGSYNC_USER_SEARCH_FILTER = "ranger.usersync.ldap.user.searchfilter"; + private static final String LGSYNC_USER_NAME_ATTRIBUTE = "ranger.usersync.ldap.user.nameattribute"; + private static final String DEFAULT_USER_NAME_ATTRIBUTE = "cn"; + private static final String LGSYNC_USER_GROUP_NAME_ATTRIBUTE = "ranger.usersync.ldap.user.groupnameattribute"; + private static final String DEFAULT_USER_GROUP_NAME_ATTRIBUTE = "memberof,ismemberof"; + private static final String LGSYNC_USER_CLOUDID_ATTRIBUTE = "ranger.usersync.ldap.user.cloudid.attribute"; + private static final String DEFAULT_USER_CLOUDID_ATTRIBUTE = "objectid"; + private static final String LGSYNC_USER_CLOUDID_ATTRIBUTE_DATATYPE = "ranger.usersync.ldap.user.cloudid.attribute.datatype"; + private static final String DEFAULT_USER_CLOUDID_ATTRIBUTE_DATATYPE = "byte[]"; + private static final String LGSYNC_OTHER_USER_ATTRIBUTES = "ranger.usersync.ldap.user.otherattributes"; + private static final String DEFAULT_OTHER_USER_ATTRIBUTES = "userurincipaluame,"; + private static final String UGSYNC_USERNAME_CASE_CONVERSION_PARAM = "ranger.usersync.ldap.username.caseconversion"; + private static final String DEFAULT_UGSYNC_USERNAME_CASE_CONVERSION_VALUE = UGSYNC_NONE_CASE_CONVERSION_VALUE; + private static final String UGSYNC_GROUPNAME_CASE_CONVERSION_PARAM = "ranger.usersync.ldap.groupname.caseconversion"; + private static final String DEFAULT_UGSYNC_GROUPNAME_CASE_CONVERSION_VALUE = UGSYNC_NONE_CASE_CONVERSION_VALUE; + private static final String DEFAULT_USER_GROUP_TEXTFILE_DELIMITER = ","; + private static final String LGSYNC_PAGED_RESULTS_ENABLED = "ranger.usersync.pagedresultsenabled"; + private static final boolean DEFAULT_LGSYNC_PAGED_RESULTS_ENABLED = true; + private static final String LGSYNC_PAGED_RESULTS_SIZE = "ranger.usersync.pagedresultssize"; + private static final int DEFAULT_LGSYNC_PAGED_RESULTS_SIZE = 500; + private static final String LGSYNC_GROUP_SEARCH_ENABLED = "ranger.usersync.group.searchenabled"; + private static final boolean DEFAULT_LGSYNC_GROUP_SEARCH_ENABLED = true; + private static final String LGSYNC_GROUP_SEARCH_FIRST_ENABLED = "ranger.usersync.group.search.first.enabled"; + private static final boolean DEFAULT_LGSYNC_GROUP_SEARCH_FIRST_ENABLED = true; + /** This flag (ranger.usersync.user.searchenabled) is used only when group search first is enabled to get username either - + * from the group member attribute of the group or + * from the additional user search based on the user attribute configuration + */ + private static final String LGSYNC_USER_SEARCH_ENABLED = "ranger.usersync.user.searchenabled"; + private static final boolean DEFAULT_LGSYNC_USER_SEARCH_ENABLED = true; + private static final String LGSYNC_GROUP_SEARCH_BASE = "ranger.usersync.group.searchbase"; + private static final String LGSYNC_GROUP_SEARCH_SCOPE = "ranger.usersync.group.searchscope"; + private static final String LGSYNC_GROUP_OBJECT_CLASS = "ranger.usersync.group.objectclass"; + private static final String DEFAULT_LGSYNC_GROUP_OBJECT_CLASS = "groupofnames"; + private static final String LGSYNC_GROUP_SEARCH_FILTER = "ranger.usersync.group.searchfilter"; + private static final String LGSYNC_GROUP_NAME_ATTRIBUTE = "ranger.usersync.group.nameattribute"; + private static final String DEFAULT_LGSYNC_GROUP_NAME_ATTRIBUTE = "cn"; + private static final String LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = "ranger.usersync.group.memberattributename"; + private static final String DEFAULT_LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = "member"; + private static final String LGSYNC_GROUP_CLOUDID_ATTRIBUTE = "ranger.usersync.ldap.group.cloudid.attribute"; + private static final String DEFAULT_GROUP_CLOUDID_ATTRIBUTE = "objectid"; + private static final String LGSYNC_GROUP_CLOUDID_ATTRIBUTE_DATATYPE = "ranger.usersync.ldap.group.cloudid.attribute.datatype"; + private static final String DEFAULT_GROUP_CLOUDID_ATTRIBUTE_DATATYPE = "byte[]"; + private static final String LGSYNC_OTHER_GROUP_ATTRIBUTES = "ranger.usersync.ldap.group.otherattributes"; + private static final String DEFAULT_OTHER_GROUP_ATTRIBUTES = "displayname,"; + private static final String LGSYNC_GROUP_HIERARCHY_LEVELS = "ranger.usersync.ldap.grouphierarchylevels"; + private static final int DEFAULT_LGSYNC_GROUP_HIERARCHY_LEVELS = 0; + private static final String UGSYNC_UPDATE_MILLIS_MIN = "ranger.usersync.unix.updatemillismin"; + private static final long DEFAULT_UGSYNC_UPDATE_MILLIS_MIN = 60_000; // ms + private static final String UGSYNC_UNIX_BACKEND = "ranger.usersync.unix.backend"; + private static final String DEFAULT_UGSYNC_UNIX_BACKEND = "passwd"; + private static final String UGSYNC_GROUP_ENUMERATE_ENABLED = "ranger.usersync.group.enumerate"; + private static final String UGSYNC_GROUP_ENUMERATE_GROUPS = "ranger.usersync.group.enumerategroup"; + private static final String SYNC_POLICY_MGR_KEYSTORE = "ranger.usersync.policymgr.keystore"; + private static final String SYNC_POLICY_MGR_ALIAS = "ranger.usersync.policymgr.alias"; + private static final String SYNC_POLICY_MGR_PASSWORD = "ranger.usersync.policymgr.password"; + private static final String SYNC_POLICY_MGR_USERNAME = "ranger.usersync.policymgr.username"; + private static final String SYNC_POLICY_MGR_MAX_RETRY_ATTEMPTS = "ranger.usersync.policymgr.max.retry.attempts"; + private static final String SYNC_POLICY_MGR_RETRY_INTERVAL_MS = "ranger.usersync.policymgr.retry.interval.ms"; + private static final String DEFAULT_POLICYMGR_USERNAME = "rangerusersync"; + private static final String SYNC_SOURCE = "ranger.usersync.sync.source"; + private static final String LGSYNC_REFERRAL = "ranger.usersync.ldap.referral"; + private static final String DEFAULT_LGSYNC_REFERRAL = "ignore"; + private static final String SYNC_MAPPING_USERNAME_HANDLER = "ranger.usersync.mapping.username.handler"; + private static final String DEFAULT_SYNC_MAPPING_USERNAME_HANDLER = "org.apache.ranger.usergroupsync.RegEx"; + private static final String SYNC_MAPPING_GROUPNAME_HANDLER = "ranger.usersync.mapping.groupname.handler"; + private static final String DEFAULT_SYNC_MAPPING_GROUPNAME_HANDLER = "org.apache.ranger.usergroupsync.RegEx"; + private static final String SYNC_MAPPING_SEPARATOR = "ranger.usersync.mapping.regex.separator"; + private static final String DEFAULT_MAPPING_SEPARATOR = "/"; + private static final String ROLE_ASSIGNMENT_LIST_DELIMITER = "ranger.usersync.role.assignment.list.delimiter"; + private static final String USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER = "ranger.usersync.users.groups.assignment.list.delimiter"; + private static final String USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER = "ranger.usersync.username.groupname.assignment.list.delimiter"; + private static final String GROUP_BASED_ROLE_ASSIGNMENT_RULES = "ranger.usersync.group.based.role.assignment.rules"; + private static final String WHITELIST_USER_ROLE_ASSIGNMENT_RULES = "ranger.usersync.whitelist.users.role.assignment.rules"; + private static final String DEFAULT_WHITELIST_USER_ROLE_ASSIGNMENT_RULES = "&ROLE_SYS_ADMIN:u:admin,rangerusersync,rangertagsync&ROLE_KEY_ADMIN:u:keyadmin"; + private static final String USERSYNC_RANGER_COOKIE_ENABLED_PROP = "ranger.usersync.cookie.enabled"; + private static final String RANGER_ADMIN_COOKIE_NAME_PROPS = "ranger.usersync.dest.ranger.session.cookie.name"; + private static final String UGSYNC_METRICS_FILEPATH = "ranger.usersync.metrics.filepath"; + private static final String DEFAULT_UGSYNC_METRICS_FILEPATH = "/tmp/"; + private static final String UGSYNC_METRICS_FILENAME = "ranger.usersync.metrics.filename"; + private static final String DEFAULT_UGSYNC_METRICS_FILENAME = "ranger_usersync_metric.json"; + private static final String UGSYNC_METRICS_FREQUENCY_TIME_IN_MILLIS_PARAM = "ranger.usersync.metrics.frequencytimeinmillis"; + private static final long DEFAULT_UGSYNC_METRICS_FREQUENCY_TIME_IN_MILLIS = 10000L; + private static final String UGSYNC_DELETES_ENABLED = "ranger.usersync.deletes.enabled"; + private static final boolean DEFAULT_UGSYNC_DELETES_ENABLED = false; + private static final String UGSYNC_DELETES_FREQUENCY = "ranger.usersync.deletes.frequency"; + private static final long DEFAULT_UGSYNC_DELETES_FREQUENCY = 10L; // After every 10 sync cycles + private static final boolean DEFAULT_UGSYNC_NAME_VALIDATION_ENABLED = false; + private static final long UGSYNC_INIT_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE_FOR_HA = 5000L; + private static final boolean DEFAULT_UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED = true; + private static volatile UserGroupSyncConfig me; + private Properties prop = new Properties(); + private Configuration userGroupConfig; + private UserGroupSyncConfig() { + init(); + } -public class UserGroupSyncConfig { + public static UserGroupSyncConfig getInstance() { + UserGroupSyncConfig result = me; + if (result == null) { + synchronized (UserGroupSyncConfig.class) { + result = me; + if (result == null) { + me = new UserGroupSyncConfig(); + result = me; + } + } + } + return result; + } - public static final String CONFIG_FILE = "ranger-ugsync-site.xml"; - private static final Logger LOG = LoggerFactory.getLogger(UserGroupSyncConfig.class); + public static synchronized boolean isUgsyncServiceActive() { + return UserSyncHAInitializerImpl.getInstance(UserGroupSyncConfig.getInstance().getUserGroupConfig()).isActive(); + } - public static final String DEFAULT_CONFIG_FILE = "ranger-ugsync-default.xml"; + public Configuration getConfig() { + Configuration ret = new Configuration(); - private static final String CORE_SITE_CONFIG_FILE = "core-site.xml"; + for (String propName : prop.stringPropertyNames()) { + ret.set(propName, prop.getProperty(propName)); + } - public static final String UGSYNC_ENABLED_PROP = "ranger.usersync.enabled"; + return ret; + } - public static final String UGSYNC_PM_URL_PROP = "ranger.usersync.policymanager.baseURL"; + public Configuration getUserGroupConfig() { + return userGroupConfig; + } - public static final String UGSYNC_UNIX_PASSWORD_FILE = "ranger.usersync.unix.password.file"; - public static final String DEFAULT_UGSYNC_UNIX_PASSWORD_FILE = "/etc/passwd"; - - public static final String UGSYNC_UNIX_GROUP_FILE = "ranger.usersync.unix.group.file"; - public static final String DEFAULT_UGSYNC_UNIX_GROUP_FILE = "/etc/group"; + public String getUserSyncFileSource() { + return prop.getProperty(UGSYNC_SOURCE_FILE_PROC); + } - public static final String UGSYNC_MIN_USERID_PROP = "ranger.usersync.unix.minUserId"; + public String getUserSyncFileSourceDelimiter() { + String val = prop.getProperty(UGSYNC_SOURCE_FILE_DELIMITER); + if (val == null) { + val = prop.getProperty(UGSYNC_SOURCE_FILE_DELIMITERER); + } - public static final String UGSYNC_MIN_GROUPID_PROP = "ranger.usersync.unix.minGroupId"; - public static final String DEFAULT_UGSYNC_MIN_GROUPID = "0"; + if (val == null) { + val = DEFAULT_USER_GROUP_TEXTFILE_DELIMITER; + } + return val; + } - public static final String UGSYNC_MAX_RECORDS_PER_API_CALL_PROP = "ranger.usersync.policymanager.maxrecordsperapicall"; + public String getUnixPasswordFile() { + String val = prop.getProperty(UGSYNC_UNIX_PASSWORD_FILE); + if (val == null) { + val = DEFAULT_UGSYNC_UNIX_PASSWORD_FILE; + } - public static final String UGSYNC_MOCK_RUN_PROP = "ranger.usersync.policymanager.mockrun"; + return val; + } - public static final String UGSYNC_TEST_RUN_PROP = "ranger.usersync.policymanager.testrun"; + public String getUnixGroupFile() { + String val = prop.getProperty(UGSYNC_UNIX_GROUP_FILE); + if (val == null) { + val = DEFAULT_UGSYNC_UNIX_GROUP_FILE; + } - public static final String UGSYNC_SOURCE_FILE_PROC = "ranger.usersync.filesource.file"; + return val; + } - public static final String UGSYNC_SOURCE_FILE_DELIMITER = "ranger.usersync.filesource.text.delimiter"; - public static final String UGSYNC_SOURCE_FILE_DELIMITERER = "ranger.usersync.filesource.text.delimiterer"; + public String getUnixBackend() { + String val = prop.getProperty(UGSYNC_UNIX_BACKEND); + if (val == null) { + val = DEFAULT_UGSYNC_UNIX_BACKEND; + } - private static final String SSL_KEYSTORE_FILE_TYPE_PARAM = "ranger.keystore.file.type"; + return val; + } - private static final String SSL_TRUSTSTORE_FILE_TYPE_PARAM = "ranger.truststore.file.type"; + public boolean isUserSyncEnabled() { + String val = prop.getProperty(UGSYNC_ENABLED_PROP); + return (val != null && val.trim().equalsIgnoreCase("true")); + } - private static final String SSL_KEYSTORE_PATH_PARAM = "ranger.usersync.keystore.file"; + public String getEnumerateGroups() { + return prop.getProperty(UGSYNC_GROUP_ENUMERATE_GROUPS); + } - private static final String SSL_KEYSTORE_PATH_PASSWORD_PARAM = "ranger.usersync.keystore.password"; + public boolean isGroupEnumerateEnabled() { + String val = prop.getProperty(UGSYNC_GROUP_ENUMERATE_ENABLED); + return (val != null && val.trim().equalsIgnoreCase("true")); + } - private static final String SSL_KEYSTORE_PATH_PASSWORD_ALIAS = "usersync.ssl.key.password"; + public boolean isMockRunEnabled() { + String val = prop.getProperty(UGSYNC_MOCK_RUN_PROP); + return (val != null && val.trim().equalsIgnoreCase("true")); + } - private static final String SSL_TRUSTSTORE_PATH_PARAM = "ranger.usersync.truststore.file"; + public boolean isTestRunEnabled() { + String val = prop.getProperty(UGSYNC_TEST_RUN_PROP); + return (val != null && val.trim().equalsIgnoreCase("true")); + } - private static final String SSL_TRUSTSTORE_PATH_PASSWORD_PARAM = "ranger.usersync.truststore.password"; + public String getPolicyManagerBaseURL() { + return prop.getProperty(UGSYNC_PM_URL_PROP); + } - private static final String SSL_TRUSTSTORE_PATH_PASSWORD_ALIAS = "usersync.ssl.truststore.password"; + public String getMinUserId() { + return prop.getProperty(UGSYNC_MIN_USERID_PROP); + } - private static final String UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM = "ranger.usersync.sleeptimeinmillisbetweensynccycle"; + public String getMinGroupId() { + String mgid = prop.getProperty(UGSYNC_MIN_GROUPID_PROP); + if (mgid == null) { + mgid = DEFAULT_UGSYNC_MIN_GROUPID; + } + return mgid; + } - private static final String UGSYNC_SLEEP_LDAP_FORCE_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM_ENABLED = "ranger.usersync.ldap.force.sleeptimeinmillisbetweensynccycle.enabled"; + public String getMaxRecordsPerAPICall() { + return prop.getProperty(UGSYNC_MAX_RECORDS_PER_API_CALL_PROP); + } - private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE = 60000L; + public String getSSLKeyStoreType() { + return prop.getProperty(SSL_KEYSTORE_FILE_TYPE_PARAM, KeyStore.getDefaultType()); + } - private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_UNIX_DEFAULT_VALUE = 60000L; + public String getSSLTrustStoreType() { + return prop.getProperty(SSL_TRUSTSTORE_FILE_TYPE_PARAM, KeyStore.getDefaultType()); + } - private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_LDAP_DEFAULT_VALUE = 3600000L; + public String getSSLKeyStorePath() { + return prop.getProperty(SSL_KEYSTORE_PATH_PARAM); + } - private static final String UGSYNC_SOURCE_CLASS_PARAM = "ranger.usersync.source.impl.class"; + public String getSSLKeyStorePathPassword() { + if (prop == null) { + return null; + } + if (prop.containsKey(LGSYNC_LDAP_BIND_KEYSTORE)) { + String path = prop.getProperty(LGSYNC_LDAP_BIND_KEYSTORE); + String alias = SSL_KEYSTORE_PATH_PASSWORD_ALIAS; + if (path != null && alias != null) { + if (!path.trim().isEmpty() && !alias.trim().isEmpty()) { + if ("bcfks".equalsIgnoreCase(getSSLKeyStoreType())) { + String crendentialProviderPrefixBcfks = "bcfks" + "://file"; + path = crendentialProviderPrefixBcfks + path; + } + String password = CredentialReader.getDecryptedString(path.trim(), alias.trim(), getSSLKeyStoreType()); + if (password != null && !password.trim().isEmpty() && !"none".equalsIgnoreCase(password.trim()) && !"_".equalsIgnoreCase(password.trim())) { + prop.setProperty(SSL_KEYSTORE_PATH_PASSWORD_PARAM, password); + } + } + } + } + return prop.getProperty(SSL_KEYSTORE_PATH_PASSWORD_PARAM); + } - private static final String UGSYNC_SINK_CLASS_PARAM = "ranger.usersync.sink.impl.class"; + public String getSSLTrustStorePath() { + return prop.getProperty(SSL_TRUSTSTORE_PATH_PARAM); + } - private static final String UGSYNC_SOURCE_CLASS = "org.apache.ranger.unixusersync.process.UnixUserGroupBuilder"; + public String getSSLTrustStorePathPassword() { + if (prop == null) { + return null; + } + if (prop.containsKey(LGSYNC_LDAP_BIND_KEYSTORE)) { + String path = prop.getProperty(LGSYNC_LDAP_BIND_KEYSTORE); + if (path != null) { + if (!path.trim().isEmpty()) { + if ("bcfks".equalsIgnoreCase(getSSLKeyStoreType())) { + String crendentialProviderPrefixBcfks = "bcfks://file"; + path = crendentialProviderPrefixBcfks + path; + } + String password = CredentialReader.getDecryptedString(path.trim(), SSL_TRUSTSTORE_PATH_PASSWORD_ALIAS, getSSLKeyStoreType()); + if (password != null && !password.trim().isEmpty() && !"none".equalsIgnoreCase(password.trim()) && !"_".equalsIgnoreCase(password.trim())) { + prop.setProperty(SSL_TRUSTSTORE_PATH_PASSWORD_PARAM, password); + } + } + } + } + return prop.getProperty(SSL_TRUSTSTORE_PATH_PASSWORD_PARAM); + } - private static final String UGSYNC_SINK_CLASS = "org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder"; + public long getUpdateMillisMin() { + String val = prop.getProperty(UGSYNC_UPDATE_MILLIS_MIN); + if (val == null) { + return DEFAULT_UGSYNC_UPDATE_MILLIS_MIN; + } - private static final String LGSYNC_SOURCE_CLASS = "org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder"; + long ret = Long.parseLong(val); + return Math.max(ret, DEFAULT_UGSYNC_UPDATE_MILLIS_MIN); + } - private static final String LGSYNC_LDAP_URL = "ranger.usersync.ldap.url"; + public long getInitSleepTimeInMillisBetweenCycle() throws Throwable { + long initSleepValue = 0; + Configuration config = getUserGroupConfig(); + if (config.getBoolean(UGSYNC_SERVER_HA_ENABLED_PARAM, false)) { + initSleepValue = UGSYNC_INIT_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE_FOR_HA; + } else { + initSleepValue = getSleepTimeInMillisBetweenCycle(); + } + return initSleepValue; + } - private static final String LGSYNC_LDAP_DELTASYNC_ENABLED = "ranger.usersync.ldap.deltasync"; - private static final boolean DEFAULT_LGSYNC_LDAP_DELTASYNC_ENABLED = false; + public long getSleepTimeInMillisBetweenCycle() throws Throwable { + String val = prop.getProperty(UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM); + boolean isLdapForce = Boolean.parseBoolean(prop.getProperty(UGSYNC_SLEEP_LDAP_FORCE_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM_ENABLED)); + String className = getUserGroupSource().getClass().getName(); + if (val == null) { + if (LGSYNC_SOURCE_CLASS.equals(className)) { + return UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_LDAP_DEFAULT_VALUE; + } else { + return UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_UNIX_DEFAULT_VALUE; + } + } else { + long ret = Long.parseLong(val); + long minInterval; + if (LGSYNC_SOURCE_CLASS.equals(className)) { + if (isLdapForce && ret < UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_LDAP_DEFAULT_VALUE) { + minInterval = ret; + LOG.info("If you force the synchronization time of ldap users to be less than the default of 3600s, this setting [{}] millisec will take effect", minInterval); + } else { + minInterval = UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_LDAP_DEFAULT_VALUE; + } + } else if (UGSYNC_SOURCE_CLASS.equals(className)) { + minInterval = UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_UNIX_DEFAULT_VALUE; + } else { + minInterval = UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE; + } + if ((!isTestRunEnabled()) && (ret < minInterval)) { + LOG.info("Sleep Time Between Cycle can not be lower than [{}] millisec. resetting to min value.", minInterval); + ret = minInterval; + } + return ret; + } + } - private static final String LGSYNC_LDAP_STARTTLS_ENABLED = "ranger.usersync.ldap.starttls"; - private static final boolean DEFAULT_LGSYNC_LDAP_STARTTLS_ENABLED = false; + public UserGroupSource getUserGroupSource() throws Throwable { + String className = getUserGroupSourceClassName(); - private static final String LGSYNC_LDAP_BIND_DN = "ranger.usersync.ldap.binddn"; + Class ugSourceClass = (Class) Class.forName(className); - private static final String LGSYNC_LDAP_BIND_KEYSTORE = "ranger.usersync.credstore.filename"; + UserGroupSource ret = ugSourceClass.newInstance(); - private static final String LGSYNC_LDAP_BIND_ALIAS = "ranger.usersync.ldap.bindalias"; + return ret; + } - private static final String LGSYNC_LDAP_BIND_PASSWORD = "ranger.usersync.ldap.ldapbindpassword"; + public UserGroupSink getUserGroupSink() throws Throwable { + String val = prop.getProperty(UGSYNC_SINK_CLASS_PARAM); - private static final String LGSYNC_LDAP_AUTHENTICATION_MECHANISM = "ranger.usersync.ldap.authentication.mechanism"; - private static final String DEFAULT_AUTHENTICATION_MECHANISM = "simple"; + if (val == null || val.trim().isEmpty()) { + val = UGSYNC_SINK_CLASS; + } - private static final String LGSYNC_SEARCH_BASE = "ranger.usersync.ldap.searchBase"; + Class ugSinkClass = (Class) Class.forName(val); - private static final String LGSYNC_USER_SEARCH_BASE = "ranger.usersync.ldap.user.searchbase"; + UserGroupSink ret = ugSinkClass.newInstance(); - private static final String LGSYNC_USER_SEARCH_SCOPE = "ranger.usersync.ldap.user.searchscope"; + return ret; + } - private static final String LGSYNC_USER_OBJECT_CLASS = "ranger.usersync.ldap.user.objectclass"; - private static final String DEFAULT_USER_OBJECT_CLASS = "person"; + public String getLdapUrl() throws Throwable { + String val = prop.getProperty(LGSYNC_LDAP_URL); + if (val == null || val.trim().isEmpty()) { + throw new Exception(LGSYNC_LDAP_URL + " for LdapGroupSync is not specified"); + } + return val; + } - private static final String LGSYNC_GROUPNAMES = "ranger.usersync.ldap.groupnames"; - private static final String LGSYNC_USER_SEARCH_FILTER = "ranger.usersync.ldap.user.searchfilter"; + public String getLdapBindDn() throws Throwable { + String val = prop.getProperty(LGSYNC_LDAP_BIND_DN); + if (val == null || val.trim().isEmpty()) { + throw new Exception(LGSYNC_LDAP_BIND_DN + " for LdapGroupSync is not specified"); + } + return val; + } - private static final String LGSYNC_USER_NAME_ATTRIBUTE = "ranger.usersync.ldap.user.nameattribute"; - private static final String DEFAULT_USER_NAME_ATTRIBUTE = "cn"; + public String getLdapBindPassword() { + //update credential from keystore + if (prop == null) { + return null; + } + if (prop.containsKey(LGSYNC_LDAP_BIND_KEYSTORE)) { + String path = prop.getProperty(LGSYNC_LDAP_BIND_KEYSTORE); + if (path != null) { + if (!path.trim().isEmpty()) { + if ("bcfks".equalsIgnoreCase(getSSLKeyStoreType())) { + String crendentialProviderPrefixBcfks = "bcfks://file"; + path = crendentialProviderPrefixBcfks + path; + } + String password = CredentialReader.getDecryptedString(path.trim(), LGSYNC_LDAP_BIND_ALIAS, getSSLKeyStoreType()); + if (password != null && !password.trim().isEmpty() && !password.trim().equalsIgnoreCase("none")) { + prop.setProperty(LGSYNC_LDAP_BIND_PASSWORD, password); + } + } + } + } + return prop.getProperty(LGSYNC_LDAP_BIND_PASSWORD); + } - private static final String LGSYNC_USER_GROUP_NAME_ATTRIBUTE = "ranger.usersync.ldap.user.groupnameattribute"; - private static final String DEFAULT_USER_GROUP_NAME_ATTRIBUTE = "memberof,ismemberof"; + public String getLdapAuthenticationMechanism() { + String val = prop.getProperty(LGSYNC_LDAP_AUTHENTICATION_MECHANISM); + if (val == null || val.trim().isEmpty()) { + return DEFAULT_AUTHENTICATION_MECHANISM; + } + return val; + } - private static final String LGSYNC_USER_CLOUDID_ATTRIBUTE = "ranger.usersync.ldap.user.cloudid.attribute"; - private static final String DEFAULT_USER_CLOUDID_ATTRIBUTE = "objectid"; + public String getUserSearchBase() throws Throwable { + String val = prop.getProperty(LGSYNC_USER_SEARCH_BASE); + if (val == null || val.trim().isEmpty()) { + val = getSearchBase(); + } + if (val == null || val.trim().isEmpty()) { + throw new Exception(LGSYNC_USER_SEARCH_BASE + " for LdapGroupSync is not specified"); + } + return val; + } - private static final String LGSYNC_USER_CLOUDID_ATTRIBUTE_DATATYPE = "ranger.usersync.ldap.user.cloudid.attribute.datatype"; - private static final String DEFAULT_USER_CLOUDID_ATTRIBUTE_DATATYPE = "byte[]"; + /* Used only for unit testing */ + public void setUserSearchBase(String userSearchBase) throws Throwable { + prop.setProperty(LGSYNC_USER_SEARCH_BASE, userSearchBase); + } - private static final String LGSYNC_OTHER_USER_ATTRIBUTES = "ranger.usersync.ldap.user.otherattributes"; - private static final String DEFAULT_OTHER_USER_ATTRIBUTES = "userurincipaluame,"; + public int getUserSearchScope() { + String val = prop.getProperty(LGSYNC_USER_SEARCH_SCOPE); + if (val == null || val.trim().isEmpty()) { + return 2; //subtree scope + } - public static final String UGSYNC_NONE_CASE_CONVERSION_VALUE = "none"; - public static final String UGSYNC_LOWER_CASE_CONVERSION_VALUE = "lower"; - public static final String UGSYNC_UPPER_CASE_CONVERSION_VALUE = "upper"; + val = val.trim().toLowerCase(); + if (val.equals("0") || val.startsWith("base")) { + return 0; // object scope + } else if (val.equals("1") || val.startsWith("one")) { + return 1; // one level scope + } else { + return 2; // subtree scope + } + } - private static final String UGSYNC_USERNAME_CASE_CONVERSION_PARAM = "ranger.usersync.ldap.username.caseconversion"; - private static final String DEFAULT_UGSYNC_USERNAME_CASE_CONVERSION_VALUE = UGSYNC_NONE_CASE_CONVERSION_VALUE; + public String getUserObjectClass() { + String val = prop.getProperty(LGSYNC_USER_OBJECT_CLASS); + if (val == null || val.trim().isEmpty()) { + return DEFAULT_USER_OBJECT_CLASS; + } + return val; + } - private static final String UGSYNC_GROUPNAME_CASE_CONVERSION_PARAM = "ranger.usersync.ldap.groupname.caseconversion"; - private static final String DEFAULT_UGSYNC_GROUPNAME_CASE_CONVERSION_VALUE = UGSYNC_NONE_CASE_CONVERSION_VALUE; + /* Used only for unit testing */ + public void setUserObjectClass(String userObjectClass) { + prop.setProperty(LGSYNC_USER_OBJECT_CLASS, userObjectClass); + } - private static final String DEFAULT_USER_GROUP_TEXTFILE_DELIMITER = ","; + public String getUserSearchFilter() { + return prop.getProperty(LGSYNC_USER_SEARCH_FILTER); + } - private static final String LGSYNC_PAGED_RESULTS_ENABLED = "ranger.usersync.pagedresultsenabled"; - private static final boolean DEFAULT_LGSYNC_PAGED_RESULTS_ENABLED = true; + /* Used only for unit testing */ + public void setUserSearchFilter(String filter) { + prop.setProperty(LGSYNC_USER_SEARCH_FILTER, filter); + } - private static final String LGSYNC_PAGED_RESULTS_SIZE = "ranger.usersync.pagedresultssize"; - private static final int DEFAULT_LGSYNC_PAGED_RESULTS_SIZE = 500; + public String getUserNameAttribute() { + String val = prop.getProperty(LGSYNC_USER_NAME_ATTRIBUTE); + if (val == null || val.trim().isEmpty()) { + return DEFAULT_USER_NAME_ATTRIBUTE; + } + return val; + } - private static final String LGSYNC_GROUP_SEARCH_ENABLED = "ranger.usersync.group.searchenabled"; - private static final boolean DEFAULT_LGSYNC_GROUP_SEARCH_ENABLED = true; + /* Used only for unit testing */ + public void setUserNameAttribute(String userNameAttr) { + prop.setProperty(LGSYNC_USER_NAME_ATTRIBUTE, userNameAttr); + } - private static final String LGSYNC_GROUP_SEARCH_FIRST_ENABLED = "ranger.usersync.group.search.first.enabled"; - private static final boolean DEFAULT_LGSYNC_GROUP_SEARCH_FIRST_ENABLED = true; + public String getUserGroupNameAttribute() { + String val = prop.getProperty(LGSYNC_USER_GROUP_NAME_ATTRIBUTE); + if (val == null || val.trim().isEmpty()) { + return DEFAULT_USER_GROUP_NAME_ATTRIBUTE; + } + return val; + } - /*This flag (ranger.usersync.user.searchenabled) is used only when group search first is enabled to get username either - - * from the group member attribute of the group or - * from the additional user search based on the user attribute configuration - */ - private static final String LGSYNC_USER_SEARCH_ENABLED = "ranger.usersync.user.searchenabled"; - private static final boolean DEFAULT_LGSYNC_USER_SEARCH_ENABLED = true; + public String getGroupNames() { + return prop.getProperty(LGSYNC_GROUPNAMES); + } - private static final String LGSYNC_GROUP_SEARCH_BASE = "ranger.usersync.group.searchbase"; + public Set getGroupNameSet() { + String groupNames = getGroupNames(); + Set groupNamegSet = new HashSet(); + if (StringUtils.isNotEmpty(groupNames)) { + StringTokenizer st = new StringTokenizer(groupNames, ";"); + while (st.hasMoreTokens()) { + groupNamegSet.add(st.nextToken().trim().toLowerCase()); + } + } + return groupNamegSet; + } - private static final String LGSYNC_GROUP_SEARCH_SCOPE = "ranger.usersync.group.searchscope"; + public Set getUserGroupNameAttributeSet() { + String uga = getUserGroupNameAttribute(); + StringTokenizer st = new StringTokenizer(uga, ","); + Set userGroupNameAttributeSet = new HashSet(); + while (st.hasMoreTokens()) { + userGroupNameAttributeSet.add(st.nextToken().trim()); + } + return userGroupNameAttributeSet; + } - private static final String LGSYNC_GROUP_OBJECT_CLASS = "ranger.usersync.group.objectclass"; - private static final String DEFAULT_LGSYNC_GROUP_OBJECT_CLASS = "groupofnames"; + public Set getOtherUserAttributes() { + String otherAttributes = prop.getProperty(LGSYNC_OTHER_USER_ATTRIBUTES); + if (otherAttributes == null || otherAttributes.trim().isEmpty()) { + otherAttributes = DEFAULT_OTHER_USER_ATTRIBUTES; + } + StringTokenizer st = new StringTokenizer(otherAttributes, ","); + Set otherUserAttributes = new HashSet(); + while (st.hasMoreTokens()) { + otherUserAttributes.add(st.nextToken().trim()); + } + return otherUserAttributes; + } - private static final String LGSYNC_GROUP_SEARCH_FILTER = "ranger.usersync.group.searchfilter"; + public String getUserCloudIdAttribute() { + String val = prop.getProperty(LGSYNC_USER_CLOUDID_ATTRIBUTE); + if (val == null || val.trim().isEmpty()) { + return DEFAULT_USER_CLOUDID_ATTRIBUTE; + } + return val; + } - private static final String LGSYNC_GROUP_NAME_ATTRIBUTE = "ranger.usersync.group.nameattribute"; - private static final String DEFAULT_LGSYNC_GROUP_NAME_ATTRIBUTE = "cn"; + public String getUserCloudIdAttributeDataType() { + String val = prop.getProperty(LGSYNC_USER_CLOUDID_ATTRIBUTE_DATATYPE); + if (val == null || val.trim().isEmpty()) { + return DEFAULT_USER_CLOUDID_ATTRIBUTE_DATATYPE; + } + return val; + } - private static final String LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = "ranger.usersync.group.memberattributename"; - private static final String DEFAULT_LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = "member"; + public String getOtherUserAttributeDataType(String attrName) { + String attrType = prop.getProperty(LGSYNC_OTHER_USER_ATTRIBUTES + "." + attrName + "datatype"); + if (attrType == null || attrType.isEmpty()) { + attrType = "String"; + } + return attrType.trim(); + } - private static final String LGSYNC_GROUP_CLOUDID_ATTRIBUTE = "ranger.usersync.ldap.group.cloudid.attribute"; - private static final String DEFAULT_GROUP_CLOUDID_ATTRIBUTE = "objectid"; + public String getUserNameCaseConversion() { + String ret = prop.getProperty(UGSYNC_USERNAME_CASE_CONVERSION_PARAM, DEFAULT_UGSYNC_USERNAME_CASE_CONVERSION_VALUE); + return ret.trim().toLowerCase(); + } - private static final String LGSYNC_GROUP_CLOUDID_ATTRIBUTE_DATATYPE = "ranger.usersync.ldap.group.cloudid.attribute.datatype"; - private static final String DEFAULT_GROUP_CLOUDID_ATTRIBUTE_DATATYPE = "byte[]"; + public String getGroupNameCaseConversion() { + String ret = prop.getProperty(UGSYNC_GROUPNAME_CASE_CONVERSION_PARAM, DEFAULT_UGSYNC_GROUPNAME_CASE_CONVERSION_VALUE); + return ret.trim().toLowerCase(); + } - private static final String LGSYNC_OTHER_GROUP_ATTRIBUTES = "ranger.usersync.ldap.group.otherattributes"; - private static final String DEFAULT_OTHER_GROUP_ATTRIBUTES = "displayname,"; + public String getSearchBase() { + return prop.getProperty(LGSYNC_SEARCH_BASE); + } - private static final String LGSYNC_GROUP_HIERARCHY_LEVELS = "ranger.usersync.ldap.grouphierarchylevels"; - private static final int DEFAULT_LGSYNC_GROUP_HIERARCHY_LEVELS = 0; + public boolean isPagedResultsEnabled() { + boolean pagedResultsEnabled; + String val = prop.getProperty(LGSYNC_PAGED_RESULTS_ENABLED); + if (val == null || val.trim().isEmpty()) { + pagedResultsEnabled = DEFAULT_LGSYNC_PAGED_RESULTS_ENABLED; + } else { + pagedResultsEnabled = Boolean.valueOf(val); + } + return pagedResultsEnabled; + } - private static final String UGSYNC_UPDATE_MILLIS_MIN = "ranger.usersync.unix.updatemillismin"; - private final static long DEFAULT_UGSYNC_UPDATE_MILLIS_MIN = 1 * 60 * 1000; // ms + /* Used only for unit testing */ + public void setPagedResultsEnabled(boolean pagedResultsEnabled) { + prop.setProperty(LGSYNC_PAGED_RESULTS_ENABLED, String.valueOf(pagedResultsEnabled)); + } - private static final String UGSYNC_UNIX_BACKEND = "ranger.usersync.unix.backend"; - private final static String DEFAULT_UGSYNC_UNIX_BACKEND = "passwd"; + public int getPagedResultsSize() { + int pagedResultsSize = DEFAULT_LGSYNC_PAGED_RESULTS_SIZE; + String val = prop.getProperty(LGSYNC_PAGED_RESULTS_SIZE); + if (val == null || val.trim().isEmpty()) { + pagedResultsSize = DEFAULT_LGSYNC_PAGED_RESULTS_SIZE; + } else { + pagedResultsSize = Integer.parseInt(val); + } + if (pagedResultsSize < 1) { + pagedResultsSize = DEFAULT_LGSYNC_PAGED_RESULTS_SIZE; + } + return pagedResultsSize; + } - private static final String UGSYNC_GROUP_ENUMERATE_ENABLED = "ranger.usersync.group.enumerate"; + public boolean isGroupSearchEnabled() { + boolean groupSearchEnabled; + String val = prop.getProperty(LGSYNC_GROUP_SEARCH_ENABLED); + if (val == null || val.trim().isEmpty()) { + groupSearchEnabled = DEFAULT_LGSYNC_GROUP_SEARCH_ENABLED; + } else { + groupSearchEnabled = Boolean.valueOf(val); + } + return groupSearchEnabled; + } - private static final String UGSYNC_GROUP_ENUMERATE_GROUPS = "ranger.usersync.group.enumerategroup"; + /* Used only for unit testing */ + public void setGroupSearchEnabled(boolean groupSearchEnabled) { + prop.setProperty(LGSYNC_GROUP_SEARCH_ENABLED, String.valueOf(groupSearchEnabled)); + } - private static final String SYNC_POLICY_MGR_KEYSTORE = "ranger.usersync.policymgr.keystore"; + public boolean isGroupSearchFirstEnabled() { + boolean groupSearchFirstEnabled; + String val = prop.getProperty(LGSYNC_GROUP_SEARCH_FIRST_ENABLED); + if (val == null || val.trim().isEmpty()) { + groupSearchFirstEnabled = DEFAULT_LGSYNC_GROUP_SEARCH_FIRST_ENABLED; + } else { + groupSearchFirstEnabled = Boolean.valueOf(val); + } + if (!isGroupSearchEnabled()) { + groupSearchFirstEnabled = false; + } + return groupSearchFirstEnabled; + } - private static final String SYNC_POLICY_MGR_ALIAS = "ranger.usersync.policymgr.alias"; + /* Used only for unit testing */ + public void setGroupSearchFirstEnabled(boolean groupSearchFirstEnabled) { + prop.setProperty(LGSYNC_GROUP_SEARCH_FIRST_ENABLED, String.valueOf(groupSearchFirstEnabled)); + } - private static final String SYNC_POLICY_MGR_PASSWORD = "ranger.usersync.policymgr.password"; + public boolean isUserSearchEnabled() { + boolean userSearchEnabled; + String val = prop.getProperty(LGSYNC_USER_SEARCH_ENABLED); + if (val == null || val.trim().isEmpty()) { + userSearchEnabled = DEFAULT_LGSYNC_USER_SEARCH_ENABLED; + } else { + userSearchEnabled = Boolean.valueOf(val); + } + if (!isGroupSearchFirstEnabled()) { + userSearchEnabled = true; + } + return userSearchEnabled; + } - private static final String SYNC_POLICY_MGR_USERNAME = "ranger.usersync.policymgr.username"; + /* Used only for unit testing */ + public void setUserSearchEnabled(boolean userSearchEnabled) { + prop.setProperty(LGSYNC_USER_SEARCH_ENABLED, String.valueOf(userSearchEnabled)); + } - private static final String SYNC_POLICY_MGR_MAX_RETRY_ATTEMPTS = "ranger.usersync.policymgr.max.retry.attempts"; - private static final String SYNC_POLICY_MGR_RETRY_INTERVAL_MS = "ranger.usersync.policymgr.retry.interval.ms"; + public String getGroupSearchBase() throws Throwable { + String val = prop.getProperty(LGSYNC_GROUP_SEARCH_BASE); + if (val == null || val.trim().isEmpty()) { + val = getSearchBase(); + } + if (val == null || val.trim().isEmpty()) { + val = getUserSearchBase(); + } + return val; + } - private static final String DEFAULT_POLICYMGR_USERNAME = "rangerusersync"; + /* Used only for unit testing */ + public void setGroupSearchBase(String groupSearchBase) throws Throwable { + prop.setProperty(LGSYNC_GROUP_SEARCH_BASE, groupSearchBase); + } - private static final String SYNC_SOURCE = "ranger.usersync.sync.source"; - private static final String LGSYNC_REFERRAL = "ranger.usersync.ldap.referral"; - private static final String DEFAULT_LGSYNC_REFERRAL = "ignore"; + public int getGroupSearchScope() { + String val = prop.getProperty(LGSYNC_GROUP_SEARCH_SCOPE); + if (val == null || val.trim().isEmpty()) { + return 2; //subtree scope + } - public static final String SYNC_MAPPING_USERNAME = "ranger.usersync.mapping.username.regex"; + val = val.trim().toLowerCase(); + if (val.equals("0") || val.startsWith("base")) { + return 0; // object scope + } else if (val.equals("1") || val.startsWith("one")) { + return 1; // one level scope + } else { + return 2; // subtree scope + } + } - public static final String SYNC_MAPPING_GROUPNAME = "ranger.usersync.mapping.groupname.regex"; + public String getGroupObjectClass() { + String val = prop.getProperty(LGSYNC_GROUP_OBJECT_CLASS); + if (val == null || val.trim().isEmpty()) { + return DEFAULT_LGSYNC_GROUP_OBJECT_CLASS; + } + return val; + } - private static final String SYNC_MAPPING_USERNAME_HANDLER = "ranger.usersync.mapping.username.handler"; - private static final String DEFAULT_SYNC_MAPPING_USERNAME_HANDLER = "org.apache.ranger.usergroupsync.RegEx"; + /* Used only for unit testing */ + public void setGroupObjectClass(String groupObjectClass) { + prop.setProperty(LGSYNC_GROUP_OBJECT_CLASS, groupObjectClass); + } - private static final String SYNC_MAPPING_GROUPNAME_HANDLER = "ranger.usersync.mapping.groupname.handler"; - private static final String DEFAULT_SYNC_MAPPING_GROUPNAME_HANDLER = "org.apache.ranger.usergroupsync.RegEx"; + public String getGroupSearchFilter() { + return prop.getProperty(LGSYNC_GROUP_SEARCH_FILTER); + } - private static final String SYNC_MAPPING_SEPARATOR = "ranger.usersync.mapping.regex.separator"; + /* Used only for unit testing */ + public void setGroupSearchFilter(String filter) { + prop.setProperty(LGSYNC_GROUP_SEARCH_FILTER, filter); + } - private static final String DEFAULT_MAPPING_SEPARATOR = "/"; - private static final String ROLE_ASSIGNMENT_LIST_DELIMITER = "ranger.usersync.role.assignment.list.delimiter"; + public String getUserGroupMemberAttributeName() { + String val = prop.getProperty(LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME); + if (val == null || val.trim().isEmpty()) { + return DEFAULT_LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME; + } + return val; + } - private static final String USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER = "ranger.usersync.users.groups.assignment.list.delimiter"; + /* Used only for unit testing */ + public void setUserGroupMemberAttributeName(String groupMemberAttrName) { + prop.setProperty(LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME, groupMemberAttrName); + } - private static final String USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER = "ranger.usersync.username.groupname.assignment.list.delimiter"; + public String getGroupNameAttribute() { + String val = prop.getProperty(LGSYNC_GROUP_NAME_ATTRIBUTE); + if (val == null || val.trim().isEmpty()) { + return DEFAULT_LGSYNC_GROUP_NAME_ATTRIBUTE; + } + return val; + } - private static final String GROUP_BASED_ROLE_ASSIGNMENT_RULES = "ranger.usersync.group.based.role.assignment.rules"; + public String getGroupCloudIdAttribute() { + String val = prop.getProperty(LGSYNC_GROUP_CLOUDID_ATTRIBUTE); + if (val == null || val.trim().isEmpty()) { + return DEFAULT_GROUP_CLOUDID_ATTRIBUTE; + } + return val; + } - private static final String WHITELIST_USER_ROLE_ASSIGNMENT_RULES = "ranger.usersync.whitelist.users.role.assignment.rules"; - private static final String DEFAULT_WHITELIST_USER_ROLE_ASSIGNMENT_RULES = "&ROLE_SYS_ADMIN:u:admin,rangerusersync,rangertagsync&ROLE_KEY_ADMIN:u:keyadmin"; + public String getGroupCloudIdAttributeDataType() { + String val = prop.getProperty(LGSYNC_GROUP_CLOUDID_ATTRIBUTE_DATATYPE); + if (val == null || val.trim().isEmpty()) { + return DEFAULT_GROUP_CLOUDID_ATTRIBUTE_DATATYPE; + } + return val; + } - private static final String USERSYNC_RANGER_COOKIE_ENABLED_PROP = "ranger.usersync.cookie.enabled"; + public Set getOtherGroupAttributes() { + String otherAttributes = prop.getProperty(LGSYNC_OTHER_GROUP_ATTRIBUTES); + if (otherAttributes == null || otherAttributes.trim().isEmpty()) { + otherAttributes = DEFAULT_OTHER_GROUP_ATTRIBUTES; + } + StringTokenizer st = new StringTokenizer(otherAttributes, ","); + Set otherGroupAttributes = new HashSet(); + while (st.hasMoreTokens()) { + otherGroupAttributes.add(st.nextToken().trim()); + } + return otherGroupAttributes; + } - private static final String RANGER_ADMIN_COOKIE_NAME_PROPS = "ranger.usersync.dest.ranger.session.cookie.name"; - - private static final String UGSYNC_METRICS_FILEPATH = "ranger.usersync.metrics.filepath"; - private static final String DEFAULT_UGSYNC_METRICS_FILEPATH = "/tmp/"; - private static final String UGSYNC_METRICS_FILENAME = "ranger.usersync.metrics.filename"; - private static final String DEFAULT_UGSYNC_METRICS_FILENAME = "ranger_usersync_metric.json"; - private static final String UGSYNC_METRICS_FREQUENCY_TIME_IN_MILLIS_PARAM = "ranger.usersync.metrics.frequencytimeinmillis"; - private static final long DEFAULT_UGSYNC_METRICS_FREQUENCY_TIME_IN_MILLIS = 10000L; - public static final String UGSYNC_METRICS_ENABLED_PROP = "ranger.usersync.metrics.enabled"; - - private static final String UGSYNC_DELETES_ENABLED = "ranger.usersync.deletes.enabled"; - private static final boolean DEFAULT_UGSYNC_DELETES_ENABLED = false; - private static final String UGSYNC_DELETES_FREQUENCY = "ranger.usersync.deletes.frequency"; - private static final long DEFAULT_UGSYNC_DELETES_FREQUENCY = 10L; // After every 10 sync cycles - public static final String UGSYNC_NAME_VALIDATION_ENABLED = "ranger.usersync.name.validation.enabled"; - private static final boolean DEFAULT_UGSYNC_NAME_VALIDATION_ENABLED = false; - private static final long UGSYNC_INIT_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE_FOR_HA = 5000L; - public static final String UGSYNC_SERVER_HA_ENABLED_PARAM = "ranger-ugsync.server.ha.enabled"; - public static final String UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED = "ranger.usersync.syncsource.validation.enabled"; - private static final boolean DEFAULT_UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED = true; + public String getOtherGroupAttributeDataType(String attrName) { + String attrType = prop.getProperty(LGSYNC_OTHER_GROUP_ATTRIBUTES + "." + attrName + "datatype"); + if (attrType == null || attrType.isEmpty()) { + attrType = "String"; + } + return attrType.trim(); + } - private Properties prop = new Properties(); - private Configuration userGroupConfig = null; - - private static volatile UserGroupSyncConfig me = null; - - public static UserGroupSyncConfig getInstance() { - UserGroupSyncConfig result = me; - if (result == null) { - synchronized(UserGroupSyncConfig.class) { - result = me; - if (result == null) { - me = result = new UserGroupSyncConfig(); - } - } - } - return result; - } - - private UserGroupSyncConfig() { - init(); - } - - private void init() { - XMLUtils.loadConfig(DEFAULT_CONFIG_FILE, prop); - XMLUtils.loadConfig(CORE_SITE_CONFIG_FILE, prop); - XMLUtils.loadConfig(CONFIG_FILE, prop); - userGroupConfig = getConfig(); - } - - public Configuration getConfig() { - Configuration ret = new Configuration(); - - for (String propName : prop.stringPropertyNames()) { - ret.set(propName, prop.getProperty(propName)); - } - - return ret; - } - - public Configuration getUserGroupConfig(){ - return userGroupConfig; - } - synchronized public static boolean isUgsyncServiceActive() { - return UserSyncHAInitializerImpl.getInstance(UserGroupSyncConfig.getInstance().getUserGroupConfig()).isActive(); - } - - public String getUserSyncFileSource(){ - String val = prop.getProperty(UGSYNC_SOURCE_FILE_PROC); - return val; - } - - public String getUserSyncFileSourceDelimiter(){ - String val = prop.getProperty(UGSYNC_SOURCE_FILE_DELIMITER); - if (val == null) { - val = prop.getProperty(UGSYNC_SOURCE_FILE_DELIMITERER); - } - - if ( val == null) { - val = DEFAULT_USER_GROUP_TEXTFILE_DELIMITER; - } - return val; - } - - public String getUnixPasswordFile() { - String val = prop.getProperty(UGSYNC_UNIX_PASSWORD_FILE); - if ( val == null ) { - val = DEFAULT_UGSYNC_UNIX_PASSWORD_FILE; - } - - return val; - } - - public String getUnixGroupFile() { - String val = prop.getProperty(UGSYNC_UNIX_GROUP_FILE); - if ( val == null ) { - val = DEFAULT_UGSYNC_UNIX_GROUP_FILE; - } - - return val; - } - - public String getUnixBackend() { - String val = prop.getProperty(UGSYNC_UNIX_BACKEND); - if ( val == null ) { - val = DEFAULT_UGSYNC_UNIX_BACKEND; - } - - return val; - } - - public boolean isUserSyncEnabled() { - String val = prop.getProperty(UGSYNC_ENABLED_PROP); - return (val != null && val.trim().equalsIgnoreCase("true")); - } - - public String getEnumerateGroups() { - return prop.getProperty(UGSYNC_GROUP_ENUMERATE_GROUPS); - } - - public boolean isGroupEnumerateEnabled() { - String val = prop.getProperty(UGSYNC_GROUP_ENUMERATE_ENABLED); - return (val != null && val.trim().equalsIgnoreCase("true")); - } - - public boolean isMockRunEnabled() { - String val = prop.getProperty(UGSYNC_MOCK_RUN_PROP); - return (val != null && val.trim().equalsIgnoreCase("true")); - } - - public boolean isTestRunEnabled() { - String val = prop.getProperty(UGSYNC_TEST_RUN_PROP); - return (val != null && val.trim().equalsIgnoreCase("true")); - } - - public String getPolicyManagerBaseURL() { - return prop.getProperty(UGSYNC_PM_URL_PROP); - } - - - public String getMinUserId() { - return prop.getProperty(UGSYNC_MIN_USERID_PROP); - } - - public String getMinGroupId() { - String mgid = prop.getProperty(UGSYNC_MIN_GROUPID_PROP); - if (mgid == null) { - mgid = DEFAULT_UGSYNC_MIN_GROUPID; - } - return mgid; - } - - public String getMaxRecordsPerAPICall() { - return prop.getProperty(UGSYNC_MAX_RECORDS_PER_API_CALL_PROP); - } - - public String getSSLKeyStoreType() { - return prop.getProperty(SSL_KEYSTORE_FILE_TYPE_PARAM, KeyStore.getDefaultType()); - } - - public String getSSLTrustStoreType() { - return prop.getProperty(SSL_TRUSTSTORE_FILE_TYPE_PARAM, KeyStore.getDefaultType()); - } - - public String getSSLKeyStorePath() { - return prop.getProperty(SSL_KEYSTORE_PATH_PARAM); - } - - - public String getSSLKeyStorePathPassword() { - if (prop == null) { - return null; - } - if(prop.containsKey(LGSYNC_LDAP_BIND_KEYSTORE)){ - String path=prop.getProperty(LGSYNC_LDAP_BIND_KEYSTORE); - String alias=SSL_KEYSTORE_PATH_PASSWORD_ALIAS; - if(path!=null && alias!=null){ - if(!path.trim().isEmpty() && !alias.trim().isEmpty()){ - if ("bcfks".equalsIgnoreCase(getSSLKeyStoreType())) { - String crendentialProviderPrefixBcfks= "bcfks" + "://file"; - path = crendentialProviderPrefixBcfks + path; - } - String password=CredentialReader.getDecryptedString(path.trim(),alias.trim(), getSSLKeyStoreType()); - if(password!=null&& !password.trim().isEmpty() && !"none".equalsIgnoreCase(password.trim()) && !"_".equalsIgnoreCase(password.trim())){ - prop.setProperty(SSL_KEYSTORE_PATH_PASSWORD_PARAM,password); - } - } - } - } - return prop.getProperty(SSL_KEYSTORE_PATH_PASSWORD_PARAM); - } - - public String getSSLTrustStorePath() { - return prop.getProperty(SSL_TRUSTSTORE_PATH_PARAM); - } - - - public String getSSLTrustStorePathPassword() { - if (prop == null) { - return null; - } - if(prop.containsKey(LGSYNC_LDAP_BIND_KEYSTORE)){ - String path=prop.getProperty(LGSYNC_LDAP_BIND_KEYSTORE); - String alias=SSL_TRUSTSTORE_PATH_PASSWORD_ALIAS; - if(path!=null && alias!=null){ - if(!path.trim().isEmpty() && !alias.trim().isEmpty()){ - if ("bcfks".equalsIgnoreCase(getSSLKeyStoreType())) { - String crendentialProviderPrefixBcfks= "bcfks" + "://file"; - path = crendentialProviderPrefixBcfks + path; - } - String password=CredentialReader.getDecryptedString(path.trim(),alias.trim(), getSSLKeyStoreType()); - if(password!=null&& !password.trim().isEmpty() && !"none".equalsIgnoreCase(password.trim()) && !"_".equalsIgnoreCase(password.trim())){ - prop.setProperty(SSL_TRUSTSTORE_PATH_PASSWORD_PARAM,password); - } - } - } - } - return prop.getProperty(SSL_TRUSTSTORE_PATH_PASSWORD_PARAM); - } - - public long getUpdateMillisMin() { - String val = prop.getProperty(UGSYNC_UPDATE_MILLIS_MIN); - if (val == null) { - return DEFAULT_UGSYNC_UPDATE_MILLIS_MIN; - } - - long ret = Long.parseLong(val); - if (ret < DEFAULT_UGSYNC_UPDATE_MILLIS_MIN) { - return DEFAULT_UGSYNC_UPDATE_MILLIS_MIN; - } - - return ret; - } - - public long getInitSleepTimeInMillisBetweenCycle() throws Throwable{ - long initSleepValue = 0; - Configuration config = getUserGroupConfig(); - if(config.getBoolean(UGSYNC_SERVER_HA_ENABLED_PARAM, false)){ - initSleepValue = UGSYNC_INIT_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE_FOR_HA; - }else{ - initSleepValue = getSleepTimeInMillisBetweenCycle(); - } - return initSleepValue; - } - public long getSleepTimeInMillisBetweenCycle() throws Throwable { - String val = prop.getProperty(UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM); - boolean is_ldap_force = Boolean.parseBoolean(prop.getProperty(UGSYNC_SLEEP_LDAP_FORCE_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM_ENABLED)); - String className = getUserGroupSource().getClass().getName(); - if (val == null) { - if (LGSYNC_SOURCE_CLASS.equals(className)) { - return UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_LDAP_DEFAULT_VALUE; - } else { - return UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_UNIX_DEFAULT_VALUE; - } - } - else { - long ret = Long.parseLong(val); - long min_interval; - if (LGSYNC_SOURCE_CLASS.equals(className)) { - if (is_ldap_force && ret < UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_LDAP_DEFAULT_VALUE) { - min_interval = ret; - LOG.info("If you force the synchronization time of ldap users to be less than the default of 3600s, this setting [" + min_interval + "] millisec will take effect."); - } else { - min_interval = UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_LDAP_DEFAULT_VALUE; - } - }else if(UGSYNC_SOURCE_CLASS.equals(className)){ - min_interval = UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_UNIX_DEFAULT_VALUE; - } else { - min_interval = UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE; - } - if((!isTestRunEnabled()) && (ret < min_interval)) - { - LOG.info("Sleep Time Between Cycle can not be lower than [" + min_interval + "] millisec. resetting to min value."); - ret = min_interval; - } - return ret; - } - } - - private String getUserGroupSourceClassName() { - String val = prop.getProperty(UGSYNC_SOURCE_CLASS_PARAM); - String className = UGSYNC_SOURCE_CLASS; - - String syncSource = null; - - if(val == null || val.trim().isEmpty()) { - syncSource=getSyncSource(); - } - else { - if (val.equalsIgnoreCase(LGSYNC_SOURCE_CLASS)) { - val = LGSYNC_SOURCE_CLASS; - } - syncSource = val; - } - - className = val; - - if(syncSource!=null && syncSource.equalsIgnoreCase("UNIX")){ - className = UGSYNC_SOURCE_CLASS; - }else if(syncSource!=null && syncSource.equalsIgnoreCase("LDAP")){ - className = LGSYNC_SOURCE_CLASS; - } - - return className; - } - - public UserGroupSource getUserGroupSource() throws Throwable { - - String className = getUserGroupSourceClassName(); - - Class ugSourceClass = (Class)Class.forName(className); - - UserGroupSource ret = ugSourceClass.newInstance(); - - return ret; - } - - public UserGroupSink getUserGroupSink() throws Throwable { - String val = prop.getProperty(UGSYNC_SINK_CLASS_PARAM); - - if(val == null || val.trim().isEmpty()) { - val = UGSYNC_SINK_CLASS; - } - - Class ugSinkClass = (Class)Class.forName(val); - - UserGroupSink ret = ugSinkClass.newInstance(); - - return ret; - } - - - public String getLdapUrl() throws Throwable { - String val = prop.getProperty(LGSYNC_LDAP_URL); - if(val == null || val.trim().isEmpty()) { - throw new Exception(LGSYNC_LDAP_URL + " for LdapGroupSync is not specified"); - } - return val; - } - - - public String getLdapBindDn() throws Throwable { - String val = prop.getProperty(LGSYNC_LDAP_BIND_DN); - if(val == null || val.trim().isEmpty()) { - throw new Exception(LGSYNC_LDAP_BIND_DN + " for LdapGroupSync is not specified"); - } - return val; - } - - - public String getLdapBindPassword() { - //update credential from keystore - if (prop == null) { - return null; - } - if(prop.containsKey(LGSYNC_LDAP_BIND_KEYSTORE)){ - String path=prop.getProperty(LGSYNC_LDAP_BIND_KEYSTORE); - String alias=LGSYNC_LDAP_BIND_ALIAS; - if(path!=null && alias!=null){ - if(!path.trim().isEmpty() && !alias.trim().isEmpty()){ - if ("bcfks".equalsIgnoreCase(getSSLKeyStoreType())) { - String crendentialProviderPrefixBcfks= "bcfks" + "://file"; - path = crendentialProviderPrefixBcfks + path; - } - String password=CredentialReader.getDecryptedString(path.trim(),alias.trim(), getSSLKeyStoreType()); - if(password!=null&& !password.trim().isEmpty() && !password.trim().equalsIgnoreCase("none")){ - prop.setProperty(LGSYNC_LDAP_BIND_PASSWORD,password); - } - } - } - } - return prop.getProperty(LGSYNC_LDAP_BIND_PASSWORD); - } - - - public String getLdapAuthenticationMechanism() { - String val = prop.getProperty(LGSYNC_LDAP_AUTHENTICATION_MECHANISM); - if(val == null || val.trim().isEmpty()) { - return DEFAULT_AUTHENTICATION_MECHANISM; - } - return val; - } - - - public String getUserSearchBase() throws Throwable { - String val = prop.getProperty(LGSYNC_USER_SEARCH_BASE); - if(val == null || val.trim().isEmpty()) { - val = getSearchBase(); - } - if(val == null || val.trim().isEmpty()) { - throw new Exception(LGSYNC_USER_SEARCH_BASE + " for LdapGroupSync is not specified"); - } - return val; - } - - - public int getUserSearchScope() { - String val = prop.getProperty(LGSYNC_USER_SEARCH_SCOPE); - if (val == null || val.trim().isEmpty()) { - return 2; //subtree scope - } - - val = val.trim().toLowerCase(); - if (val.equals("0") || val.startsWith("base")) { - return 0; // object scope - } else if (val.equals("1") || val.startsWith("one")) { - return 1; // one level scope - } else { - return 2; // subtree scope - } - } - - - public String getUserObjectClass() { - String val = prop.getProperty(LGSYNC_USER_OBJECT_CLASS); - if (val == null || val.trim().isEmpty()) { - return DEFAULT_USER_OBJECT_CLASS; - } - return val; - } - - public String getUserSearchFilter() { - return prop.getProperty(LGSYNC_USER_SEARCH_FILTER); - } - - - public String getUserNameAttribute() { - String val = prop.getProperty(LGSYNC_USER_NAME_ATTRIBUTE); - if(val == null || val.trim().isEmpty()) { - return DEFAULT_USER_NAME_ATTRIBUTE; - } - return val; - } - - public String getUserGroupNameAttribute() { - String val = prop.getProperty(LGSYNC_USER_GROUP_NAME_ATTRIBUTE); - if(val == null || val.trim().isEmpty()) { - return DEFAULT_USER_GROUP_NAME_ATTRIBUTE; - } - return val; - } + public int getGroupHierarchyLevels() { + int groupHierarchyLevels; + String val = prop.getProperty(LGSYNC_GROUP_HIERARCHY_LEVELS); + if (val == null || val.trim().isEmpty()) { + groupHierarchyLevels = DEFAULT_LGSYNC_GROUP_HIERARCHY_LEVELS; + } else { + groupHierarchyLevels = Integer.parseInt(val); + } + if (groupHierarchyLevels < 0) { + groupHierarchyLevels = DEFAULT_LGSYNC_GROUP_HIERARCHY_LEVELS; + } + return groupHierarchyLevels; + } - public String getGroupNames() { - return prop.getProperty(LGSYNC_GROUPNAMES); + public String getProperty(String aPropertyName) { + return prop.getProperty(aPropertyName); } - public Set getGroupNameSet() { - String groupNames = getGroupNames(); - Set groupNamegSet = new HashSet(); - if (StringUtils.isNotEmpty(groupNames)) { - StringTokenizer st = new StringTokenizer(groupNames, ";"); - while (st.hasMoreTokens()) { - groupNamegSet.add(st.nextToken().trim().toLowerCase()); - } - } - return groupNamegSet; + public String getProperty(String aPropertyName, String aDefaultValue) { + return prop.getProperty(aPropertyName, aDefaultValue); } - public Set getUserGroupNameAttributeSet() { - String uga = getUserGroupNameAttribute(); - StringTokenizer st = new StringTokenizer(uga, ","); - Set userGroupNameAttributeSet = new HashSet(); - while (st.hasMoreTokens()) { - userGroupNameAttributeSet.add(st.nextToken().trim()); - } - return userGroupNameAttributeSet; - } - - public Set getOtherUserAttributes() { - String otherAttributes = prop.getProperty(LGSYNC_OTHER_USER_ATTRIBUTES); - if(otherAttributes == null || otherAttributes.trim().isEmpty()) { - otherAttributes = DEFAULT_OTHER_USER_ATTRIBUTES; - } - StringTokenizer st = new StringTokenizer(otherAttributes, ","); - Set otherUserAttributes = new HashSet(); - while (st.hasMoreTokens()) { - otherUserAttributes.add(st.nextToken().trim()); - } - return otherUserAttributes; - } - - public String getUserCloudIdAttribute() { - String val = prop.getProperty(LGSYNC_USER_CLOUDID_ATTRIBUTE); - if (val == null || val.trim().isEmpty()) { - return DEFAULT_USER_CLOUDID_ATTRIBUTE; - } - return val; - } - - public String getUserCloudIdAttributeDataType() { - String val = prop.getProperty(LGSYNC_USER_CLOUDID_ATTRIBUTE_DATATYPE); - if (val == null || val.trim().isEmpty()) { - return DEFAULT_USER_CLOUDID_ATTRIBUTE_DATATYPE; - } - return val; - } - - public String getOtherUserAttributeDataType(String attrName) { - String attrType = prop.getProperty(LGSYNC_OTHER_USER_ATTRIBUTES + "." + attrName + "datatype"); - if (attrType == null || attrType.isEmpty()) { - attrType = "String"; - } - return attrType.trim(); - } - - public String getUserNameCaseConversion() { - String ret = prop.getProperty(UGSYNC_USERNAME_CASE_CONVERSION_PARAM, DEFAULT_UGSYNC_USERNAME_CASE_CONVERSION_VALUE); - return ret.trim().toLowerCase(); - } - - public String getGroupNameCaseConversion() { - String ret = prop.getProperty(UGSYNC_GROUPNAME_CASE_CONVERSION_PARAM, DEFAULT_UGSYNC_GROUPNAME_CASE_CONVERSION_VALUE); - return ret.trim().toLowerCase(); - } - - public String getSearchBase() { - return prop.getProperty(LGSYNC_SEARCH_BASE); - } - - public boolean isPagedResultsEnabled() { - boolean pagedResultsEnabled; - String val = prop.getProperty(LGSYNC_PAGED_RESULTS_ENABLED); - if(val == null || val.trim().isEmpty()) { - pagedResultsEnabled = DEFAULT_LGSYNC_PAGED_RESULTS_ENABLED; - } else { - pagedResultsEnabled = Boolean.valueOf(val); - } - return pagedResultsEnabled; - } - - public int getPagedResultsSize() { - int pagedResultsSize = DEFAULT_LGSYNC_PAGED_RESULTS_SIZE; - String val = prop.getProperty(LGSYNC_PAGED_RESULTS_SIZE); - if(val == null || val.trim().isEmpty()) { - pagedResultsSize = DEFAULT_LGSYNC_PAGED_RESULTS_SIZE; - } else { - pagedResultsSize = Integer.parseInt(val); - } - if (pagedResultsSize < 1) { - pagedResultsSize = DEFAULT_LGSYNC_PAGED_RESULTS_SIZE; - } - return pagedResultsSize; - } - - public boolean isGroupSearchEnabled() { - boolean groupSearchEnabled; - String val = prop.getProperty(LGSYNC_GROUP_SEARCH_ENABLED); - if(val == null || val.trim().isEmpty()) { - groupSearchEnabled = DEFAULT_LGSYNC_GROUP_SEARCH_ENABLED; - } else { - groupSearchEnabled = Boolean.valueOf(val); - } - return groupSearchEnabled; - } - - public boolean isGroupSearchFirstEnabled() { - boolean groupSearchFirstEnabled; - String val = prop.getProperty(LGSYNC_GROUP_SEARCH_FIRST_ENABLED); - if(val == null || val.trim().isEmpty()) { - groupSearchFirstEnabled = DEFAULT_LGSYNC_GROUP_SEARCH_FIRST_ENABLED; - } else { - groupSearchFirstEnabled = Boolean.valueOf(val); - } - if (isGroupSearchEnabled() == false) { - groupSearchFirstEnabled = false; - } - return groupSearchFirstEnabled; - } - - public boolean isUserSearchEnabled() { - boolean userSearchEnabled; - String val = prop.getProperty(LGSYNC_USER_SEARCH_ENABLED); - if(val == null || val.trim().isEmpty()) { - userSearchEnabled = DEFAULT_LGSYNC_USER_SEARCH_ENABLED; - } else { - userSearchEnabled = Boolean.valueOf(val); - } - if (!isGroupSearchFirstEnabled()) { - userSearchEnabled = true; - } - return userSearchEnabled; - } - - public String getGroupSearchBase() throws Throwable { - String val = prop.getProperty(LGSYNC_GROUP_SEARCH_BASE); - if(val == null || val.trim().isEmpty()) { - val = getSearchBase(); - } - if(val == null || val.trim().isEmpty()) { - val = getUserSearchBase(); - } - return val; - } - - public int getGroupSearchScope() { - String val = prop.getProperty(LGSYNC_GROUP_SEARCH_SCOPE); - if (val == null || val.trim().isEmpty()) { - return 2; //subtree scope - } - - val = val.trim().toLowerCase(); - if (val.equals("0") || val.startsWith("base")) { - return 0; // object scope - } else if (val.equals("1") || val.startsWith("one")) { - return 1; // one level scope - } else { - return 2; // subtree scope - } - } - - public String getGroupObjectClass() { - String val = prop.getProperty(LGSYNC_GROUP_OBJECT_CLASS); - if (val == null || val.trim().isEmpty()) { - return DEFAULT_LGSYNC_GROUP_OBJECT_CLASS; - } - return val; - } - - public String getGroupSearchFilter() { - return prop.getProperty(LGSYNC_GROUP_SEARCH_FILTER); - } - - public String getUserGroupMemberAttributeName() { - String val = prop.getProperty(LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME); - if (val == null || val.trim().isEmpty()) { - return DEFAULT_LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME; - } - return val; - } - - public String getGroupNameAttribute() { - String val = prop.getProperty(LGSYNC_GROUP_NAME_ATTRIBUTE); - if (val == null || val.trim().isEmpty()) { - return DEFAULT_LGSYNC_GROUP_NAME_ATTRIBUTE; - } - return val; - } - - public String getGroupCloudIdAttribute() { - String val = prop.getProperty(LGSYNC_GROUP_CLOUDID_ATTRIBUTE); - if (val == null || val.trim().isEmpty()) { - return DEFAULT_GROUP_CLOUDID_ATTRIBUTE; - } - return val; - } - - public String getGroupCloudIdAttributeDataType() { - String val = prop.getProperty(LGSYNC_GROUP_CLOUDID_ATTRIBUTE_DATATYPE); - if (val == null || val.trim().isEmpty()) { - return DEFAULT_GROUP_CLOUDID_ATTRIBUTE_DATATYPE; - } - return val; - } - - public Set getOtherGroupAttributes() { - String otherAttributes = prop.getProperty(LGSYNC_OTHER_GROUP_ATTRIBUTES); - if(otherAttributes == null || otherAttributes.trim().isEmpty()) { - otherAttributes = DEFAULT_OTHER_GROUP_ATTRIBUTES; - } - StringTokenizer st = new StringTokenizer(otherAttributes, ","); - Set otherGroupAttributes = new HashSet(); - while (st.hasMoreTokens()) { - otherGroupAttributes.add(st.nextToken().trim()); - } - return otherGroupAttributes; - } - - public String getOtherGroupAttributeDataType(String attrName) { - String attrType = prop.getProperty(LGSYNC_OTHER_GROUP_ATTRIBUTES + "." + attrName + "datatype"); - if (attrType == null || attrType.isEmpty()) { - attrType = "String"; + public String getPolicyMgrPassword() { + //update credential from keystore + String password = null; + if (prop != null && prop.containsKey(SYNC_POLICY_MGR_KEYSTORE)) { + password = prop.getProperty(SYNC_POLICY_MGR_PASSWORD); + if (password != null && !password.isEmpty()) { + return password; + } + } + if (prop != null && prop.containsKey(SYNC_POLICY_MGR_KEYSTORE) && prop.containsKey(SYNC_POLICY_MGR_ALIAS)) { + String path = prop.getProperty(SYNC_POLICY_MGR_KEYSTORE); + String alias = prop.getProperty(SYNC_POLICY_MGR_ALIAS, "policymgr.user.password"); + if (path != null && alias != null) { + if (!path.trim().isEmpty() && !alias.trim().isEmpty()) { + if ("bcfks".equalsIgnoreCase(getSSLKeyStoreType())) { + String crendentialProviderPrefixBcfks = "bcfks" + "://file"; + path = crendentialProviderPrefixBcfks + path; + } + try { + password = CredentialReader.getDecryptedString(path.trim(), alias.trim(), getSSLKeyStoreType()); + } catch (Exception ex) { + password = null; + } + if (password != null && !password.trim().isEmpty() && !password.trim().equalsIgnoreCase("none")) { + prop.setProperty(SYNC_POLICY_MGR_PASSWORD, password); + return password; + } } - return attrType.trim(); - } - - public int getGroupHierarchyLevels() { - int groupHierarchyLevels; - String val = prop.getProperty(LGSYNC_GROUP_HIERARCHY_LEVELS); - if(val == null || val.trim().isEmpty()) { - groupHierarchyLevels = DEFAULT_LGSYNC_GROUP_HIERARCHY_LEVELS; - } else { - groupHierarchyLevels = Integer.parseInt(val); - } - if (groupHierarchyLevels < 0) { - groupHierarchyLevels = DEFAULT_LGSYNC_GROUP_HIERARCHY_LEVELS; - } - return groupHierarchyLevels; - } - - public String getProperty(String aPropertyName) { - return prop.getProperty(aPropertyName); - } - - public String getProperty(String aPropertyName, String aDefaultValue) { - return prop.getProperty(aPropertyName, aDefaultValue); - } - - public String getPolicyMgrPassword(){ - //update credential from keystore - String password=null; - if(prop!=null && prop.containsKey(SYNC_POLICY_MGR_KEYSTORE)){ - password=prop.getProperty(SYNC_POLICY_MGR_PASSWORD); - if(password!=null && !password.isEmpty()){ - return password; - } - } - if(prop!=null && prop.containsKey(SYNC_POLICY_MGR_KEYSTORE) && prop.containsKey(SYNC_POLICY_MGR_ALIAS)){ - String path=prop.getProperty(SYNC_POLICY_MGR_KEYSTORE); - String alias=prop.getProperty(SYNC_POLICY_MGR_ALIAS,"policymgr.user.password"); - if(path!=null && alias!=null){ - if(!path.trim().isEmpty() && !alias.trim().isEmpty()){ - if ("bcfks".equalsIgnoreCase(getSSLKeyStoreType())) { - String crendentialProviderPrefixBcfks= "bcfks" + "://file"; - path = crendentialProviderPrefixBcfks + path; - } - try{ - password=CredentialReader.getDecryptedString(path.trim(),alias.trim(), getSSLKeyStoreType()); - }catch(Exception ex){ - password=null; - } - if(password!=null&& !password.trim().isEmpty() && !password.trim().equalsIgnoreCase("none")){ - prop.setProperty(SYNC_POLICY_MGR_PASSWORD,password); - return password; - } - } - } - } - return null; - } - - public String getPolicyMgrUserName() { - String userName=null; - if(prop!=null && prop.containsKey(SYNC_POLICY_MGR_USERNAME)){ - userName=prop.getProperty(SYNC_POLICY_MGR_USERNAME); - } - if (userName == null || userName.isEmpty()) { - userName = DEFAULT_POLICYMGR_USERNAME; - } - return userName; - } - - public int getPolicyMgrMaxRetryAttempts() { - return getIntProperty(prop, SYNC_POLICY_MGR_MAX_RETRY_ATTEMPTS, 0); - } - - public int getPolicyMgrRetryIntervalMs() { - return getIntProperty(prop, SYNC_POLICY_MGR_RETRY_INTERVAL_MS, 1 * 1000); - } - - public String getSyncSource() { - String syncSource=null; - if(prop!=null && prop.containsKey(SYNC_SOURCE)){ - syncSource=prop.getProperty(SYNC_SOURCE); - if(syncSource==null||syncSource.trim().isEmpty()){ - syncSource=null; - }else{ - syncSource=syncSource.trim(); - } - } - return syncSource; - } - public String getContextReferral() { - String referral="ignore"; - if(prop!=null && prop.containsKey(LGSYNC_REFERRAL)){ - referral=prop.getProperty(LGSYNC_REFERRAL); - if(referral==null||referral.trim().isEmpty()){ - referral=DEFAULT_LGSYNC_REFERRAL; - }else{ - referral=referral.trim().toLowerCase(); - } - } - return referral; - } - - public List getAllRegexPatterns(String baseProperty) throws Throwable { - List regexPatterns = new ArrayList(); - if (prop != null) { - String baseRegex = prop.getProperty(baseProperty); - if (baseRegex == null) { - return regexPatterns; - } - regexPatterns.add(baseRegex); - int i = 1; - String nextRegex = prop.getProperty(baseProperty + "." + i); - while (nextRegex != null) { - regexPatterns.add(nextRegex); - i++; - nextRegex = prop.getProperty(baseProperty + "." + i); - } - - } - return regexPatterns; - } - - public String getUserSyncMappingUserNameHandler() { - String val = prop.getProperty(SYNC_MAPPING_USERNAME_HANDLER); - - if(val == null) { - val = DEFAULT_SYNC_MAPPING_USERNAME_HANDLER; - } - return val; - } - - public String getUserSyncMappingGroupNameHandler() { - String val = prop.getProperty(SYNC_MAPPING_GROUPNAME_HANDLER); - - if(val == null) { - val = DEFAULT_SYNC_MAPPING_GROUPNAME_HANDLER; - } - return val; - } + } + } + return null; + } + + public String getPolicyMgrUserName() { + String userName = null; + if (prop != null && prop.containsKey(SYNC_POLICY_MGR_USERNAME)) { + userName = prop.getProperty(SYNC_POLICY_MGR_USERNAME); + } + if (userName == null || userName.isEmpty()) { + userName = DEFAULT_POLICYMGR_USERNAME; + } + return userName; + } + + public int getPolicyMgrMaxRetryAttempts() { + return getIntProperty(prop, SYNC_POLICY_MGR_MAX_RETRY_ATTEMPTS, 0); + } + + public int getPolicyMgrRetryIntervalMs() { + return getIntProperty(prop, SYNC_POLICY_MGR_RETRY_INTERVAL_MS, 1_000); + } + + public String getSyncSource() { + String syncSource = null; + if (prop != null && prop.containsKey(SYNC_SOURCE)) { + syncSource = prop.getProperty(SYNC_SOURCE); + if (syncSource == null || syncSource.trim().isEmpty()) { + syncSource = null; + } else { + syncSource = syncSource.trim(); + } + } + return syncSource; + } + + public String getContextReferral() { + String referral = "ignore"; + if (prop != null && prop.containsKey(LGSYNC_REFERRAL)) { + referral = prop.getProperty(LGSYNC_REFERRAL); + if (referral == null || referral.trim().isEmpty()) { + referral = DEFAULT_LGSYNC_REFERRAL; + } else { + referral = referral.trim().toLowerCase(); + } + } + return referral; + } + + public List getAllRegexPatterns(String baseProperty) throws Throwable { + List regexPatterns = new ArrayList(); + if (prop != null) { + String baseRegex = prop.getProperty(baseProperty); + if (baseRegex == null) { + return regexPatterns; + } + regexPatterns.add(baseRegex); + int i = 1; + String nextRegex = prop.getProperty(baseProperty + "." + i); + while (nextRegex != null) { + regexPatterns.add(nextRegex); + i++; + nextRegex = prop.getProperty(baseProperty + "." + i); + } + } + return regexPatterns; + } + + public String getUserSyncMappingUserNameHandler() { + String val = prop.getProperty(SYNC_MAPPING_USERNAME_HANDLER); + + if (val == null) { + val = DEFAULT_SYNC_MAPPING_USERNAME_HANDLER; + } + return val; + } + + public String getUserSyncMappingGroupNameHandler() { + String val = prop.getProperty(SYNC_MAPPING_GROUPNAME_HANDLER); + + if (val == null) { + val = DEFAULT_SYNC_MAPPING_GROUPNAME_HANDLER; + } + return val; + } public String getGroupRoleRules() { if (prop != null && prop.containsKey(GROUP_BASED_ROLE_ASSIGNMENT_RULES)) { - String GroupRoleRules = prop - .getProperty(GROUP_BASED_ROLE_ASSIGNMENT_RULES); - if (StringUtils.isNotBlank(GroupRoleRules)) { - return GroupRoleRules.trim(); + String groupRoleRules = prop.getProperty(GROUP_BASED_ROLE_ASSIGNMENT_RULES); + if (StringUtils.isNotBlank(groupRoleRules)) { + return groupRoleRules.trim(); } } return null; } - public String getWhileListUserRoleRules() { - if (prop != null && prop.containsKey(WHITELIST_USER_ROLE_ASSIGNMENT_RULES)) { - String whiteListUserRoleRules = prop - .getProperty(WHITELIST_USER_ROLE_ASSIGNMENT_RULES); - if (StringUtils.isNotBlank(whiteListUserRoleRules) ) { - return whiteListUserRoleRules.trim(); - } - } - return DEFAULT_WHITELIST_USER_ROLE_ASSIGNMENT_RULES; - } + public String getWhileListUserRoleRules() { + if (prop != null && prop.containsKey(WHITELIST_USER_ROLE_ASSIGNMENT_RULES)) { + String whiteListUserRoleRules = prop.getProperty(WHITELIST_USER_ROLE_ASSIGNMENT_RULES); + if (StringUtils.isNotBlank(whiteListUserRoleRules)) { + return whiteListUserRoleRules.trim(); + } + } + return DEFAULT_WHITELIST_USER_ROLE_ASSIGNMENT_RULES; + } public String getUserGroupDelimiter() { - if (prop != null - && prop.containsKey(USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER)) { - String UserGroupDelimiter = prop - .getProperty(USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER); - if (UserGroupDelimiter != null && !UserGroupDelimiter.isEmpty()) { - return UserGroupDelimiter; + if (prop != null && prop.containsKey(USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER)) { + String userGroupDelimiter = prop.getProperty(USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER); + if (userGroupDelimiter != null && !userGroupDelimiter.isEmpty()) { + return userGroupDelimiter; } } return null; } public String getUserGroupNameDelimiter() { - if (prop != null - && prop.containsKey(USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER)) { - String UserGroupNameDelimiter = prop - .getProperty(USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER); - if (UserGroupNameDelimiter != null - && !UserGroupNameDelimiter.isEmpty()) { - return UserGroupNameDelimiter; + if (prop != null && prop.containsKey(USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER)) { + String userGroupNameDelimiter = prop.getProperty(USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER); + if (userGroupNameDelimiter != null && !userGroupNameDelimiter.isEmpty()) { + return userGroupNameDelimiter; } } return null; } - public boolean isUserSyncRangerCookieEnabled() { - String val = prop.getProperty(USERSYNC_RANGER_COOKIE_ENABLED_PROP); - return val == null || Boolean.valueOf(val.trim()); - } + public boolean isUserSyncRangerCookieEnabled() { + String val = prop.getProperty(USERSYNC_RANGER_COOKIE_ENABLED_PROP); + return val == null || Boolean.valueOf(val.trim()); + } - public String getRangerAdminCookieName() { - String ret = RangerCommonConstants.DEFAULT_COOKIE_NAME; - String val = prop.getProperty(RANGER_ADMIN_COOKIE_NAME_PROPS); - if (StringUtils.isNotBlank(val)) { - ret = val; - } - return ret; - } + public String getRangerAdminCookieName() { + String ret = RangerCommonConstants.DEFAULT_COOKIE_NAME; + String val = prop.getProperty(RANGER_ADMIN_COOKIE_NAME_PROPS); + if (StringUtils.isNotBlank(val)) { + ret = val; + } + return ret; + } public String getRoleDelimiter() { if (prop != null && prop.containsKey(ROLE_ASSIGNMENT_LIST_DELIMITER)) { @@ -1156,253 +1059,225 @@ public String getRoleDelimiter() { } return null; } - public boolean isStartTlsEnabled() { - boolean starttlsEnabled; - String val = prop.getProperty(LGSYNC_LDAP_STARTTLS_ENABLED); - if(val == null || val.trim().isEmpty()) { - starttlsEnabled = DEFAULT_LGSYNC_LDAP_STARTTLS_ENABLED; - } else { - starttlsEnabled = Boolean.valueOf(val); - } - return starttlsEnabled; - } - - public boolean isDeltaSyncEnabled() { - boolean deltaSyncEnabled; - String val = prop.getProperty(LGSYNC_LDAP_DELTASYNC_ENABLED); - if(val == null || val.trim().isEmpty()) { - deltaSyncEnabled = DEFAULT_LGSYNC_LDAP_DELTASYNC_ENABLED; - } else { - deltaSyncEnabled = Boolean.valueOf(val); - } - return deltaSyncEnabled; - } - - /* Used only for unit testing */ - public void setUserSearchFilter(String filter) { - prop.setProperty(LGSYNC_USER_SEARCH_FILTER, filter); - } - - /* Used only for unit testing */ - public void setGroupSearchFilter(String filter) { - prop.setProperty(LGSYNC_GROUP_SEARCH_FILTER, filter); - } - - /* Used only for unit testing */ - public void setGroupSearchEnabled(boolean groupSearchEnabled) { - prop.setProperty(LGSYNC_GROUP_SEARCH_ENABLED, String.valueOf(groupSearchEnabled)); - } - - /* Used only for unit testing */ - public void setPagedResultsEnabled(boolean pagedResultsEnabled) { - prop.setProperty(LGSYNC_PAGED_RESULTS_ENABLED, String.valueOf(pagedResultsEnabled)); - } - - /* Used only for unit testing */ - public void setProperty(String name, String value) { - prop.setProperty(name, value); - } - - /* Used only for unit testing */ - public void setUserSearchBase(String userSearchBase) throws Throwable { - prop.setProperty(LGSYNC_USER_SEARCH_BASE, userSearchBase); - } - - /* Used only for unit testing */ - public void setGroupSearchBase(String groupSearchBase) throws Throwable { - prop.setProperty(LGSYNC_GROUP_SEARCH_BASE, groupSearchBase); - } - - /* Used only for unit testing */ - public void setGroupSearchFirstEnabled(boolean groupSearchFirstEnabled) { - prop.setProperty(LGSYNC_GROUP_SEARCH_FIRST_ENABLED, String.valueOf(groupSearchFirstEnabled)); - } - - /* Used only for unit testing */ - public void setUserSearchEnabled(boolean userSearchEnabled) { - prop.setProperty(LGSYNC_USER_SEARCH_ENABLED, String.valueOf(userSearchEnabled)); - } - - /* Used only for unit testing */ - public void setUserGroupMemberAttributeName(String groupMemberAttrName) { - prop.setProperty(LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME, groupMemberAttrName); - } - - /* Used only for unit testing */ - public void setUserObjectClass(String userObjectClass) { - prop.setProperty(LGSYNC_USER_OBJECT_CLASS, userObjectClass); - } - - /* Used only for unit testing */ - public void setGroupObjectClass(String groupObjectClass) { - prop.setProperty(LGSYNC_GROUP_OBJECT_CLASS, groupObjectClass); - } - - /* Used only for unit testing */ - public void setDeltaSync(boolean deltaSyncEnabled) { - prop.setProperty(LGSYNC_LDAP_DELTASYNC_ENABLED, String.valueOf(deltaSyncEnabled)); - } - - /* Used only for unit testing */ - public void setUserNameAttribute(String userNameAttr) { - prop.setProperty(LGSYNC_USER_NAME_ATTRIBUTE, userNameAttr); - } - - /* Used only for unit testing */ - public void setGroupHierarchyLevel(int groupHierarchyLevel) { - prop.setProperty(LGSYNC_GROUP_HIERARCHY_LEVELS, String.valueOf(groupHierarchyLevel)); - } - - /* Used only for unit testing */ - public void setGroupnames(String groupnames) { - prop.setProperty(LGSYNC_GROUPNAMES, groupnames); - } - - public String getUserSyncMetricsFileName() throws IOException { - String val = prop.getProperty(UGSYNC_METRICS_FILEPATH); - if (StringUtils.isBlank(val)) { - if (StringUtils.isBlank(prop.getProperty("ranger.usersync.logdir"))) { - if (StringUtils.isBlank(System.getProperty("logdir"))) { - val = DEFAULT_UGSYNC_METRICS_FILEPATH; - } else { - val = System.getProperty("logdir"); - } - } else { - val = prop.getProperty("ranger.usersync.logdir"); - } - } - - if (Files.notExists(Paths.get(val))) { - String current = new File(".").getCanonicalPath(); - val = current + "/" + val; - if (Files.notExists(Paths.get(val))) { - return null; - } - } - - StringBuilder pathAndFileName = new StringBuilder(val); - if (!val.endsWith("/")) { - pathAndFileName.append("/"); - } - - String fileName = prop.getProperty(UGSYNC_METRICS_FILENAME); - if (StringUtils.isBlank(fileName)) { - fileName = DEFAULT_UGSYNC_METRICS_FILENAME; - } - pathAndFileName.append(fileName); - return pathAndFileName.toString(); - } - - public long getUserSyncMetricsFrequency() { - long ret = DEFAULT_UGSYNC_METRICS_FREQUENCY_TIME_IN_MILLIS; - String val = prop.getProperty(UGSYNC_METRICS_FREQUENCY_TIME_IN_MILLIS_PARAM); - if (StringUtils.isNotBlank(val)) { - try { - ret = Long.valueOf(val); - } catch (NumberFormatException exception) { - // Ignore - } - } - return ret; - } - - public boolean isUserSyncMetricsEnabled() { - String val = prop.getProperty(UGSYNC_METRICS_ENABLED_PROP); - return "true".equalsIgnoreCase(StringUtils.trimToEmpty(val)); - } - - - public boolean isUserSyncDeletesEnabled() { - boolean isUserSyncDeletesEnabled; - String val = prop.getProperty(UGSYNC_DELETES_ENABLED); - if(StringUtils.isEmpty(val)) { - isUserSyncDeletesEnabled = DEFAULT_UGSYNC_DELETES_ENABLED; - } else { - isUserSyncDeletesEnabled = Boolean.valueOf(val); - } - return isUserSyncDeletesEnabled; - } - - /* - * This is the frequency of computing deleted users/groups from the sync source. - * Default and minimum value is 8hrs - * If the delete frequency interval value is less than sync interval and greater than 8hrs, - * then deleted objects are computed at every sync cycle. - */ - public long getUserSyncDeletesFrequency() throws Throwable { - long ret = 1; - - String val = prop.getProperty(UGSYNC_DELETES_FREQUENCY); - if (StringUtils.isNotBlank(val)) { - ret = Long.valueOf(val); - if (!isTestRunEnabled() && ret < DEFAULT_UGSYNC_DELETES_FREQUENCY) { - LOG.info("Frequency of computing deletes cannot be set below " + DEFAULT_UGSYNC_DELETES_FREQUENCY); - ret = DEFAULT_UGSYNC_DELETES_FREQUENCY; - } - } - return ret; - } - - public String getCurrentSyncSource() throws Throwable{ - String currentSyncSource; - String className = getUserGroupSource().getClass().getName(); - if (LGSYNC_SOURCE_CLASS.equals(className)) { - currentSyncSource = "LDAP/AD"; - } else if (UGSYNC_SOURCE_CLASS.equalsIgnoreCase(className)){ - currentSyncSource = "Unix"; - } else { - currentSyncSource = "File"; - } - return currentSyncSource; - } - - public boolean isUserSyncNameValidationEnabled() { - boolean isUserSyncNameValidationEnabled; - String val = prop.getProperty(UGSYNC_NAME_VALIDATION_ENABLED); - if(StringUtils.isEmpty(val)) { - isUserSyncNameValidationEnabled = DEFAULT_UGSYNC_NAME_VALIDATION_ENABLED; - } else { - isUserSyncNameValidationEnabled = Boolean.valueOf(val); - } - return isUserSyncNameValidationEnabled; - } - - public String getRegexSeparator() { - String ret = DEFAULT_MAPPING_SEPARATOR; - String val = prop.getProperty(SYNC_MAPPING_SEPARATOR); - if(StringUtils.isNotEmpty(val)) { - if (val.length() == 1) { - ret = val; - } else { - LOG.warn("More than one character found in RegEx Separator, using default RegEx Separator /"); - } - } - LOG.info(String.format("Using %s as the RegEx Separator", ret)); - return ret; - } - - - private int getIntProperty(Properties prop, String key, int defaultValue) { - int ret = defaultValue; - String val = prop.getProperty(key); - - if (StringUtils.isNotBlank(val)) { - try { - ret = Integer.parseInt(val); - } catch (NumberFormatException excp) { - LOG.warn("Invalid value for property: " + key + "=" + val + ". Will use default value: " + defaultValue, excp); - } - } - - return ret; - } - - public boolean isSyncSourceValidationEnabled() { - boolean isSyncSourceValidationEnabled = DEFAULT_UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED; - String val = prop.getProperty(UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED); - if(StringUtils.isNotEmpty(val)) { - isSyncSourceValidationEnabled = Boolean.parseBoolean(val); - } - return isSyncSourceValidationEnabled; - } + + public boolean isStartTlsEnabled() { + boolean starttlsEnabled; + String val = prop.getProperty(LGSYNC_LDAP_STARTTLS_ENABLED); + if (val == null || val.trim().isEmpty()) { + starttlsEnabled = DEFAULT_LGSYNC_LDAP_STARTTLS_ENABLED; + } else { + starttlsEnabled = Boolean.valueOf(val); + } + return starttlsEnabled; + } + + public boolean isDeltaSyncEnabled() { + boolean deltaSyncEnabled; + String val = prop.getProperty(LGSYNC_LDAP_DELTASYNC_ENABLED); + if (val == null || val.trim().isEmpty()) { + deltaSyncEnabled = DEFAULT_LGSYNC_LDAP_DELTASYNC_ENABLED; + } else { + deltaSyncEnabled = Boolean.valueOf(val); + } + return deltaSyncEnabled; + } + + /* Used only for unit testing */ + public void setProperty(String name, String value) { + prop.setProperty(name, value); + } + + /* Used only for unit testing */ + public void setDeltaSync(boolean deltaSyncEnabled) { + prop.setProperty(LGSYNC_LDAP_DELTASYNC_ENABLED, String.valueOf(deltaSyncEnabled)); + } + + /* Used only for unit testing */ + public void setGroupHierarchyLevel(int groupHierarchyLevel) { + prop.setProperty(LGSYNC_GROUP_HIERARCHY_LEVELS, String.valueOf(groupHierarchyLevel)); + } + + /* Used only for unit testing */ + public void setGroupnames(String groupnames) { + prop.setProperty(LGSYNC_GROUPNAMES, groupnames); + } + + public String getUserSyncMetricsFileName() throws IOException { + String val = prop.getProperty(UGSYNC_METRICS_FILEPATH); + if (StringUtils.isBlank(val)) { + if (StringUtils.isBlank(prop.getProperty("ranger.usersync.logdir"))) { + if (StringUtils.isBlank(System.getProperty("logdir"))) { + val = DEFAULT_UGSYNC_METRICS_FILEPATH; + } else { + val = System.getProperty("logdir"); + } + } else { + val = prop.getProperty("ranger.usersync.logdir"); + } + } + + if (Files.notExists(Paths.get(val))) { + String current = new File(".").getCanonicalPath(); + val = current + "/" + val; + if (Files.notExists(Paths.get(val))) { + return null; + } + } + + StringBuilder pathAndFileName = new StringBuilder(val); + if (!val.endsWith("/")) { + pathAndFileName.append("/"); + } + + String fileName = prop.getProperty(UGSYNC_METRICS_FILENAME); + if (StringUtils.isBlank(fileName)) { + fileName = DEFAULT_UGSYNC_METRICS_FILENAME; + } + pathAndFileName.append(fileName); + return pathAndFileName.toString(); + } + + public long getUserSyncMetricsFrequency() { + long ret = DEFAULT_UGSYNC_METRICS_FREQUENCY_TIME_IN_MILLIS; + String val = prop.getProperty(UGSYNC_METRICS_FREQUENCY_TIME_IN_MILLIS_PARAM); + if (StringUtils.isNotBlank(val)) { + try { + ret = Long.valueOf(val); + } catch (NumberFormatException exception) { + // Ignore + } + } + return ret; + } + + public boolean isUserSyncMetricsEnabled() { + String val = prop.getProperty(UGSYNC_METRICS_ENABLED_PROP); + return "true".equalsIgnoreCase(StringUtils.trimToEmpty(val)); + } + + public boolean isUserSyncDeletesEnabled() { + boolean isUserSyncDeletesEnabled; + String val = prop.getProperty(UGSYNC_DELETES_ENABLED); + if (StringUtils.isEmpty(val)) { + isUserSyncDeletesEnabled = DEFAULT_UGSYNC_DELETES_ENABLED; + } else { + isUserSyncDeletesEnabled = Boolean.valueOf(val); + } + return isUserSyncDeletesEnabled; + } + + /* + * This is the frequency of computing deleted users/groups from the sync source. + * Default and minimum value is 8hrs + * If the delete frequency interval value is less than sync interval and greater than 8hrs, + * then deleted objects are computed at every sync cycle. + */ + public long getUserSyncDeletesFrequency() throws Throwable { + long ret = 1; + + String val = prop.getProperty(UGSYNC_DELETES_FREQUENCY); + if (StringUtils.isNotBlank(val)) { + ret = Long.valueOf(val); + if (!isTestRunEnabled() && ret < DEFAULT_UGSYNC_DELETES_FREQUENCY) { + LOG.info("Frequency of computing deletes cannot be set below " + DEFAULT_UGSYNC_DELETES_FREQUENCY); + ret = DEFAULT_UGSYNC_DELETES_FREQUENCY; + } + } + return ret; + } + + public String getCurrentSyncSource() throws Throwable { + String currentSyncSource; + String className = getUserGroupSource().getClass().getName(); + if (LGSYNC_SOURCE_CLASS.equals(className)) { + currentSyncSource = "LDAP/AD"; + } else if (UGSYNC_SOURCE_CLASS.equalsIgnoreCase(className)) { + currentSyncSource = "Unix"; + } else { + currentSyncSource = "File"; + } + return currentSyncSource; + } + + public boolean isUserSyncNameValidationEnabled() { + boolean isUserSyncNameValidationEnabled; + String val = prop.getProperty(UGSYNC_NAME_VALIDATION_ENABLED); + if (StringUtils.isEmpty(val)) { + isUserSyncNameValidationEnabled = DEFAULT_UGSYNC_NAME_VALIDATION_ENABLED; + } else { + isUserSyncNameValidationEnabled = Boolean.valueOf(val); + } + return isUserSyncNameValidationEnabled; + } + + public String getRegexSeparator() { + String ret = DEFAULT_MAPPING_SEPARATOR; + String val = prop.getProperty(SYNC_MAPPING_SEPARATOR); + if (StringUtils.isNotEmpty(val)) { + if (val.length() == 1) { + ret = val; + } else { + LOG.warn("More than one character found in RegEx Separator, using default RegEx Separator /"); + } + } + LOG.info(String.format("Using %s as the RegEx Separator", ret)); + return ret; + } + + public boolean isSyncSourceValidationEnabled() { + boolean isSyncSourceValidationEnabled = DEFAULT_UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED; + String val = prop.getProperty(UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED); + if (StringUtils.isNotEmpty(val)) { + isSyncSourceValidationEnabled = Boolean.parseBoolean(val); + } + return isSyncSourceValidationEnabled; + } + + private void init() { + XMLUtils.loadConfig(DEFAULT_CONFIG_FILE, prop); + XMLUtils.loadConfig(CORE_SITE_CONFIG_FILE, prop); + XMLUtils.loadConfig(CONFIG_FILE, prop); + userGroupConfig = getConfig(); + } + + private String getUserGroupSourceClassName() { + String val = prop.getProperty(UGSYNC_SOURCE_CLASS_PARAM); + String className = UGSYNC_SOURCE_CLASS; + + String syncSource = null; + + if (val == null || val.trim().isEmpty()) { + syncSource = getSyncSource(); + } else { + if (val.equalsIgnoreCase(LGSYNC_SOURCE_CLASS)) { + val = LGSYNC_SOURCE_CLASS; + } + syncSource = val; + } + + className = val; + + if (syncSource != null && syncSource.equalsIgnoreCase("UNIX")) { + className = UGSYNC_SOURCE_CLASS; + } else if (syncSource != null && syncSource.equalsIgnoreCase("LDAP")) { + className = LGSYNC_SOURCE_CLASS; + } + + return className; + } + + private int getIntProperty(Properties prop, String key, int defaultValue) { + int ret = defaultValue; + String val = prop.getProperty(key); + + if (StringUtils.isNotBlank(val)) { + try { + ret = Integer.parseInt(val); + } catch (NumberFormatException excp) { + LOG.warn("Invalid value for property: " + key + "=" + val + ". Will use default value: " + defaultValue, excp); + } + } + + return ret; + } } diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/ha/UserSyncHAInitializerImpl.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/ha/UserSyncHAInitializerImpl.java index 8e892c44af..fb56cc7ec1 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/ha/UserSyncHAInitializerImpl.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/ha/UserSyncHAInitializerImpl.java @@ -19,11 +19,6 @@ package org.apache.ranger.unixusersync.ha; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - import org.apache.hadoop.conf.Configuration; import org.apache.ranger.RangerHAInitializer; import org.apache.ranger.ha.ActiveInstanceElectorService; @@ -35,36 +30,51 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + public class UserSyncHAInitializerImpl extends RangerHAInitializer { private static final Logger LOG = LoggerFactory.getLogger(UserSyncHAInitializerImpl.class); - ActiveInstanceElectorService activeInstanceElectorService = null; - ActiveStateChangeHandler activeStateChangeHandler = null; - List haRangerService = null; - ServiceManager serviceManager = null; - private static UserSyncHAInitializerImpl theInstance = null; + private static UserSyncHAInitializerImpl theInstance; + ActiveInstanceElectorService activeInstanceElectorService; + ActiveStateChangeHandler activeStateChangeHandler; + List haRangerService; + ServiceManager serviceManager; private UserSyncHAInitializerImpl(Configuration configuration) { if (LOG.isDebugEnabled()) { LOG.debug("==> UserSyncHAInitializerImpl.UserSyncHAInitializerImpl()"); } try { - LOG.info("Ranger UserSync server is HA enabled : "+configuration.getBoolean(UserGroupSyncConfig.UGSYNC_SERVER_HA_ENABLED_PARAM, false) ); + LOG.info("Ranger UserSync server is HA enabled : " + configuration.getBoolean(UserGroupSyncConfig.UGSYNC_SERVER_HA_ENABLED_PARAM, false)); init(configuration); } catch (Exception e) { - LOG.error("UserSyncHAInitializerImpl initialization failed", e.getMessage()); + LOG.error("UserSyncHAInitializerImpl initialization failed {}", e.getMessage()); } if (LOG.isDebugEnabled()) { LOG.info("<== UserSyncHAInitializerImpl.UserSyncHAInitializerImpl()"); } } + public static UserSyncHAInitializerImpl getInstance(Configuration configuration) { + if (theInstance == null) { + synchronized (UserSyncHAInitializerImpl.class) { + if (theInstance == null) { + theInstance = new UserSyncHAInitializerImpl(configuration); + } + } + } + return theInstance; + } + public void init(Configuration configuration) throws Exception { super.init(configuration); LOG.info("==> UserSyncHAInitializerImpl.init() initialization started "); Set activeStateChangeHandlerProviders = new HashSet(); - activeInstanceElectorService = new ActiveInstanceElectorService(activeStateChangeHandlerProviders, - curatorFactory, activeInstanceState, serviceState, configuration); + activeInstanceElectorService = new ActiveInstanceElectorService(activeStateChangeHandlerProviders, curatorFactory, activeInstanceState, serviceState, configuration); haRangerService = new ArrayList(); haRangerService.add(activeInstanceElectorService); @@ -88,16 +98,6 @@ public void stop() { } } - public static UserSyncHAInitializerImpl getInstance(Configuration configuration) { - if(theInstance == null){ - synchronized(UserSyncHAInitializerImpl.class){ - if(theInstance == null){ - theInstance = new UserSyncHAInitializerImpl(configuration); - } - } - } - return theInstance; - } public boolean isActive() { try { // To let the curator thread a chance to run and set the active state if needed diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/model/GetXGroupListResponse.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/model/GetXGroupListResponse.java index 20ebaf39e9..991a48c927 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/model/GetXGroupListResponse.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/model/GetXGroupListResponse.java @@ -17,35 +17,33 @@ * under the License. */ - package org.apache.ranger.unixusersync.model; - -import java.util.List; +package org.apache.ranger.unixusersync.model; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import org.apache.ranger.ugsyncutil.model.XGroupInfo; +import java.util.List; + @JsonSerialize public class GetXGroupListResponse { - private int totalCount; - - @JsonProperty("vXGroups") - List xgroupInfoList; - - public int getTotalCount() { - return totalCount; - } + @JsonProperty("vXGroups") + List xgroupInfoList; + private int totalCount; - public void setTotalCount(int totalCount) { - this.totalCount = totalCount; - } + public int getTotalCount() { + return totalCount; + } - public List getXgroupInfoList() { - return xgroupInfoList; - } + public void setTotalCount(int totalCount) { + this.totalCount = totalCount; + } - public void setXgroupInfoList(List xgroupInfoList) { - this.xgroupInfoList = xgroupInfoList; - } + public List getXgroupInfoList() { + return xgroupInfoList; + } + public void setXgroupInfoList(List xgroupInfoList) { + this.xgroupInfoList = xgroupInfoList; + } } diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/model/GetXUserListResponse.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/model/GetXUserListResponse.java index 2c4bfe4417..4037c76fd0 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/model/GetXUserListResponse.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/model/GetXUserListResponse.java @@ -17,38 +17,33 @@ * under the License. */ - package org.apache.ranger.unixusersync.model; - -import java.util.List; +package org.apache.ranger.unixusersync.model; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import org.apache.ranger.ugsyncutil.model.XUserInfo; +import java.util.List; + @JsonSerialize public class GetXUserListResponse { + @JsonProperty("vXUsers") + List xuserInfoList; + private int totalCount; - private int totalCount; - - @JsonProperty("vXUsers") - List xuserInfoList; - - public int getTotalCount() { - return totalCount; - } - - public void setTotalCount(int totalCount) { - this.totalCount = totalCount; - } + public int getTotalCount() { + return totalCount; + } - public List getXuserInfoList() { - return xuserInfoList; - } + public void setTotalCount(int totalCount) { + this.totalCount = totalCount; + } - public void setXuserInfoList(List xuserInfoList) { - this.xuserInfoList = xuserInfoList; - } + public List getXuserInfoList() { + return xuserInfoList; + } - - + public void setXuserInfoList(List xuserInfoList) { + this.xuserInfoList = xuserInfoList; + } } diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java index 37c780f9df..2d7b3432d9 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java @@ -19,290 +19,268 @@ package org.apache.ranger.unixusersync.process; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Map; -import java.util.Set; -import java.util.List; -import java.util.HashMap; -import java.util.Date; -import java.util.HashSet; -import java.util.ArrayList; - +import com.fasterxml.jackson.core.type.TypeReference; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; import org.apache.ranger.authorization.utils.JsonUtils; -import org.apache.ranger.ugsyncutil.util.UgsyncCommonConstants; -import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; import org.apache.ranger.ugsyncutil.model.FileSyncSourceInfo; import org.apache.ranger.ugsyncutil.model.UgsyncAuditInfo; +import org.apache.ranger.ugsyncutil.util.UgsyncCommonConstants; +import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; import org.apache.ranger.usergroupsync.AbstractUserGroupSource; import org.apache.ranger.usergroupsync.UserGroupSink; +import org.apache.ranger.usergroupsync.UserGroupSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.core.type.TypeReference; - -import org.apache.ranger.usergroupsync.UserGroupSource; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; -public class FileSourceUserGroupBuilder extends AbstractUserGroupSource implements UserGroupSource { - private static final Logger LOG = LoggerFactory.getLogger(FileSourceUserGroupBuilder.class); - - private Map> user2GroupListMap = new HashMap>(); - private String userGroupFilename = null; - private Map> sourceUsers; // Stores username and attr name & value pairs - private Map> sourceGroups; // Stores groupname and attr name & value pairs - private Map> sourceGroupUsers; - private long usergroupFileModified = 0; - private UgsyncAuditInfo ugsyncAuditInfo; - private FileSyncSourceInfo fileSyncSourceInfo; - private boolean isStartupFlag = false; - - private boolean isUpdateSinkSucc = true; - private int deleteCycles; - private boolean computeDeletes = false; - private String currentSyncSource; - - public static void main(String[] args) throws Throwable { - FileSourceUserGroupBuilder filesourceUGBuilder = new FileSourceUserGroupBuilder(); - - if (args.length > 0) { - filesourceUGBuilder.setUserGroupFilename(args[0]); - } - - filesourceUGBuilder.init(); - - UserGroupSink ugSink = UserGroupSyncConfig.getInstance().getUserGroupSink(); - LOG.info("initializing sink: " + ugSink.getClass().getName()); - ugSink.init(); - - filesourceUGBuilder.updateSink(ugSink); - - if ( LOG.isDebugEnabled()) { - filesourceUGBuilder.print(); - } - } - - public FileSourceUserGroupBuilder() { - super(); - } - - @Override - public void init() throws Throwable { - isStartupFlag = true; - deleteCycles = 1; - currentSyncSource = config.getCurrentSyncSource(); - if(userGroupFilename == null) { - userGroupFilename = config.getUserSyncFileSource(); - } - ugsyncAuditInfo = new UgsyncAuditInfo(); - fileSyncSourceInfo = new FileSyncSourceInfo(); - ugsyncAuditInfo.setSyncSource(currentSyncSource); - ugsyncAuditInfo.setFileSyncSourceInfo(fileSyncSourceInfo); - fileSyncSourceInfo.setFileName(userGroupFilename); - buildUserGroupInfo(); - } - - @Override - public boolean isChanged() { - computeDeletes = false; - // If previous update to Ranger admin fails, - // we want to retry the sync process even if there are no changes to the sync files - if (!isUpdateSinkSucc) { - LOG.info("Previous updateSink failed and hence retry!!"); - isUpdateSinkSucc = true; - return true; - } - try { - if (config.isUserSyncDeletesEnabled() && deleteCycles >= config.getUserSyncDeletesFrequency()) { - deleteCycles = 1; - computeDeletes = true; - if (LOG.isDebugEnabled()) { - LOG.debug("Compute deleted users/groups is enabled for this sync cycle"); - } - return true; - } - } catch (Throwable t) { - LOG.error("Failed to get information about usersync delete frequency", t); - } - if (config.isUserSyncDeletesEnabled()) { - deleteCycles++; - } - - long TempUserGroupFileModifedAt = new File(userGroupFilename).lastModified(); - if (usergroupFileModified != TempUserGroupFileModifedAt) { - return true; - } - return false; - } - - @Override - public void updateSink(UserGroupSink sink) throws Throwable { - DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - Date lastModifiedTime = new Date(usergroupFileModified); - Date syncTime = new Date(System.currentTimeMillis()); - fileSyncSourceInfo.setLastModified(formatter.format(lastModifiedTime)); - fileSyncSourceInfo.setSyncTime(formatter.format(syncTime)); - - if (isChanged() || isStartupFlag) { - buildUserGroupInfo(); - - for (Map.Entry> entry : user2GroupListMap.entrySet()) { - String userName = entry.getKey(); - Map userAttrMap = new HashMap<>(); - userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName); - userAttrMap.put(UgsyncCommonConstants.FULL_NAME, userName); - userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); - sourceUsers.put(userName, userAttrMap); - List groups = entry.getValue(); - if (groups != null) { - for(String groupName : groups) { - Map groupAttrMap = new HashMap<>(); - groupAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, groupName); - groupAttrMap.put(UgsyncCommonConstants.FULL_NAME, groupName); - groupAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); - sourceGroups.put(groupName, groupAttrMap); - Set groupUsers = sourceGroupUsers.get(groupName); - if (CollectionUtils.isNotEmpty(groupUsers)) { - groupUsers.add(userName); - } else { - groupUsers = new HashSet<>(); - groupUsers.add(userName); - } - sourceGroupUsers.put(groupName, groupUsers); - } - } - - } - if (LOG.isDebugEnabled()) { - LOG.debug("Users = " + sourceUsers.keySet()); - LOG.debug("Groups = " + sourceGroups.keySet()); - LOG.debug("GroupUsers = " + sourceGroupUsers.keySet()); - } - - try { - sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers, sourceGroupUsers, computeDeletes); - } catch (Throwable t) { - LOG.error("Failed to update ranger admin. Will retry in next sync cycle!!", t); - isUpdateSinkSucc = false; - } - } - try { - sink.postUserGroupAuditInfo(ugsyncAuditInfo); - } catch (Throwable t) { - LOG.error("sink.postUserGroupAuditInfo failed with exception: " + t.getMessage()); - } - isStartupFlag = false; - } - - private void setUserGroupFilename(String filename) { - userGroupFilename = filename; - } - - private void print() { - for(String user : user2GroupListMap.keySet()) { - LOG.debug("USER:" + user); - List groups = user2GroupListMap.get(user); - if (groups != null) { - for(String group : groups) { - LOG.debug("\tGROUP: " + group); - } - } - } - } - - public void buildUserGroupInfo() throws Throwable { - sourceUsers = new HashMap<>(); - sourceGroups = new HashMap<>(); - sourceGroupUsers = new HashMap<>(); - buildUserGroupList(); - if ( LOG.isDebugEnabled()) { - print(); - } - } - - public void buildUserGroupList() throws Throwable { - if (userGroupFilename == null){ - throw new Exception("User Group Source File is not Configured. Please maintain in unixauthservice.properties or pass it as command line argument for org.apache.ranger.unixusersync.process.FileSourceUserGroupBuilder"); - } - - File f = new File(userGroupFilename); - - if (f.exists() && f.canRead()) { - Map> tmpUser2GroupListMap = null; - - if ( isJsonFile(userGroupFilename) ) { - tmpUser2GroupListMap = readJSONfile(f); - } else { - tmpUser2GroupListMap = readTextFile(f); - } - - if(tmpUser2GroupListMap != null) { - user2GroupListMap = tmpUser2GroupListMap; - - usergroupFileModified = f.lastModified(); - } else { - LOG.info("No new UserGroup to sync at this time"); - } - } else { - throw new Exception("User Group Source File " + userGroupFilename + "doesn't not exist or readable"); - } - } - - public boolean isJsonFile(String userGroupFilename) { - boolean ret = false; - - if ( userGroupFilename.toLowerCase().endsWith(".json")) { - ret = true; - } - - return ret; - } - - public Map> readJSONfile(File jsonFile) throws Exception { - return JsonUtils.getMapper().readValue(jsonFile, new TypeReference>>() {}); - } - - public Map> readTextFile(File textFile) throws Exception { - - Map> ret = new HashMap>(); - - String delimiter = config.getUserSyncFileSourceDelimiter(); - - CSVFormat csvFormat = CSVFormat.newFormat(delimiter.charAt(0)); - - try(CSVParser csvParser = new CSVParser(new BufferedReader(new FileReader(textFile)), csvFormat)) { - - List csvRecordList = csvParser.getRecords(); - - if ( csvRecordList != null) { - for(CSVRecord csvRecord : csvRecordList) { - List groups = new ArrayList(); - String user = csvRecord.get(0); - - user = user.replaceAll("^\"|\"$", ""); - - int i = csvRecord.size(); - - for (int j = 1; j < i; j ++) { - String group = csvRecord.get(j); - if ( group != null && !group.isEmpty()) { - group = group.replaceAll("^\"|\"$", ""); - groups.add(group); - } - } - ret.put(user,groups); - } - } - - } - - return ret; - } - -} \ No newline at end of file +public class FileSourceUserGroupBuilder extends AbstractUserGroupSource implements UserGroupSource { + private static final Logger LOG = LoggerFactory.getLogger(FileSourceUserGroupBuilder.class); + private String userGroupFilename; + private String currentSyncSource; + private Map> sourceUsers; // Stores username and attr name & value pairs + private Map> sourceGroups; // Stores groupname and attr name & value pairs + private Map> sourceGroupUsers; + private Map> user2GroupListMap = new HashMap>(); + private UgsyncAuditInfo ugsyncAuditInfo; + private FileSyncSourceInfo fileSyncSourceInfo; + private int deleteCycles; + private long usergroupFileModified; + private boolean computeDeletes; + private boolean isStartupFlag; + private boolean isUpdateSinkSucc = true; + + public FileSourceUserGroupBuilder() { + super(); + } + + public static void main(String[] args) throws Throwable { + FileSourceUserGroupBuilder filesourceUGBuilder = new FileSourceUserGroupBuilder(); + + if (args.length > 0) { + filesourceUGBuilder.setUserGroupFilename(args[0]); + } + + filesourceUGBuilder.init(); + + UserGroupSink ugSink = UserGroupSyncConfig.getInstance().getUserGroupSink(); + LOG.info("Initializing Sink: {}", ugSink.getClass().getName()); + ugSink.init(); + + filesourceUGBuilder.updateSink(ugSink); + + if (LOG.isDebugEnabled()) { + filesourceUGBuilder.print(); + } + } + + @Override + public void init() throws Throwable { + isStartupFlag = true; + deleteCycles = 1; + currentSyncSource = config.getCurrentSyncSource(); + if (userGroupFilename == null) { + userGroupFilename = config.getUserSyncFileSource(); + } + ugsyncAuditInfo = new UgsyncAuditInfo(); + fileSyncSourceInfo = new FileSyncSourceInfo(); + ugsyncAuditInfo.setSyncSource(currentSyncSource); + ugsyncAuditInfo.setFileSyncSourceInfo(fileSyncSourceInfo); + fileSyncSourceInfo.setFileName(userGroupFilename); + buildUserGroupInfo(); + } + + @Override + public boolean isChanged() { + computeDeletes = false; + // If previous update to Ranger admin fails, we want to retry the sync process even if there are no changes to the sync files + if (!isUpdateSinkSucc) { + LOG.info("Previous updateSink failed and hence retry!!"); + isUpdateSinkSucc = true; + return true; + } + try { + if (config.isUserSyncDeletesEnabled() && deleteCycles >= config.getUserSyncDeletesFrequency()) { + deleteCycles = 1; + computeDeletes = true; + LOG.debug("Compute deleted users/groups is enabled for this sync cycle"); + return true; + } + } catch (Throwable t) { + LOG.error("Failed to get information about usersync delete frequency", t); + } + if (config.isUserSyncDeletesEnabled()) { + deleteCycles++; + } + + long tempUserGroupFileModifedAt = new File(userGroupFilename).lastModified(); + return usergroupFileModified != tempUserGroupFileModifedAt; + } + + @Override + public void updateSink(UserGroupSink sink) throws Throwable { + DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + Date lastModifiedTime = new Date(usergroupFileModified); + Date syncTime = new Date(System.currentTimeMillis()); + fileSyncSourceInfo.setLastModified(formatter.format(lastModifiedTime)); + fileSyncSourceInfo.setSyncTime(formatter.format(syncTime)); + + if (isChanged() || isStartupFlag) { + buildUserGroupInfo(); + + for (Map.Entry> entry : user2GroupListMap.entrySet()) { + String userName = entry.getKey(); + Map userAttrMap = new HashMap<>(); + userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName); + userAttrMap.put(UgsyncCommonConstants.FULL_NAME, userName); + userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); + sourceUsers.put(userName, userAttrMap); + List groups = entry.getValue(); + if (groups != null) { + for (String groupName : groups) { + Map groupAttrMap = new HashMap<>(); + groupAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, groupName); + groupAttrMap.put(UgsyncCommonConstants.FULL_NAME, groupName); + groupAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); + sourceGroups.put(groupName, groupAttrMap); + Set groupUsers = sourceGroupUsers.get(groupName); + if (CollectionUtils.isNotEmpty(groupUsers)) { + groupUsers.add(userName); + } else { + groupUsers = new HashSet<>(); + groupUsers.add(userName); + } + sourceGroupUsers.put(groupName, groupUsers); + } + } + } + + LOG.debug("Users = " + sourceUsers.keySet()); + LOG.debug("Groups = " + sourceGroups.keySet()); + LOG.debug("GroupUsers = " + sourceGroupUsers.keySet()); + + try { + sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers, sourceGroupUsers, computeDeletes); + } catch (Throwable t) { + LOG.error("Failed to update ranger admin. Will retry in next sync cycle!!", t); + isUpdateSinkSucc = false; + } + } + try { + sink.postUserGroupAuditInfo(ugsyncAuditInfo); + } catch (Throwable t) { + LOG.error("sink.postUserGroupAuditInfo failed with exception: " + t.getMessage()); + } + isStartupFlag = false; + } + + public void buildUserGroupInfo() throws Throwable { + sourceUsers = new HashMap<>(); + sourceGroups = new HashMap<>(); + sourceGroupUsers = new HashMap<>(); + buildUserGroupList(); + if (LOG.isDebugEnabled()) { + print(); + } + } + + public void buildUserGroupList() throws Throwable { + if (userGroupFilename == null) { + throw new Exception("User Group Source File is not Configured. Please maintain in unixauthservice.properties or pass it as command line argument for org.apache.ranger.unixusersync.process.FileSourceUserGroupBuilder"); + } + + File f = new File(userGroupFilename); + + if (f.exists() && f.canRead()) { + Map> tmpUser2GroupListMap = null; + + if (isJsonFile(userGroupFilename)) { + tmpUser2GroupListMap = readJSONfile(f); + } else { + tmpUser2GroupListMap = readTextFile(f); + } + + if (tmpUser2GroupListMap != null) { + user2GroupListMap = tmpUser2GroupListMap; + + usergroupFileModified = f.lastModified(); + } else { + LOG.info("No new UserGroup to sync at this time"); + } + } else { + throw new Exception("User Group Source File " + userGroupFilename + "doesn't not exist or readable"); + } + } + + public boolean isJsonFile(String userGroupFilename) { + return userGroupFilename.toLowerCase().endsWith(".json"); + } + + public Map> readJSONfile(File jsonFile) throws Exception { + return JsonUtils.getMapper().readValue(jsonFile, new TypeReference>>() {}); + } + + public Map> readTextFile(File textFile) throws Exception { + Map> ret = new HashMap>(); + + String delimiter = config.getUserSyncFileSourceDelimiter(); + + CSVFormat csvFormat = CSVFormat.newFormat(delimiter.charAt(0)); + + try (CSVParser csvParser = new CSVParser(new BufferedReader(new FileReader(textFile)), csvFormat)) { + List csvRecordList = csvParser.getRecords(); + + if (csvRecordList != null) { + for (CSVRecord csvRecord : csvRecordList) { + List groups = new ArrayList(); + String user = csvRecord.get(0); + + user = user.replaceAll("^\"|\"$", ""); + + int i = csvRecord.size(); + + for (int j = 1; j < i; j++) { + String group = csvRecord.get(j); + if (group != null && !group.isEmpty()) { + group = group.replaceAll("^\"|\"$", ""); + groups.add(group); + } + } + ret.put(user, groups); + } + } + } + + return ret; + } + + private void setUserGroupFilename(String filename) { + userGroupFilename = filename; + } + + private void print() { + for (String user : user2GroupListMap.keySet()) { + LOG.debug("USER: {}", user); + List groups = user2GroupListMap.get(user); + if (groups != null) { + for (String group : groups) { + LOG.debug("\tGROUP: {}", group); + } + } + } + } +} diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java index df3632a837..ca842f9c39 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java @@ -19,26 +19,7 @@ package org.apache.ranger.unixusersync.process; -import java.io.IOException; -import java.net.UnknownHostException; -import java.security.PrivilegedAction; -import java.util.Collections; -import java.util.Map; -import java.util.Set; -import java.util.List; -import java.util.HashMap; -import java.util.HashSet; -import java.util.ArrayList; -import java.util.StringTokenizer; -import java.util.LinkedHashMap; -import java.util.regex.Pattern; - -import javax.security.auth.Subject; -import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.core.Cookie; -import javax.ws.rs.core.NewCookie; - -import com.fasterxml.jackson.core.type.TypeReference; +import com.sun.jersey.api.client.ClientResponse; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; @@ -53,1905 +34,1697 @@ import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; import org.apache.ranger.unixusersync.model.GetXGroupListResponse; import org.apache.ranger.unixusersync.model.GetXUserListResponse; +import org.apache.ranger.usergroupsync.AbstractUserGroupSource; +import org.apache.ranger.usergroupsync.UserGroupSink; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.sun.jersey.api.client.ClientResponse; -import org.apache.ranger.usergroupsync.AbstractUserGroupSource; -import org.apache.ranger.usergroupsync.UserGroupSink; +import javax.security.auth.Subject; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.Cookie; +import javax.ws.rs.core.NewCookie; +import java.io.IOException; +import java.net.UnknownHostException; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.StringTokenizer; +import java.util.regex.Pattern; public class PolicyMgrUserGroupBuilder extends AbstractUserGroupSource implements UserGroupSink { - - private static final Logger LOG = LoggerFactory.getLogger(PolicyMgrUserGroupBuilder.class); - private static final String AUTHENTICATION_TYPE = "hadoop.security.authentication"; - private static final String AUTH_KERBEROS = "kerberos"; - private static final String KERBEROS_PRINCIPAL = "ranger.usersync.kerberos.principal"; - private static final String KERBEROS_KEYTAB = "ranger.usersync.kerberos.keytab"; - private static final String NAME_RULE = "hadoop.security.auth_to_local"; - public static final String PM_USER_LIST_URI = "/service/xusers/users/"; // GET - private static final String PM_ADD_USERS_URI = "/service/xusers/ugsync/users"; // POST - private static final String PM_ADD_GROUP_USER_LIST_URI = "/service/xusers/ugsync/groupusers"; // POST - public static final String PM_GROUP_LIST_URI = "/service/xusers/groups/"; // GET - private static final String PM_ADD_GROUPS_URI = "/service/xusers/ugsync/groups/"; // POST - public static final String PM_GET_ALL_GROUP_USER_MAP_LIST_URI = "/service/xusers/ugsync/groupusers"; // GET - private static final String PM_AUDIT_INFO_URI = "/service/xusers/ugsync/auditinfo/"; // POST - public static final String PM_UPDATE_USERS_ROLES_URI = "/service/xusers/users/roleassignments"; // PUT - private static final String PM_UPDATE_DELETED_USERS_URI = "/service/xusers/ugsync/users/visibility"; // POST - private static final String PM_UPDATE_DELETED_GROUPS_URI = "/service/xusers/ugsync/groups/visibility"; // POST - private static final Pattern USER_OR_GROUP_NAME_VALIDATION_REGEX = - Pattern.compile("^([A-Za-z0-9_]|[\u00C0-\u017F])([a-zA-Z0-9\\s,._\\-+/@= ]|[\u00C0-\u017F])+$", Pattern.CASE_INSENSITIVE); - - private static final String SOURCE_EXTERNAL ="1"; - private static final String STATUS_ENABLED = "1"; - private static final String ISVISIBLE = "1"; - private static final String ISHIDDEN = "0"; - - private static String localHostname = "unknown"; - private String recordsToPullPerCall = "10"; - private boolean isMockRun = false; - private String policyMgrBaseUrl; - private Cookie sessionId=null; - private boolean isValidRangerCookie=false; - List cookieList=new ArrayList<>(); - private boolean isStartupFlag; - - private volatile RangerUgSyncRESTClient ldapUgSyncClient; - - private Map userCache; // Key is user name as in ranger db - private Map groupCache; // Key is group name as in ranger db - private Map> groupUsersCache; // Key is group name and value is set of user names (as stored in ranger DB) - private Map groupNameMap; // Key is group DN and value is group name as stored in ranger DB - private Map userNameMap; // Key is user DN and value is user name as stored in ranger DB - - private Map deltaGroups; - private Map deltaUsers; - private Map> deltaGroupUsers; - private Set computeRolesForUsers; - - private Map deletedGroups; - private Map deletedUsers; - - private int noOfNewUsers; - private int noOfNewGroups; - private int noOfModifiedUsers; - private int noOfModifiedGroups; - private int noOfDeletedUsers = 0; - private int noOfDeletedGroups = 0; - - private boolean userNameCaseConversionFlag; - private boolean groupNameCaseConversionFlag; - private boolean userNameLowerCaseFlag = false; - private boolean groupNameLowerCaseFlag = false; - private String currentSyncSource; - private String ldapUrl; - private boolean isUserSyncNameValidationEnabled = false; - private boolean isSyncSourceValidationEnabled = false; - - private String authenticationType = null; - String principal; - String keytab; - String policyMgrUserName; - String nameRules; - Map userMap = new LinkedHashMap<>(); - Map groupMap = new LinkedHashMap<>(); - Map whiteListUserMap = new LinkedHashMap<>(); - Map whiteListGroupMap = new LinkedHashMap<>(); - - private boolean isRangerCookieEnabled; - private String rangerCookieName; - private static String errMsgForInactiveServer = "This userGroupSync server is not in active state. Cannot commit transaction!"; - static { - try { - localHostname = java.net.InetAddress.getLocalHost().getCanonicalHostName(); - } catch (UnknownHostException e) { - localHostname = "unknown"; - } - } - - public static void main(String[] args) throws Throwable { - PolicyMgrUserGroupBuilder ugbuilder = new PolicyMgrUserGroupBuilder(); - ugbuilder.init(); - - } - - public PolicyMgrUserGroupBuilder() { - super(); - - String userNameCaseConversion = config.getUserNameCaseConversion(); - - if (UserGroupSyncConfig.UGSYNC_NONE_CASE_CONVERSION_VALUE.equalsIgnoreCase(userNameCaseConversion)) { - userNameCaseConversionFlag = false; - } - else { - userNameCaseConversionFlag = true; - userNameLowerCaseFlag = UserGroupSyncConfig.UGSYNC_LOWER_CASE_CONVERSION_VALUE.equalsIgnoreCase(userNameCaseConversion); - } - - String groupNameCaseConversion = config.getGroupNameCaseConversion(); - - if (UserGroupSyncConfig.UGSYNC_NONE_CASE_CONVERSION_VALUE.equalsIgnoreCase(groupNameCaseConversion)) { - groupNameCaseConversionFlag = false; - } - else { - groupNameCaseConversionFlag = true; - groupNameLowerCaseFlag = UserGroupSyncConfig.UGSYNC_LOWER_CASE_CONVERSION_VALUE.equalsIgnoreCase(groupNameCaseConversion); - } - } - - synchronized public void init() throws Throwable { - isUserSyncNameValidationEnabled = config.isUserSyncNameValidationEnabled(); - isSyncSourceValidationEnabled = config.isSyncSourceValidationEnabled(); - recordsToPullPerCall = config.getMaxRecordsPerAPICall(); - policyMgrBaseUrl = config.getPolicyManagerBaseURL(); - isMockRun = config.isMockRunEnabled(); - isRangerCookieEnabled = config.isUserSyncRangerCookieEnabled(); - rangerCookieName = config.getRangerAdminCookieName(); - groupNameMap = new HashMap<>(); - userNameMap = new HashMap<>(); - userCache = new HashMap<>(); - groupCache = new HashMap<>(); - groupUsersCache = new HashMap<>(); - isStartupFlag = true; - ldapUrl = null; - currentSyncSource = config.getCurrentSyncSource(); - if (StringUtils.equalsIgnoreCase(currentSyncSource, "LDAP/AD")) { - ldapUrl = config.getLdapUrl(); - } - - sessionId=null; - String keyStoreFile = config.getSSLKeyStorePath(); - String trustStoreFile = config.getSSLTrustStorePath(); - String keyStoreFilepwd = config.getSSLKeyStorePathPassword(); - String trustStoreFilepwd = config.getSSLTrustStorePathPassword(); - String keyStoreType = config.getSSLKeyStoreType(); - String trustStoreType = config.getSSLTrustStoreType(); - authenticationType = config.getProperty(AUTHENTICATION_TYPE,"simple"); - try { - principal = SecureClientLogin.getPrincipal(config.getProperty(KERBEROS_PRINCIPAL, ""), localHostname); - } catch (IOException ignored) { - // do nothing - } - keytab = config.getProperty(KERBEROS_KEYTAB, ""); - policyMgrUserName = config.getPolicyMgrUserName(); - nameRules = config.getProperty(NAME_RULE,"DEFAULT"); - ldapUgSyncClient = new RangerUgSyncRESTClient(policyMgrBaseUrl, keyStoreFile, keyStoreFilepwd, keyStoreType, - trustStoreFile, trustStoreFilepwd, trustStoreType, authenticationType, principal, keytab, - policyMgrUserName, config.getPolicyMgrPassword()); - - String userGroupRoles = config.getGroupRoleRules(); - if (userGroupRoles != null && !userGroupRoles.isEmpty()) { - getRoleForUserGroups(userGroupRoles, userMap, groupMap); - } - String whiteListUserRoles = config.getWhileListUserRoleRules(); - if (whiteListUserRoles != null && !whiteListUserRoles.isEmpty()) { - getRoleForUserGroups(whiteListUserRoles, whiteListUserMap, whiteListGroupMap); - } - String policyMgrUserRole = whiteListUserMap.get(policyMgrUserName); - if (!StringUtils.equalsIgnoreCase(policyMgrUserRole, "ROLE_SYS_ADMIN")) { - whiteListUserMap.put(policyMgrUserName, "ROLE_SYS_ADMIN"); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Entries in group role assignments: " + groupMap); - LOG.debug("Entries in whitelist group role assignments: " + whiteListGroupMap); - } - - buildUserGroupInfo(); - - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("PolicyMgrUserGroupBuilderOld.init()==> policyMgrBaseUrl: %s, KeyStore File: %s, TrustStore File: %s, Authentication Type: %s", policyMgrBaseUrl, keyStoreFile, trustStoreFile, authenticationType)); - } - - } - - @Override - public void postUserGroupAuditInfo(UgsyncAuditInfo ugsyncAuditInfo) throws Throwable { - ugsyncAuditInfo.setNoOfNewUsers(Integer.toUnsignedLong(noOfNewUsers)); - ugsyncAuditInfo.setNoOfNewGroups(Integer.toUnsignedLong(noOfNewGroups)); - ugsyncAuditInfo.setNoOfModifiedUsers(Integer.toUnsignedLong(noOfModifiedUsers)); - ugsyncAuditInfo.setNoOfModifiedGroups(Integer.toUnsignedLong(noOfModifiedGroups)); - int noOfCachedUsers = userCache.size(); - int noOfCachedGroups = groupCache.size(); - switch (ugsyncAuditInfo.getSyncSource()) { - case "LDAP/AD": - ugsyncAuditInfo.getLdapSyncSourceInfo().setTotalUsersSynced(noOfCachedUsers); - ugsyncAuditInfo.getLdapSyncSourceInfo().setTotalGroupsSynced(noOfCachedGroups); - ugsyncAuditInfo.getLdapSyncSourceInfo().setTotalUsersDeleted(noOfDeletedUsers); - ugsyncAuditInfo.getLdapSyncSourceInfo().setTotalGroupsDeleted(noOfDeletedGroups); - break; - case "Unix": - ugsyncAuditInfo.getUnixSyncSourceInfo().setTotalUsersSynced(noOfCachedUsers); - ugsyncAuditInfo.getUnixSyncSourceInfo().setTotalGroupsSynced(noOfCachedGroups); - ugsyncAuditInfo.getUnixSyncSourceInfo().setTotalUsersDeleted(noOfDeletedUsers); - ugsyncAuditInfo.getUnixSyncSourceInfo().setTotalGroupsDeleted(noOfDeletedGroups); - break; - case "File" : - ugsyncAuditInfo.getFileSyncSourceInfo().setTotalUsersSynced(noOfCachedUsers); - ugsyncAuditInfo.getFileSyncSourceInfo().setTotalGroupsSynced(noOfCachedGroups); - ugsyncAuditInfo.getFileSyncSourceInfo().setTotalUsersDeleted(noOfDeletedUsers); - ugsyncAuditInfo.getFileSyncSourceInfo().setTotalGroupsDeleted(noOfDeletedGroups); - break; - default: - break; - } - - if (!isMockRun) { - checkStatus(); - addUserGroupAuditInfo(ugsyncAuditInfo); - } - - } - - @Override - public void addOrUpdateUsersGroups(Map> sourceGroups, - Map> sourceUsers, - Map> sourceGroupUsers, - boolean computeDeletes) throws Throwable { - checkStatus(); - - noOfNewUsers = 0; - noOfNewGroups = 0; - noOfModifiedUsers = 0; - noOfModifiedGroups = 0; - computeRolesForUsers = new HashSet<>(); - - if (!isStartupFlag && computeDeletes) { - LOG.info("Computing deleted users/groups"); - if (LOG.isDebugEnabled()) { - LOG.debug("Computing deleted users/groups"); - } - if (MapUtils.isNotEmpty(sourceGroups)) { - updateDeletedGroups(sourceGroups); - } - if (MapUtils.isNotEmpty(sourceUsers)) { - updateDeletedUsers(sourceUsers); - } - - if (MapUtils.isNotEmpty(deletedGroups)) { - groupCache.putAll(deletedGroups); - } - - if (MapUtils.isNotEmpty(deletedUsers)) { - userCache.putAll(deletedUsers); - } - } - - if (MapUtils.isNotEmpty(sourceGroups)) { - addOrUpdateGroups(sourceGroups); - } - if (MapUtils.isNotEmpty(sourceUsers)) { - addOrUpdateUsers(sourceUsers); - } - - if (MapUtils.isNotEmpty(sourceGroupUsers)) { - addOrUpdateGroupUsers(sourceGroupUsers); - } - - if (isStartupFlag) { - // This is to handle any config changes for role assignments that might impact existing users in ranger db - if (MapUtils.isNotEmpty(whiteListUserMap)) { - if (LOG.isDebugEnabled()) { - LOG.debug("adding " + whiteListUserMap.keySet() + " for computing roles during startup"); - } - computeRolesForUsers.addAll(whiteListUserMap.keySet()); // Add all the user defined in the whitelist role assignment rules - } - if (MapUtils.isNotEmpty(whiteListGroupMap)) { - for (String groupName : whiteListGroupMap.keySet()) { - Set groupUsers = null; - if (CollectionUtils.isNotEmpty(groupUsersCache.get(groupName))) { - groupUsers = new HashSet<>(groupUsersCache.get(groupName)); - } else if (CollectionUtils.isNotEmpty(deltaGroupUsers.get(groupName))) { - groupUsers = new HashSet<>(deltaGroupUsers.get(groupName)); - } - if (groupUsers != null) { - if (LOG.isDebugEnabled()) { - LOG.debug("adding " + groupUsers + " from " + groupName + " for computing roles during startup"); - } - computeRolesForUsers.addAll(groupUsers); - } - } - } - - if (MapUtils.isNotEmpty(userMap)) { - if (LOG.isDebugEnabled()) { - LOG.debug("adding " + userMap.keySet() + " for computing roles during startup"); - } - computeRolesForUsers.addAll(userMap.keySet()); // Add all the user defined in the role assignment rules - } - if (MapUtils.isNotEmpty(groupMap)) { - for (String groupName : groupMap.keySet()) { - Set groupUsers = null; - if (CollectionUtils.isNotEmpty(groupUsersCache.get(groupName))) { - groupUsers = new HashSet<>(groupUsersCache.get(groupName)); - } else if (CollectionUtils.isNotEmpty(deltaGroupUsers.get(groupName))) { - groupUsers = new HashSet<>(deltaGroupUsers.get(groupName)); - } - if (groupUsers != null) { - if (LOG.isDebugEnabled()) { - LOG.debug("adding " + groupUsers + " from " + groupName + " for computing roles during startup"); - } - computeRolesForUsers.addAll(groupUsers); - } - } - } - } - - if (CollectionUtils.isNotEmpty(computeRolesForUsers)) { - updateUserRoles(); - } - isStartupFlag = false; - - if (LOG.isDebugEnabled()){ - LOG.debug("Update cache"); - } - if (MapUtils.isNotEmpty(deltaGroups)) { - groupCache.putAll(deltaGroups); - } - if (MapUtils.isNotEmpty(deltaUsers)) { - userCache.putAll(deltaUsers); - } - if (MapUtils.isNotEmpty(deltaGroupUsers)) { - groupUsersCache.putAll(deltaGroupUsers); - } - } - - private void buildUserGroupInfo() throws Throwable { - if(LOG.isDebugEnabled() && authenticationType != null && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) && SecureClientLogin.isKerberosCredentialExists(principal, keytab)) { - LOG.debug(String.format("==> Kerberos Environment : Principal is %s and Keytab is %s", principal, keytab)); - } - if (authenticationType != null && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) && SecureClientLogin.isKerberosCredentialExists(principal, keytab)) { - LOG.info(String.format("Using principal: %s and keytab: %s", principal, keytab)); - Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); - Boolean isInitDone = Subject.doAs(sub, new PrivilegedAction() { - @Override - public Boolean run() { - try { - buildGroupList(); - buildUserList(); - buildGroupUserLinkList(); - } catch (Throwable e) { - LOG.error("Failed to build Users and Groups from Ranger admin : ", e); - return false; - } - return true; - } - }); - if (isInitDone.booleanValue() == false) { - String msg = ("Failed to build Users and Groups from Ranger admin"); - LOG.error(msg); - throw new Exception(msg); - } - } else { - buildGroupList(); - buildUserList(); - buildGroupUserLinkList(); - } - } - - private void buildGroupList() throws Throwable { - if (LOG.isDebugEnabled()) { - LOG.debug("==> PolicyMgrUserGroupBuilder.buildGroupList()"); - } - int totalCount = 100; - int retrievedCount = 0; - - while (retrievedCount < totalCount) { - String response = null; - ClientResponse clientResp = null; - - Map queryParams = new HashMap<>(); - queryParams.put("pageSize", recordsToPullPerCall); - queryParams.put("startIndex", String.valueOf(retrievedCount)); - - if (isRangerCookieEnabled) { - response = cookieBasedGetEntity(PM_GROUP_LIST_URI, retrievedCount); - } else { - try { - clientResp = ldapUgSyncClient.get(PM_GROUP_LIST_URI, queryParams); - if (clientResp != null) { - response = clientResp.getEntity(String.class); - } - } catch (Exception e) { - LOG.error("Failed to get groups from Ranger, Error is : " + e.getMessage()); - throw e; - } - } - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("REST response from %s : %s", PM_GROUP_LIST_URI, response)); - } - GetXGroupListResponse groupList = JsonUtils.jsonToObject(response, GetXGroupListResponse.class); - - totalCount = groupList.getTotalCount(); - - if (groupList.getXgroupInfoList() != null) { - for (XGroupInfo g : groupList.getXgroupInfoList()) { - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("GROUP: Id: %s, Name: %s, Description: %s", g.getId(), g.getName(), g.getDescription())); - } - if(null != g.getOtherAttributes()) { - g.setOtherAttrsMap(JsonUtils.jsonToObject(g.getOtherAttributes(), Map.class)); - } - groupCache.put(g.getName(), g); - } - retrievedCount = groupCache.size(); - } - LOG.info("PolicyMgrUserGroupBuilder.buildGroupList(): No. of groups retrieved from ranger admin " + retrievedCount); - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.buildGroupList()"); - } - } - - private void buildUserList() throws Throwable { - if (LOG.isDebugEnabled()) { - LOG.debug("==> PolicyMgrUserGroupBuilder.buildUserList()"); - } - int totalCount = 100; - int retrievedCount = 0; - - while (retrievedCount < totalCount) { - String response = null; - ClientResponse clientResp = null; - - Map queryParams = new HashMap<>(); - queryParams.put("pageSize", recordsToPullPerCall); - queryParams.put("startIndex", String.valueOf(retrievedCount)); - - if (isRangerCookieEnabled) { - response = cookieBasedGetEntity(PM_USER_LIST_URI, retrievedCount); - } else { - try { - clientResp = ldapUgSyncClient.get(PM_USER_LIST_URI, queryParams); - if (clientResp != null) { - response = clientResp.getEntity(String.class); - } - } catch (Exception e) { - LOG.error("Failed to get users from Ranger admin, Error is : "+e.getMessage()); - throw e; - } - } - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("REST response from %s : %s", PM_USER_LIST_URI, response)); - } - GetXUserListResponse userList = JsonUtils.jsonToObject(response, GetXUserListResponse.class); - totalCount = userList.getTotalCount(); - - if (userList.getXuserInfoList() != null) { - for (XUserInfo u : userList.getXuserInfoList()) { - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("USER: Id: %s, Name: %s, Description: %s", u.getId(), u.getName(), u.getDescription())); - } - if(null != u.getOtherAttributes()) { - u.setOtherAttrsMap(JsonUtils.jsonToObject(u.getOtherAttributes(), Map.class)); - } - userCache.put(u.getName(), u); - } - retrievedCount = userCache.size(); - } - LOG.info("PolicyMgrUserGroupBuilder.buildUserList(): No. of users retrieved from ranger admin = " + retrievedCount); - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.buildUserList()"); - } - } - - private void buildGroupUserLinkList() throws Throwable { - if (LOG.isDebugEnabled()) { - LOG.debug("==> PolicyMgrUserGroupBuilder.buildGroupUserLinkList()"); - } - - String response = null; - ClientResponse clientResp = null; - - if (isRangerCookieEnabled) { - response = cookieBasedGetEntity(PM_GET_ALL_GROUP_USER_MAP_LIST_URI, 0); - } else { - try { - clientResp = ldapUgSyncClient.get(PM_GET_ALL_GROUP_USER_MAP_LIST_URI, null); - if (clientResp != null) { - response = clientResp.getEntity(String.class); - } - } catch (Exception e) { - LOG.error("Failed to get response, group user mappings from Ranger admin. Error is : " + e.getMessage()); - throw e; - } - } - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("REST response from %s : %s", PM_GET_ALL_GROUP_USER_MAP_LIST_URI, response)); - } - - groupUsersCache = JsonUtils.jsonToObject(response, Map.class); - if (MapUtils.isEmpty(groupUsersCache)) { - groupUsersCache = new HashMap<>(); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Group User List : " + groupUsersCache.values()); - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.buildGroupUserLinkList()"); - } - } - - private void addOrUpdateUsers(Map> sourceUsers) throws Throwable { - computeUserDelta(sourceUsers); - if (MapUtils.isNotEmpty(deltaUsers)) { - if (addOrUpdateDeltaUsers() == 0) { - String msg = "Failed to addorUpdate users to ranger admin"; - LOG.error(msg); - throw new Exception(msg); - } - } - } - - private void addOrUpdateGroups(Map> sourceGroups) throws Throwable { - computeGroupDelta(sourceGroups); - if (MapUtils.isNotEmpty(deltaGroups)) { - if (addOrUpdateDeltaGroups() == 0) { - String msg = "Failed to addorUpdate groups to ranger admin"; - LOG.error(msg); - throw new Exception(msg); - } - } - } - - private void addOrUpdateGroupUsers(Map> sourceGroupUsers) throws Throwable { - List groupUserInfoList = computeGroupUsersDelta(sourceGroupUsers); - if (CollectionUtils.isNotEmpty(groupUserInfoList)) { - noOfModifiedGroups += groupUserInfoList.size(); - if (addOrUpdateDeltaGroupUsers(groupUserInfoList) == 0) { - String msg = "Failed to addorUpdate group memberships to ranger admin"; - LOG.error(msg); - throw new Exception(msg); - } - } - } - - private void updateUserRoles() throws Throwable { - UsersGroupRoleAssignments ugRoleAssignments = new UsersGroupRoleAssignments(); - List allUsers = new ArrayList<>(computeRolesForUsers); - - ugRoleAssignments.setUsers(allUsers); - ugRoleAssignments.setGroupRoleAssignments(groupMap); - ugRoleAssignments.setUserRoleAssignments(userMap); - ugRoleAssignments.setWhiteListUserRoleAssignments(whiteListUserMap); - ugRoleAssignments.setWhiteListGroupRoleAssignments(whiteListGroupMap); - ugRoleAssignments.setReset(isStartupFlag); - String updatedUsers = updateRoles(ugRoleAssignments); - if (updatedUsers == null) { - String msg = "Unable to update roles for " + allUsers; - LOG.error(msg); - throw new Exception(msg); - } - } - - private T setOtherAttributes(T uginfo, String syncSource, Map otherAttrsMap, String otherAttributes) { - - if (uginfo instanceof XUserInfo){ - XUserInfo xUserInfo = ((XUserInfo) uginfo); - xUserInfo.setSyncSource(syncSource); - xUserInfo.setOtherAttrsMap(otherAttrsMap); - xUserInfo.setOtherAttributes(otherAttributes); - return ((T) xUserInfo); - } else if (uginfo instanceof XGroupInfo ){ - XGroupInfo xGroupInfo = ((XGroupInfo) uginfo); - xGroupInfo.setSyncSource(syncSource); - xGroupInfo.setOtherAttrsMap(otherAttrsMap); - xGroupInfo.setOtherAttributes(otherAttributes); - return ((T) xGroupInfo); - } else { - return null; - } - } - - private void computeGroupDelta(Map> sourceGroups) { - if (LOG.isDebugEnabled()) { - LOG.debug("PolicyMgrUserGroupBuilder.computeGroupDelta(" + sourceGroups.keySet() + ")"); - } - deltaGroups = new HashMap<>(); - // Check if the group exists in cache. If not, mark as new group. - // else check if other attributes are updated and mark as updated group - for (String groupDN : sourceGroups.keySet()) { - Map newGroupAttrs = sourceGroups.get(groupDN); - String newGroupAttrsStr = JsonUtils.objectToJson(newGroupAttrs); - String groupName = groupNameMap.get(groupDN); - if (StringUtils.isEmpty(groupName)) { - groupName = groupNameTransform(newGroupAttrs.get(UgsyncCommonConstants.ORIGINAL_NAME).trim()); - } - - if (!isValidString(groupName)) { - LOG.warn("Ignoring invalid group " + groupName + " Full name = " + groupDN); - continue; - } - - if (!groupCache.containsKey(groupName)) { - XGroupInfo newGroup = addXGroupInfo(groupName, newGroupAttrs, newGroupAttrsStr); - deltaGroups.put(groupName, newGroup); - noOfNewGroups++; - groupNameMap.put(groupDN, groupName); - } else { - XGroupInfo curGroup = groupCache.get(groupName); - String curSyncSource = curGroup.getSyncSource(); - String curGroupAttrsStr = curGroup.getOtherAttributes(); - Map curGroupAttrs = curGroup.getOtherAttrsMap(); - String curGroupDN = MapUtils.isEmpty(curGroupAttrs) ? groupName : curGroupAttrs.get(UgsyncCommonConstants.FULL_NAME); - String newSyncSource = newGroupAttrs.get(UgsyncCommonConstants.SYNC_SOURCE); - - if (isStartupFlag && !isSyncSourceValidationEnabled && (!StringUtils.equalsIgnoreCase(curSyncSource, newSyncSource))) { - if (LOG.isDebugEnabled()) { - LOG.debug("[" + groupName + "]: SyncSource updated to " + newSyncSource + ", previous value: " + curSyncSource); - } - curGroup = setOtherAttributes(curGroup, newSyncSource, newGroupAttrs, newGroupAttrsStr); - deltaGroups.put(groupName, curGroup); - noOfModifiedGroups++; - groupNameMap.put(groupDN, groupName); - } else { - if (MapUtils.isNotEmpty(curGroupAttrs) && !StringUtils.equalsIgnoreCase(groupDN, curGroupDN)) { // skip update - if (LOG.isDebugEnabled()) { - LOG.debug("[" + groupName + "]: SyncSource update skipped, current group DN = " + curGroupDN + " new user DN = " + groupDN ); - } - - if (StringUtils.equalsIgnoreCase(curGroupAttrsStr, newGroupAttrsStr)) { - groupNameMap.put(groupDN, groupName); - } - continue; - } - - if (StringUtils.isEmpty(curSyncSource) || (!StringUtils.equalsIgnoreCase(curGroupAttrsStr, newGroupAttrsStr) - && StringUtils.equalsIgnoreCase(curSyncSource, newSyncSource))) { // update - if (LOG.isDebugEnabled()) { - if (StringUtils.isEmpty(curSyncSource)) { - LOG.debug("[" + groupName + "]: SyncSource updated to " + newSyncSource + ", previously empty"); - } else { - LOG.debug("[" + groupName + "]: Other Attributes updated!"); - } - } - curGroup = setOtherAttributes(curGroup, newSyncSource, newGroupAttrs, newGroupAttrsStr); - deltaGroups.put(groupName, curGroup); - noOfModifiedGroups++; - groupNameMap.put(groupDN, groupName); - - } else if (LOG.isDebugEnabled()) { - if (!StringUtils.equalsIgnoreCase(curSyncSource, newSyncSource)) { - LOG.debug("[" + groupName + "]: Different sync source exists, update skipped!"); - } else { - LOG.debug("[" + groupName + "]: No change, update skipped!"); - } - } - - if (StringUtils.equalsIgnoreCase(curGroupAttrsStr, newGroupAttrsStr)) { - groupNameMap.put(groupDN, groupName); - } - } - } - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.computeGroupDelta(" + deltaGroups.keySet() + ")"); - } - } - - private void computeUserDelta(Map> sourceUsers) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> PolicyMgrUserGroupBuilder.computeUserDelta(" + sourceUsers.keySet() + ")"); - } - deltaUsers = new HashMap<>(); - // Check if the user exists in cache. If not, mark as new user. - // else check if other attributes are updated and mark as updated user - for (Map.Entry> sourceUser : sourceUsers.entrySet()) { - String userDN = sourceUser.getKey(); - Map newUserAttrs = sourceUser.getValue(); - String newUserAttrsStr = JsonUtils.objectToJson(newUserAttrs); - String userName = userNameMap.get(userDN); - if (StringUtils.isEmpty(userName)) { - userName = userNameTransform(newUserAttrs.get(UgsyncCommonConstants.ORIGINAL_NAME).trim()); - } - - if (!isValidString(userName)) { - LOG.warn("Ignoring invalid user " + userName + " Full name = " + userDN); - continue; - } - - if (!userCache.containsKey(userName)) { - XUserInfo newUser = addXUserInfo(userName, newUserAttrs, newUserAttrsStr); - deltaUsers.put(userName, newUser); - noOfNewUsers++; - userNameMap.put(userDN, userName); - } else { - // no updates allowed for rangerusersync and admin - if (StringUtils.equalsIgnoreCase(policyMgrUserName, userName) || StringUtils.equalsIgnoreCase("admin", userName)) { - if (LOG.isDebugEnabled()) { - LOG.debug("[" + userName + "]: SyncSource update skipped!"); - } - continue; - } - - XUserInfo curUser = userCache.get(userName); - String curSyncSource = curUser.getSyncSource(); - String curUserAttrsStr = curUser.getOtherAttributes(); - Map curUserAttrs = curUser.getOtherAttrsMap(); - String curUserDN = MapUtils.isEmpty(curUserAttrs) ? userName : curUserAttrs.get(UgsyncCommonConstants.FULL_NAME); - String newSyncSource = newUserAttrs.get(UgsyncCommonConstants.SYNC_SOURCE); - if (isStartupFlag && !isSyncSourceValidationEnabled && (!StringUtils.equalsIgnoreCase(curSyncSource, newSyncSource))) { - if (LOG.isDebugEnabled()) { - LOG.debug("[" + userName + "]: SyncSource updated to " + newSyncSource + ", previous value: " + curSyncSource); - } - curUser = setOtherAttributes(curUser, newSyncSource, newUserAttrs, newUserAttrsStr); - curUser.setUserSource(SOURCE_EXTERNAL); - deltaUsers.put(userName, curUser); - noOfModifiedGroups++; - userNameMap.put(userDN, userName); - } else { - if (MapUtils.isNotEmpty(curUserAttrs) && !StringUtils.equalsIgnoreCase(userDN, curUserDN)) { // skip update - if (LOG.isDebugEnabled()) { // Same username with different DN already exists - LOG.debug("[" + userName + "]: SyncSource update skipped, current user DN = " + curUserDN + " new user DN = " + userDN ); - } - - if (StringUtils.equalsIgnoreCase(curUserAttrsStr, newUserAttrsStr)) { - userNameMap.put(userDN, userName); - } - continue; - } - - if (StringUtils.isEmpty(curSyncSource) || (!StringUtils.equalsIgnoreCase(curUserAttrsStr, newUserAttrsStr) - && StringUtils.equalsIgnoreCase(curSyncSource, newSyncSource))) { // update - if (LOG.isDebugEnabled()) { - if (StringUtils.isEmpty(curSyncSource)) { - LOG.debug("[" + userName + "]: SyncSource updated to " + newSyncSource + ", previously empty"); - } else { - LOG.debug("[" + userName + "]: Other Attributes updated!"); - } - } - curUser = setOtherAttributes(curUser, newSyncSource, newUserAttrs, newUserAttrsStr); - curUser.setUserSource(SOURCE_EXTERNAL); - deltaUsers.put(userName, curUser); - noOfModifiedUsers++; - userNameMap.put(userDN, userName); - } else if (LOG.isDebugEnabled()) { - if (!StringUtils.equalsIgnoreCase(curSyncSource, newSyncSource)) { - LOG.debug("[" + userName + "]: Different sync source exists, update skipped!"); - } else { - LOG.debug("[" + userName + "]: No change, update skipped!"); - } - } - - if (StringUtils.equalsIgnoreCase(curUserAttrsStr, newUserAttrsStr)) { - userNameMap.put(userDN, userName); - } - } - } - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.computeUserDelta(" + deltaUsers.keySet() + ")"); - } - } - - private List computeGroupUsersDelta(Map> sourceGroupUsers) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> PolicyMgrUserGroupBuilder.computeGroupUsersDelta(" + sourceGroupUsers.keySet() + ")"); - } - deltaGroupUsers = new HashMap<>(); - List deltaGroupUserInfoList = new ArrayList<>(); - for (String groupDN : sourceGroupUsers.keySet()) { - String groupName = groupNameMap.get(groupDN); - if (StringUtils.isEmpty(groupName)) { - if (LOG.isDebugEnabled()) { - LOG.debug("Ignoring group membership update for " + groupDN); - } - continue; - } - Set oldUsers = new HashSet<>(); - Set newUsers = new HashSet<>(); - Set addUsers = new HashSet<>(); - Set delUsers = new HashSet<>(); - if (CollectionUtils.isNotEmpty(groupUsersCache.get(groupName))) { - oldUsers = new HashSet<>(groupUsersCache.get(groupName)); - } - - for (String userDN : sourceGroupUsers.get(groupDN)) { - String userName = userNameMap.get(userDN); - if (!StringUtils.isEmpty(userName)) { - newUsers.add(userName); - if (CollectionUtils.isEmpty(oldUsers) || !oldUsers.contains(userName)) { - addUsers.add(userName); - } - } - } - - if (CollectionUtils.isNotEmpty(oldUsers)) { - for (String userName : oldUsers) { - if (CollectionUtils.isEmpty(newUsers)|| !newUsers.contains(userName)) { - delUsers.add(userName); - } - } - } - - if (CollectionUtils.isNotEmpty(addUsers) || CollectionUtils.isNotEmpty(delUsers)) { - GroupUserInfo groupUserInfo = new GroupUserInfo(); - groupUserInfo.setGroupName(groupName); - if (CollectionUtils.isNotEmpty(addUsers)) { - groupUserInfo.setAddUsers(addUsers); - if (groupMap.containsKey(groupName) || whiteListGroupMap.containsKey(groupName)) { - // Add users to the computeRole list only if there is a rule defined for the group. - computeRolesForUsers.addAll(addUsers); - } - } - if (CollectionUtils.isNotEmpty(delUsers)) { - groupUserInfo.setDelUsers(delUsers); - if (groupMap.containsKey(groupName)|| whiteListGroupMap.containsKey(groupName)) { - // Add users to the computeRole list only if there is a rule defined for the group. - computeRolesForUsers.addAll(delUsers); - } - } - deltaGroupUserInfoList.add(groupUserInfo); - deltaGroupUsers.put(groupName, newUsers); - } - } - if (CollectionUtils.isNotEmpty(deltaGroupUserInfoList)) { - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.computeGroupUsersDelta(" + deltaGroupUserInfoList + ")"); - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.computeGroupUsersDelta(0)"); - } - } - - return deltaGroupUserInfoList; - } - - private XUserInfo addXUserInfo(String aUserName, Map otherAttrsMap, String otherAttributes) { - XUserInfo xuserInfo = new XUserInfo(); - xuserInfo.setName(aUserName); - xuserInfo.setFirstName(aUserName); - xuserInfo.setDescription(aUserName + " - add from Unix box"); - xuserInfo.setUserSource(SOURCE_EXTERNAL); - xuserInfo.setStatus(STATUS_ENABLED); - xuserInfo.setIsVisible(ISVISIBLE); - List roleList = new ArrayList(); - if (userMap.containsKey(aUserName)) { - roleList.add(userMap.get(aUserName)); - }else{ - roleList.add("ROLE_USER"); - } - xuserInfo.setUserRoleList(roleList); - xuserInfo.setOtherAttributes(otherAttributes); - xuserInfo.setSyncSource(otherAttrsMap.get(UgsyncCommonConstants.SYNC_SOURCE)); - xuserInfo.setOtherAttrsMap(otherAttrsMap); - return xuserInfo; - } - - - private XGroupInfo addXGroupInfo(String aGroupName, Map otherAttrsMap, String otherAttributes) { - - XGroupInfo addGroup = new XGroupInfo(); - - addGroup.setName(aGroupName); - - addGroup.setDescription(aGroupName + " - add from Unix box"); - - addGroup.setGroupType("1"); - addGroup.setIsVisible(ISVISIBLE); - addGroup.setGroupSource(SOURCE_EXTERNAL); - addGroup.setOtherAttributes(otherAttributes); - addGroup.setSyncSource(otherAttrsMap.get(UgsyncCommonConstants.SYNC_SOURCE)); - addGroup.setOtherAttrsMap(otherAttrsMap); - - return addGroup; - } - - private int addOrUpdateDeltaUsers() throws Throwable{ - if (LOG.isDebugEnabled()) { - LOG.debug("PolicyMgrUserGroupBuilder.addOrUpdateDeltaUsers(" + deltaUsers.keySet() + ")"); - } - int ret = 0; - - GetXUserListResponse xUserList = new GetXUserListResponse(); - xUserList.setTotalCount(deltaUsers.size()); - xUserList.setXuserInfoList(new ArrayList<>(deltaUsers.values())); - - if (authenticationType != null - && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) - && SecureClientLogin.isKerberosCredentialExists(principal, - keytab)) { - try { - Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); - final GetXUserListResponse xUserListFinal = xUserList; - ret = Subject.doAs(sub, new PrivilegedAction() { - @Override - public Integer run() { - try { - return getUsers(xUserListFinal); - } catch (Throwable e) { - LOG.error("Failed to add or update Users : ", e); - } - return 0; - } - }); - } catch (Exception e) { - LOG.error("Failed to add or update Users : " , e); - throw e; - } - } else { - ret = getUsers(xUserList); - } - if (LOG.isDebugEnabled()) { - LOG.debug("PolicyMgrUserGroupBuilder.addOrUpdateDeltaUsers(" + deltaUsers.keySet() + ")"); - } - return ret; - } - - - private int getUsers(GetXUserListResponse xUserList) throws Throwable{ - if(LOG.isDebugEnabled()){ - LOG.debug("==> PolicyMgrUserGroupBuilder.getUsers()"); - } - int ret = 0; - int totalCount = xUserList.getTotalCount(); - int uploadedCount = 0; - int pageSize = Integer.parseInt(recordsToPullPerCall); - while (uploadedCount < totalCount) { - checkStatus(); - GetXUserListResponse pagedXUserList = new GetXUserListResponse(); - int pagedXUserListLen = uploadedCount+pageSize; - pagedXUserList.setXuserInfoList(xUserList.getXuserInfoList().subList(uploadedCount, - pagedXUserListLen>totalCount?totalCount:pagedXUserListLen)); - pagedXUserList.setTotalCount(pageSize); - if (pagedXUserList.getXuserInfoList().isEmpty()) { - LOG.info("PolicyMgrUserGroupBuilder.getUsers() done updating users"); - return 1; - } - - String response = getDataFromLdap(PM_ADD_USERS_URI, pagedXUserList); - - if (StringUtils.isNotEmpty(response)) { - try { - ret = Integer.valueOf(response); - uploadedCount += pageSize; - } catch (NumberFormatException e) { - LOG.error("Failed to addOrUpdateUsers " + uploadedCount, e); - throw e; - } - } else { - LOG.error("Failed to addOrUpdateUsers " + uploadedCount ); - throw new Exception("Failed to addOrUpdateUsers" + uploadedCount); - } - LOG.info(String.format("API returned: %s, No. of users uploaded to ranger admin = %s", ret, (uploadedCount>totalCount?totalCount:uploadedCount))); } - - if(LOG.isDebugEnabled()){ - LOG.debug("<== PolicyMgrUserGroupBuilder.getUsers()"); - } - return ret; - } - - private String getDataFromLdap(String uri, Object pagedList) throws Exception { - String response = null; - if (isRangerCookieEnabled) { - response = cookieBasedUploadEntity(pagedList, uri); - } else { - try { - ClientResponse clientRes = ldapUgSyncClient.post(uri, null, pagedList); - if (clientRes != null) { - response = clientRes.getEntity(String.class); - } - } catch (Throwable t) { - LOG.error("Failed to get response, Error is : ", t); - throw t; - } - } - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("REST response from %s : %s", uri, response)); - } - return response; - } - - private int addOrUpdateDeltaGroups() throws Throwable{ - if (LOG.isDebugEnabled()) { - LOG.debug("PolicyMgrUserGroupBuilder.addOrUpdateDeltaGroups(" + deltaGroups.keySet() + ")"); - } - int ret = 0; - - GetXGroupListResponse xGroupList = new GetXGroupListResponse(); - xGroupList.setTotalCount(deltaGroups.size()); - xGroupList.setXgroupInfoList(new ArrayList<>(deltaGroups.values())); - - if (authenticationType != null - && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) - && SecureClientLogin.isKerberosCredentialExists(principal, - keytab)) { - try { - Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); - final GetXGroupListResponse xGroupListFinal = xGroupList; - ret = Subject.doAs(sub, new PrivilegedAction() { - @Override - public Integer run() { - try { - return getGroups(xGroupListFinal); - } catch (Throwable e) { - LOG.error("Failed to add or update groups : ", e); - } - return 0; - } - }); - } catch (Exception e) { - LOG.error("Failed to add or update groups : " , e); - throw e; - } - } else { - ret = getGroups(xGroupList); - } - if (LOG.isDebugEnabled()) { - LOG.debug("PolicyMgrUserGroupBuilder.addOrUpdateDeltaGroups(" + deltaGroups.keySet() + ")"); - } - return ret; - } - - - private int getGroups(GetXGroupListResponse xGroupList) throws Throwable{ - if(LOG.isDebugEnabled()){ - LOG.debug("==> PolicyMgrUserGroupBuilder.getGroups()"); - } - - int ret = 0; - int totalCount = xGroupList.getTotalCount(); - int uploadedCount = 0; - int pageSize = Integer.parseInt(recordsToPullPerCall); - while (uploadedCount < totalCount) { - checkStatus(); - GetXGroupListResponse pagedXGroupList = new GetXGroupListResponse(); - int pagedXGroupListLen = uploadedCount+pageSize; - pagedXGroupList.setXgroupInfoList(xGroupList.getXgroupInfoList().subList(uploadedCount, - pagedXGroupListLen>totalCount?totalCount:pagedXGroupListLen)); - pagedXGroupList.setTotalCount(pageSize); - - String response = getDataFromLdap(PM_ADD_GROUPS_URI, pagedXGroupList); - if (StringUtils.isNotEmpty(response)) { - try { - ret = Integer.valueOf(response); - uploadedCount += pageSize; - } catch (NumberFormatException e) { - LOG.error("Failed to addOrUpdateGroups " + uploadedCount, e ); - throw e; - } - } else { - LOG.error("Failed to addOrUpdateGroups " + uploadedCount ); - throw new Exception("Failed to addOrUpdateGroups " + uploadedCount); - } - LOG.info(String.format("API returned: %s, No. of groups uploaded to ranger admin = %s", ret, (uploadedCount>totalCount?totalCount:uploadedCount))); } - - if(LOG.isDebugEnabled()){ - LOG.debug("<== PolicyMgrUserGroupBuilder.getGroups()"); - } - return ret; - } - - private int addOrUpdateDeltaGroupUsers(List groupUserInfoList) throws Throwable{ - if (LOG.isDebugEnabled()) { - LOG.debug("==> PolicyMgrUserGroupBuilder.addOrUpdateDeltaGroupUsers(" + groupUserInfoList + ")"); - } - int ret = 0; - - if (authenticationType != null - && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) - && SecureClientLogin.isKerberosCredentialExists(principal, - keytab)) { - try { - Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); - final List groupUserInfoListFinal = groupUserInfoList; - ret = Subject.doAs(sub, new PrivilegedAction() { - @Override - public Integer run() { - try { - return getGroupUsers(groupUserInfoListFinal); - } catch (Throwable e) { - LOG.error("Failed to add or update group memberships : ", e); - } - return 0; - } - }); - } catch (Exception e) { - LOG.error("Failed to add or update group memberships : " , e); - throw e; - } - } else { - ret = getGroupUsers(groupUserInfoList); - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.addOrUpdateDeltaGroupUsers(" + groupUserInfoList + ")"); - } - return ret; - } - - - private int getGroupUsers(List groupUserInfoList) throws Throwable{ - if(LOG.isDebugEnabled()){ - LOG.debug("==> PolicyMgrUserGroupBuilder.getGroupUsers()"); - } - int ret = 0; - int totalCount = groupUserInfoList.size(); - int uploadedCount = 0; - int pageSize = Integer.parseInt(recordsToPullPerCall); - while (uploadedCount < totalCount) { - checkStatus(); - - int pagedGroupUserInfoListLen = uploadedCount+pageSize; - List pagedGroupUserInfoList = groupUserInfoList.subList(uploadedCount, - pagedGroupUserInfoListLen>totalCount?totalCount:pagedGroupUserInfoListLen); - - String response = getDataFromLdap(PM_ADD_GROUP_USER_LIST_URI, pagedGroupUserInfoList); - if (StringUtils.isNotEmpty(response)) { - try { - ret = Integer.valueOf(response); - uploadedCount += pageSize; - } catch (NumberFormatException e) { - LOG.error("Failed to addOrUpdateGroupUsers " + uploadedCount, e ); - throw e; - } - } else { - LOG.error("Failed to addOrUpdateGroupUsers " + uploadedCount ); - throw new Exception("Failed to addOrUpdateGroupUsers " + uploadedCount); - } - - LOG.info(String.format("API returned: %s, No. of group memberships uploaded to ranger admin = %s", ret, (uploadedCount>totalCount?totalCount:uploadedCount))); } - - if(LOG.isDebugEnabled()){ - LOG.debug("<== PolicyMgrUserGroupBuilder.getGroupUsers()"); - } - return ret; - } - private String updateRoles(UsersGroupRoleAssignments ugRoleAssignments) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> PolicyMgrUserGroupBuilder.updateUserRole(" + ugRoleAssignments.getUsers() + ")"); - } - - if (authenticationType != null && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) && SecureClientLogin.isKerberosCredentialExists(principal, keytab)){ - try { - Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); - final UsersGroupRoleAssignments result = ugRoleAssignments; - return Subject.doAs(sub, (PrivilegedAction) () -> { - try { - return updateUsersRoles(result); - } catch (Exception e) { - LOG.error("Failed to add User Group Info: ", e); - return null; - } - }); - } catch (Exception e) { - LOG.error("Failed to Authenticate Using given Principal and Keytab : " , e); - } - return null; - } else { - return updateUsersRoles(ugRoleAssignments); - } - } - - private String updateUsersRoles(UsersGroupRoleAssignments ugRoleAssignments) { - if(LOG.isDebugEnabled()){ - LOG.debug("==> PolicyMgrUserGroupBuilder.updateUserRoles(" + ugRoleAssignments.getUsers() + ")"); - } - String response = null; - try { - int totalCount = ugRoleAssignments.getUsers().size(); - int uploadedCount = 0; - int pageSize = Integer.parseInt(recordsToPullPerCall); - while (uploadedCount < totalCount) { - checkStatus(); - int pagedUgRoleAssignmentsListLen = uploadedCount + pageSize; - UsersGroupRoleAssignments pagedUgRoleAssignmentsList = new UsersGroupRoleAssignments(); - pagedUgRoleAssignmentsList.setUsers(ugRoleAssignments.getUsers().subList(uploadedCount, - pagedUgRoleAssignmentsListLen > totalCount ? totalCount : pagedUgRoleAssignmentsListLen)); - pagedUgRoleAssignmentsList.setGroupRoleAssignments(ugRoleAssignments.getGroupRoleAssignments()); - pagedUgRoleAssignmentsList.setUserRoleAssignments(ugRoleAssignments.getUserRoleAssignments()); - pagedUgRoleAssignmentsList.setWhiteListGroupRoleAssignments(ugRoleAssignments.getWhiteListGroupRoleAssignments()); - pagedUgRoleAssignmentsList.setWhiteListUserRoleAssignments(ugRoleAssignments.getWhiteListUserRoleAssignments()); - pagedUgRoleAssignmentsList.setReset(ugRoleAssignments.isReset()); - if ((uploadedCount + pageSize) >= totalCount) { // this is the last iteration of the loop - pagedUgRoleAssignmentsList.setLastPage(true); - } - ClientResponse clientRes; - String url = PM_UPDATE_USERS_ROLES_URI; - - if (LOG.isDebugEnabled()) { - String jsonString = JsonUtils.objectToJson(pagedUgRoleAssignmentsList); - LOG.debug(String.format("Paged RoleAssignments Request to %s: %s", url, jsonString)); - } - - if (isRangerCookieEnabled) { - response = cookieBasedUploadEntity(pagedUgRoleAssignmentsList, url); - } else { - try { - clientRes = ldapUgSyncClient.post(url, null, ugRoleAssignments); - if (clientRes != null) { - response = clientRes.getEntity(String.class); - } - } catch (Throwable t) { - LOG.error("Failed to get response: ", t); - } - } - - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("REST response from %s : %s", url, response)); - } - - if (response == null){ - throw new RuntimeException("Failed to get a REST response!"); - } - - uploadedCount += pageSize; - } - - } catch (Exception e) { - LOG.error("Unable to update roles for: " + ugRoleAssignments.getUsers(), e); - response = null; - } - - if(LOG.isDebugEnabled()){ - LOG.debug("<== PolicyMgrUserGroupBuilder.updateUserRoles(" + response + ")"); - } - return response; - } - - private void addUserGroupAuditInfo(UgsyncAuditInfo auditInfo) throws Throwable { - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("==> PolicyMgrUserGroupBuilder.addUserGroupAuditInfo(%s, %s, %s, %s, %s)", auditInfo.getNoOfNewUsers(), auditInfo.getNoOfNewGroups(), auditInfo.getNoOfModifiedUsers(), auditInfo.getNoOfModifiedGroups(), auditInfo.getSyncSource())); - } - - if (authenticationType != null - && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) - && SecureClientLogin.isKerberosCredentialExists(principal, - keytab)) { - try { - Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); - final UgsyncAuditInfo auditInfoFinal = auditInfo; - Subject.doAs(sub, new PrivilegedAction() { - @Override - public Void run() { - try { - getUserGroupAuditInfo(auditInfoFinal); - } catch (Throwable e) { - LOG.error("Failed to add User : ", e); - } - return null; - } - }); - } catch (Exception e) { - LOG.error("Failed to Authenticate Using given Principal and Keytab : " , e); - } - } else { - getUserGroupAuditInfo(auditInfo); - } - } - - - private void getUserGroupAuditInfo(UgsyncAuditInfo userInfo) throws Throwable{ - if(LOG.isDebugEnabled()){ - LOG.debug("==> PolicyMgrUserGroupBuilder.getUserGroupAuditInfo()"); - } - checkStatus(); - String response = null; - ClientResponse clientRes = null; - - if(isRangerCookieEnabled){ - response = cookieBasedUploadEntity(userInfo, PM_AUDIT_INFO_URI); - } - else { - try { - clientRes = ldapUgSyncClient.post(PM_AUDIT_INFO_URI, null, userInfo); - if (clientRes != null) { - response = clientRes.getEntity(String.class); - } - } - catch(Throwable t){ - LOG.error("Failed to get response, Error is : ", t); - } - } - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("REST response from %s : %s", PM_AUDIT_INFO_URI, response)); - } - JsonUtils.jsonToObject(response, UgsyncAuditInfo.class); - - if(LOG.isDebugEnabled()){ - LOG.debug("AuditInfo Creation successful "); - LOG.debug("<== PolicyMgrUserGroupBuilder.getUserGroupAuditInfo()"); - } - } - - private String cookieBasedUploadEntity(Object obj, String apiURL ) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> PolicyMgrUserGroupBuilder.cookieBasedUploadEntity()"); - } - String response = null; - if (sessionId != null && isValidRangerCookie) { - response = tryUploadEntityWithCookie(obj, apiURL); - } - else{ - response = tryUploadEntityWithCred(obj, apiURL); - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.cookieBasedUploadEntity()"); - } - return response; - } - - private String cookieBasedGetEntity(String apiURL ,int retrievedCount) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> PolicyMgrUserGroupBuilder.cookieBasedGetEntity()"); - } - String response = null; - if (sessionId != null && isValidRangerCookie) { - response = tryGetEntityWithCookie(apiURL,retrievedCount); - } - else{ - response = tryGetEntityWithCred(apiURL,retrievedCount); - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.cookieBasedGetEntity()"); - } - return response; - } - - private String tryUploadEntityWithCookie(Object obj, String apiURL) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> PolicyMgrUserGroupBuilder.tryUploadEntityWithCookie()"); - } - String response = null; - ClientResponse clientResp = null; - try { - clientResp = ldapUgSyncClient.post(apiURL, null, obj, sessionId); - } - catch(Throwable t){ - LOG.error("Failed to get response, Error is : ", t); - } - if (clientResp != null) { - if (!(clientResp.toString().contains(apiURL))) { - clientResp.setStatus(HttpServletResponse.SC_NOT_FOUND); - sessionId = null; - isValidRangerCookie = false; - } else if (clientResp.getStatus() == HttpServletResponse.SC_UNAUTHORIZED) { - sessionId = null; - isValidRangerCookie = false; - } else if (clientResp.getStatus() == HttpServletResponse.SC_NO_CONTENT || clientResp.getStatus() == HttpServletResponse.SC_OK) { - List respCookieList = clientResp.getCookies(); - for (NewCookie cookie : respCookieList) { - if (cookie.getName().equalsIgnoreCase(rangerCookieName)) { - if (!(sessionId.getValue().equalsIgnoreCase(cookie.toCookie().getValue()))) { - sessionId = cookie.toCookie(); - } - isValidRangerCookie = true; - break; - } - } - } - - if (clientResp.getStatus() != HttpServletResponse.SC_OK && clientResp.getStatus() != HttpServletResponse.SC_NO_CONTENT - && clientResp.getStatus() != HttpServletResponse.SC_BAD_REQUEST) { - sessionId = null; - isValidRangerCookie = false; - } - clientResp.bufferEntity(); - response = clientResp.getEntity(String.class); - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.tryUploadEntityWithCookie()"); - } - return response; - } - - - private String tryUploadEntityWithCred(Object obj, String apiURL){ - if(LOG.isDebugEnabled()){ - LOG.debug("==> PolicyMgrUserGroupBuilder.tryUploadEntityInfoWithCred()"); - } - String response = null; - ClientResponse clientResp = null; - - if ( LOG.isDebugEnabled() ) { - String jsonString = JsonUtils.objectToJson(obj); - LOG.debug(String.format("User Group Mapping: %s", jsonString)); - } - - try { - clientResp = ldapUgSyncClient.post(apiURL, null, obj); - } catch(Throwable t) { - LOG.error("Failed to get response, Error is : ", t); - } - if (clientResp != null) { - if (!(clientResp.toString().contains(apiURL))) { - clientResp.setStatus(HttpServletResponse.SC_NOT_FOUND); - } else if (clientResp.getStatus() == HttpServletResponse.SC_UNAUTHORIZED) { - LOG.warn("Credentials response from ranger is 401."); - } else if (clientResp.getStatus() == HttpServletResponse.SC_OK || clientResp.getStatus() == HttpServletResponse.SC_NO_CONTENT) { - cookieList = clientResp.getCookies(); - for (NewCookie cookie : cookieList) { - if (cookie.getName().equalsIgnoreCase(rangerCookieName)) { - sessionId = cookie.toCookie(); - isValidRangerCookie = true; - LOG.info("valid cookie saved "); - break; - } - } - } - if (clientResp.getStatus() != HttpServletResponse.SC_OK && clientResp.getStatus() != HttpServletResponse.SC_NO_CONTENT - && clientResp.getStatus() != HttpServletResponse.SC_BAD_REQUEST) { - sessionId = null; - isValidRangerCookie = false; - } - clientResp.bufferEntity(); - response = clientResp.getEntity(String.class); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.tryUploadEntityInfoWithCred()"); - } - return response; - } - - private String tryGetEntityWithCred(String apiURL, int retrievedCount) { - if(LOG.isDebugEnabled()){ - LOG.debug("==> PolicyMgrUserGroupBuilder.tryGetEntityWithCred()"); - } - String response = null; - ClientResponse clientResp = null; - - Map queryParams = new HashMap(); - queryParams.put("pageSize", recordsToPullPerCall); - queryParams.put("startIndex", String.valueOf(retrievedCount)); - try{ - clientResp = ldapUgSyncClient.get(apiURL, queryParams); - } - catch(Throwable t){ - LOG.error("Failed to get response, Error is : ", t); - } - if (clientResp != null) { - if (!(clientResp.toString().contains(apiURL))) { - clientResp.setStatus(HttpServletResponse.SC_NOT_FOUND); - } else if (clientResp.getStatus() == HttpServletResponse.SC_UNAUTHORIZED) { - LOG.warn("Credentials response from ranger is 401."); - } else if (clientResp.getStatus() == HttpServletResponse.SC_OK || clientResp.getStatus() == HttpServletResponse.SC_NO_CONTENT) { - cookieList = clientResp.getCookies(); - for (NewCookie cookie : cookieList) { - if (cookie.getName().equalsIgnoreCase(rangerCookieName)) { - sessionId = cookie.toCookie(); - isValidRangerCookie = true; - LOG.info("valid cookie saved "); - break; - } - } - } - if (clientResp.getStatus() != HttpServletResponse.SC_OK && clientResp.getStatus() != HttpServletResponse.SC_NO_CONTENT - && clientResp.getStatus() != HttpServletResponse.SC_BAD_REQUEST) { - sessionId = null; - isValidRangerCookie = false; - } - clientResp.bufferEntity(); - response = clientResp.getEntity(String.class); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.tryGetEntityWithCred()"); - } - return response; - } - - - private String tryGetEntityWithCookie(String apiURL, int retrievedCount) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> PolicyMgrUserGroupBuilder.tryGetEntityWithCookie()"); - } - String response = null; - ClientResponse clientResp = null; - - Map queryParams = new HashMap(); - queryParams.put("pageSize", recordsToPullPerCall); - queryParams.put("startIndex", String.valueOf(retrievedCount)); - try { - clientResp = ldapUgSyncClient.get(apiURL, queryParams, sessionId); - } - catch(Throwable t){ - LOG.error("Failed to get response, Error is : ", t); - } - if (clientResp != null) { - if (!(clientResp.toString().contains(apiURL))) { - clientResp.setStatus(HttpServletResponse.SC_NOT_FOUND); - sessionId = null; - isValidRangerCookie = false; - } else if (clientResp.getStatus() == HttpServletResponse.SC_UNAUTHORIZED) { - sessionId = null; - isValidRangerCookie = false; - } else if (clientResp.getStatus() == HttpServletResponse.SC_NO_CONTENT || clientResp.getStatus() == HttpServletResponse.SC_OK) { - List respCookieList = clientResp.getCookies(); - for (NewCookie cookie : respCookieList) { - if (cookie.getName().equalsIgnoreCase(rangerCookieName)) { - if (!(sessionId.getValue().equalsIgnoreCase(cookie.toCookie().getValue()))) { - sessionId = cookie.toCookie(); - } - isValidRangerCookie = true; - break; - } - } - } - - if (clientResp.getStatus() != HttpServletResponse.SC_OK && clientResp.getStatus() != HttpServletResponse.SC_NO_CONTENT - && clientResp.getStatus() != HttpServletResponse.SC_BAD_REQUEST) { - sessionId = null; - isValidRangerCookie = false; - } - clientResp.bufferEntity(); - response = clientResp.getEntity(String.class); - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.tryGetEntityWithCookie()"); - } - return response; - } - - private void getRoleForUserGroups(String userGroupRolesData, Map userMap, Map groupMap) { - if (LOG.isDebugEnabled()) { - LOG.debug("==>> getRoleForUserGroups(" + userGroupRolesData + ")"); - } - Map reverseGroupMap = new LinkedHashMap<>(); - String roleDelimiter = config.getRoleDelimiter(); - String userGroupDelimiter = config.getUserGroupDelimiter(); - String userNameDelimiter = config.getUserGroupNameDelimiter(); - roleDelimiter = StringUtils.isEmpty(roleDelimiter) ? "&" : roleDelimiter; - userGroupDelimiter = StringUtils.isEmpty(userGroupDelimiter) ? ":" : userGroupDelimiter; - userNameDelimiter = StringUtils.isEmpty(userNameDelimiter) ? "," : userNameDelimiter; - StringTokenizer str = new StringTokenizer(userGroupRolesData, - roleDelimiter); - int flag = 0; - String userGroupCheck = null; - String roleName = null; - while (str.hasMoreTokens()) { - flag = 0; - String tokens = str.nextToken(); - if (StringUtils.isNotEmpty(tokens)) { - StringTokenizer userGroupRoles = new StringTokenizer(tokens, - userGroupDelimiter); - if (userGroupRoles != null) { - while (userGroupRoles.hasMoreElements()) { - String userGroupRolesTokens = userGroupRoles - .nextToken(); - if (StringUtils.isNotEmpty(userGroupRolesTokens)) { - flag++; - switch (flag) { - case 1: - roleName = userGroupRolesTokens; - break; - case 2: - userGroupCheck = userGroupRolesTokens; - break; - case 3: - StringTokenizer userGroupNames = new StringTokenizer( - userGroupRolesTokens, userNameDelimiter); - if (userGroupNames != null) { - while (userGroupNames.hasMoreElements()) { - String userGroup = userGroupNames - .nextToken(); - if (StringUtils.isNotEmpty(userGroup)) { - if (userGroupCheck.trim().equalsIgnoreCase("u")) { - userMap.put(userGroup.trim(), roleName.trim()); - } else if (userGroupCheck.trim().equalsIgnoreCase("g")) { - reverseGroupMap.put(userGroup.trim(), - roleName.trim()); - } - } - } - } - break; - default: - userMap.clear(); - reverseGroupMap.clear(); - break; - } - } - } - } - } - } - // Reversing the order of group keys so that the last role specified in the configuration rules is applied when a user belongs to multiple groups with different roles - if (MapUtils.isNotEmpty(reverseGroupMap)) { - List groupNames = new ArrayList<>(reverseGroupMap.keySet()); - Collections.reverse(groupNames); - groupNames.forEach(group -> groupMap.put(group, reverseGroupMap.get(group))); - } - } - - protected String userNameTransform(String userName) { - if (userNameCaseConversionFlag) { - if (userNameLowerCaseFlag) { - userName = userName.toLowerCase(); - } - else { - userName = userName.toUpperCase(); - } - } - - if (userNameRegExInst != null) { - userName = userNameRegExInst.transform(userName); - } - - return userName; - } - - protected String groupNameTransform(String groupName) { - if (groupNameCaseConversionFlag) { - if (groupNameLowerCaseFlag) { - groupName = groupName.toLowerCase(); - } - else { - groupName = groupName.toUpperCase(); - } - } - - if (groupNameRegExInst != null) { - groupName = groupNameRegExInst.transform(groupName); - } - - return groupName; - } - - protected boolean isValidString(final String name) { - if (StringUtils.isBlank(name)) { - return false; - } - if (isUserSyncNameValidationEnabled) { - return USER_OR_GROUP_NAME_VALIDATION_REGEX.matcher(name).matches(); - } - return true; - } - - private void updateDeletedGroups(Map> sourceGroups) throws Throwable { - computeDeletedGroups(sourceGroups); - if (MapUtils.isNotEmpty(deletedGroups)) { - if (updateDeletedGroups() == 0) { - String msg = "Failed to update deleted groups to ranger admin"; - LOG.error(msg); - throw new Exception(msg); - } - } - LOG.info("No. of groups marked for delete = " + deletedGroups.size()); - noOfDeletedGroups += deletedGroups.size(); - } - - private void computeDeletedGroups(Map> sourceGroups) { - if (LOG.isDebugEnabled()) { - LOG.debug("PolicyMgrUserGroupBuilder.computeDeletedGroups(" + sourceGroups.keySet() + ")"); - } - deletedGroups = new HashMap<>(); - // Check if the group from cache exists in the sourceGroups. If not, mark as deleted group. - for (XGroupInfo groupInfo : groupCache.values()) { - Map groupOtherAttrs = groupInfo.getOtherAttrsMap(); - String groupDN = groupOtherAttrs != null ? groupOtherAttrs.get(UgsyncCommonConstants.FULL_NAME) : null; - if (StringUtils.isNotEmpty(groupDN) && !sourceGroups.containsKey(groupDN) - && StringUtils.equalsIgnoreCase(groupOtherAttrs.get(UgsyncCommonConstants.SYNC_SOURCE), currentSyncSource) - && StringUtils.equalsIgnoreCase(groupOtherAttrs.get(UgsyncCommonConstants.LDAP_URL), ldapUrl)) { - if (ISHIDDEN.equals(groupInfo.getIsVisible())) { - groupInfo.setIsVisible(ISHIDDEN); - deletedGroups.put(groupInfo.getName(), groupInfo); - } else { - LOG.info("group " + groupInfo.getName() + " already marked for delete "); - } - } - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.computeDeletedGroups(" + deletedGroups + ")"); - } - } - - private int updateDeletedGroups() throws Throwable{ - if (LOG.isDebugEnabled()) { - LOG.debug("==> PolicyMgrUserGroupBuilder.updateDeletedGroups(" + deletedGroups + ")"); - } - int ret = 0; - - if (authenticationType != null - && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) - && SecureClientLogin.isKerberosCredentialExists(principal, - keytab)) { - try { - Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); - ret = Subject.doAs(sub, new PrivilegedAction() { - @Override - public Integer run() { - try { - return getDeletedGroups(); - } catch (Throwable e) { - LOG.error("Failed to add or update deleted groups : ", e); - } - return 0; - } - }); - } catch (Exception e) { - LOG.error("Failed to add or update deleted groups : " , e); - throw e; - } - } else { - ret = getDeletedGroups(); - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.updateDeletedGroups(" + deletedGroups + ")"); - } - return ret; - } - - - private int getDeletedGroups() throws Throwable{ - if(LOG.isDebugEnabled()){ - LOG.debug("==> PolicyMgrUserGroupBuilder.getDeletedGroups()"); - } - checkStatus(); - int ret = 0; - String response = null; - ClientResponse clientRes = null; - - if(isRangerCookieEnabled){ - response = cookieBasedUploadEntity(deletedGroups.keySet(), PM_UPDATE_DELETED_GROUPS_URI); - } - else { - try { - clientRes = ldapUgSyncClient.post(PM_UPDATE_DELETED_GROUPS_URI, null, deletedGroups.keySet()); - if (clientRes != null) { - response = clientRes.getEntity(String.class); - } - } - catch(Throwable t){ - LOG.error("Failed to get response, Error is : ", t); - } - } - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("REST response from %s : %s", PM_UPDATE_DELETED_GROUPS_URI, response)); - } - if (response != null) { - try { - ret = Integer.valueOf(response); - } catch (NumberFormatException e) { - LOG.error("Failed to update deleted groups", e ); - throw e; - } - } else { - LOG.error("Failed to update deleted groups "); - throw new Exception("Failed to update deleted groups "); - } - - if(LOG.isDebugEnabled()){ - LOG.debug("<== PolicyMgrUserGroupBuilder.getDeletedGroups()" + ret); - } - - return ret; - } - - - private void updateDeletedUsers(Map> sourceUsers) throws Throwable { - computeDeletedUsers(sourceUsers); - if (MapUtils.isNotEmpty(deletedUsers)) { - if (updateDeletedUsers() == 0) { - String msg = "Failed to update deleted users to ranger admin"; - LOG.error(msg); - throw new Exception(msg); - } - } - LOG.info("No. of users marked for delete = " + deletedUsers.size()); - noOfDeletedUsers += deletedUsers.size(); - } - - private void computeDeletedUsers(Map> sourceUsers) { - if (LOG.isDebugEnabled()) { - LOG.debug("PolicyMgrUserGroupBuilder.computeDeletedUsers(" + sourceUsers.keySet() + ")"); - } - deletedUsers = new HashMap<>(); - // Check if the group from cache exists in the sourceGroups. If not, mark as deleted group. - for (XUserInfo userInfo : userCache.values()) { - Map userOtherAttrs = userInfo.getOtherAttrsMap(); - String userDN = userOtherAttrs != null ? userOtherAttrs.get(UgsyncCommonConstants.FULL_NAME) : null; - if (StringUtils.isNotEmpty(userDN) && !sourceUsers.containsKey(userDN) - && StringUtils.equalsIgnoreCase(userOtherAttrs.get(UgsyncCommonConstants.SYNC_SOURCE), currentSyncSource) - && StringUtils.equalsIgnoreCase(userOtherAttrs.get(UgsyncCommonConstants.LDAP_URL), ldapUrl)) { - if (userInfo.getIsVisible() != ISHIDDEN) { - userInfo.setIsVisible(ISHIDDEN); - deletedUsers.put(userInfo.getName(), userInfo); - } else { - LOG.info("user " + userInfo.getName() + " already marked for delete "); - } - } - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.computeDeletedUsers(" + deletedUsers + ")"); - } - } - - private int updateDeletedUsers() throws Throwable{ - if (LOG.isDebugEnabled()) { - LOG.debug("==> PolicyMgrUserGroupBuilder.updateDeletedUsers(" + deletedUsers + ")"); - } - int ret = 0; - - if (authenticationType != null - && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) - && SecureClientLogin.isKerberosCredentialExists(principal, - keytab)) { - try { - Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); - ret = Subject.doAs(sub, new PrivilegedAction() { - @Override - public Integer run() { - try { - return getDeletedUsers(); - } catch (Throwable e) { - LOG.error("Failed to add or update deleted users : ", e); - } - return 0; - } - }); - } catch (Exception e) { - LOG.error("Failed to add or update deleted users : " , e); - throw e; - } - } else { - ret = getDeletedUsers(); - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== PolicyMgrUserGroupBuilder.updateDeletedUsers(" + deletedUsers + ")"); - } - return ret; - } - - - private int getDeletedUsers() throws Throwable{ - if(LOG.isDebugEnabled()){ - LOG.debug("==> PolicyMgrUserGroupBuilder.getDeletedUsers()"); - } - checkStatus(); - int ret = 0; - String response = null; - ClientResponse clientRes = null; - - if(isRangerCookieEnabled){ - response = cookieBasedUploadEntity(deletedUsers.keySet(), PM_UPDATE_DELETED_USERS_URI); - } - else { - try { - clientRes = ldapUgSyncClient.post(PM_UPDATE_DELETED_USERS_URI, null, deletedUsers.keySet()); - if (clientRes != null) { - response = clientRes.getEntity(String.class); - } - } - catch(Throwable t){ - LOG.error("Failed to get response, Error is : ", t); - } - } - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("REST response from %s : %s", PM_UPDATE_DELETED_USERS_URI, response)); - } - if (response != null) { - try { - ret = Integer.valueOf(response); - } catch (NumberFormatException e) { - LOG.error("Failed to update deleted users", e ); - throw e; - } - } else { - LOG.error("Failed to update deleted users "); - throw new Exception("Failed to update deleted users "); - } - - if(LOG.isDebugEnabled()){ - LOG.debug("<== PolicyMgrUserGroupBuilder.getDeletedUsers()" + ret); - } - - return ret; - } - - //Only for testing purpose - protected void setUserSyncNameValidationEnabled(String isNameValidationEnabled) { - config.setProperty(UserGroupSyncConfig.UGSYNC_NAME_VALIDATION_ENABLED, isNameValidationEnabled); - this.isUserSyncNameValidationEnabled = config.isUserSyncNameValidationEnabled(); - } - - // This will throw RuntimeException if Server is not Active - private void checkStatus() throws Exception { - if(!UserGroupSyncConfig.isUgsyncServiceActive()) { - LOG.error(errMsgForInactiveServer); - throw new RuntimeException(errMsgForInactiveServer); - } - } + private static final Logger LOG = LoggerFactory.getLogger(PolicyMgrUserGroupBuilder.class); + /* ***** POST APIs **** */ + private static final String PM_ADD_USERS_URI = "/service/xusers/ugsync/users"; + private static final String PM_ADD_GROUPS_URI = "/service/xusers/ugsync/groups/"; + private static final String PM_ADD_GROUP_USER_LIST_URI = "/service/xusers/ugsync/groupusers"; + private static final String PM_AUDIT_INFO_URI = "/service/xusers/ugsync/auditinfo/"; + private static final String PM_UPDATE_DELETED_USERS_URI = "/service/xusers/ugsync/users/visibility"; + private static final String PM_UPDATE_DELETED_GROUPS_URI = "/service/xusers/ugsync/groups/visibility"; + /* ******************* */ + + /* ***** GET APIs **** */ + public static final String PM_USER_LIST_URI = "/service/xusers/users/"; + public static final String PM_GROUP_LIST_URI = "/service/xusers/groups/"; + public static final String PM_GET_ALL_GROUP_USER_MAP_LIST_URI = "/service/xusers/ugsync/groupusers"; + /* ******************* */ + + /* ***** PUT API **** */ + public static final String PM_UPDATE_USERS_ROLES_URI = "/service/xusers/users/roleassignments"; + /* ******************* */ + + private static final String AUTHENTICATION_TYPE = "hadoop.security.authentication"; + private static final String AUTH_KERBEROS = "kerberos"; + private static final String KERBEROS_PRINCIPAL = "ranger.usersync.kerberos.principal"; + private static final String KERBEROS_KEYTAB = "ranger.usersync.kerberos.keytab"; + private static final String NAME_RULE = "hadoop.security.auth_to_local"; + private static final String SOURCE_EXTERNAL = "1"; + private static final String STATUS_ENABLED = "1"; + private static final String ISVISIBLE = "1"; + private static final String ISHIDDEN = "0"; + private static final Pattern USER_OR_GROUP_NAME_VALIDATION_REGEX = Pattern.compile("^([A-Za-z0-9_]|[\u00C0-\u017F])([a-zA-Z0-9\\s,._\\-+/@= ]|[\u00C0-\u017F])+$", Pattern.CASE_INSENSITIVE); + private static String localHostname = "unknown"; + private static String errMsgForInactiveServer = "This userGroupSync server is not in active state. Cannot commit transaction!"; + private volatile RangerUgSyncRESTClient ldapUgSyncClient; + /* {key: user name in DB} */ + private Map userCache; + /* {key: group name in DB} */ + private Map groupCache; + /* {key: group name, value: set of user names as stored in DB} */ + private Map> groupUsersCache; + /* {key: groupDN, value: group name in DB} */ + private Map groupNameMap; + /* {key: userDN, value: user name in DB} */ + private Map userNameMap; + private Map deltaGroups; + private Map deltaUsers; + private Map> deltaGroupUsers; + private Set computeRolesForUsers; + private Map deletedGroups; + private Map deletedUsers; + private int noOfNewUsers; + private int noOfNewGroups; + private int noOfModifiedUsers; + private int noOfModifiedGroups; + private int noOfDeletedUsers; + private int noOfDeletedGroups; + private boolean isStartupFlag; + private boolean isValidRangerCookie; + private boolean isMockRun; + private boolean userNameCaseConversionFlag; + private boolean groupNameCaseConversionFlag; + private boolean userNameLowerCaseFlag; + private boolean groupNameLowerCaseFlag; + private boolean isRangerCookieEnabled; + private boolean isUserSyncNameValidationEnabled; + private boolean isSyncSourceValidationEnabled; + private String recordsToPullPerCall = "10"; + private String currentSyncSource; + private String ldapUrl; + private String authenticationType; + private String rangerCookieName; + private String policyMgrBaseUrl; + private Cookie sessionId; + String principal; + String keytab; + String policyMgrUserName; + String nameRules; + List cookieList = new ArrayList<>(); + Map userMap = new LinkedHashMap<>(); + Map groupMap = new LinkedHashMap<>(); + Map whiteListUserMap = new LinkedHashMap<>(); + Map whiteListGroupMap = new LinkedHashMap<>(); + + public PolicyMgrUserGroupBuilder() { + super(); + + String userNameCaseConversion = config.getUserNameCaseConversion(); + + if (UserGroupSyncConfig.UGSYNC_NONE_CASE_CONVERSION_VALUE.equalsIgnoreCase(userNameCaseConversion)) { + userNameCaseConversionFlag = false; + } else { + userNameCaseConversionFlag = true; + userNameLowerCaseFlag = UserGroupSyncConfig.UGSYNC_LOWER_CASE_CONVERSION_VALUE.equalsIgnoreCase(userNameCaseConversion); + } + + String groupNameCaseConversion = config.getGroupNameCaseConversion(); + + if (UserGroupSyncConfig.UGSYNC_NONE_CASE_CONVERSION_VALUE.equalsIgnoreCase(groupNameCaseConversion)) { + groupNameCaseConversionFlag = false; + } else { + groupNameCaseConversionFlag = true; + groupNameLowerCaseFlag = UserGroupSyncConfig.UGSYNC_LOWER_CASE_CONVERSION_VALUE.equalsIgnoreCase(groupNameCaseConversion); + } + } + + public static void main(String[] args) throws Throwable { + PolicyMgrUserGroupBuilder ugbuilder = new PolicyMgrUserGroupBuilder(); + ugbuilder.init(); + } + + public synchronized void init() throws Throwable { + isUserSyncNameValidationEnabled = config.isUserSyncNameValidationEnabled(); + isSyncSourceValidationEnabled = config.isSyncSourceValidationEnabled(); + recordsToPullPerCall = config.getMaxRecordsPerAPICall(); + policyMgrBaseUrl = config.getPolicyManagerBaseURL(); + isMockRun = config.isMockRunEnabled(); + isRangerCookieEnabled = config.isUserSyncRangerCookieEnabled(); + rangerCookieName = config.getRangerAdminCookieName(); + groupNameMap = new HashMap<>(); + userNameMap = new HashMap<>(); + userCache = new HashMap<>(); + groupCache = new HashMap<>(); + groupUsersCache = new HashMap<>(); + isStartupFlag = true; + ldapUrl = null; + currentSyncSource = config.getCurrentSyncSource(); + if (StringUtils.equalsIgnoreCase(currentSyncSource, "LDAP/AD")) { + ldapUrl = config.getLdapUrl(); + } + + sessionId = null; + String keyStoreFile = config.getSSLKeyStorePath(); + String trustStoreFile = config.getSSLTrustStorePath(); + String keyStoreFilepwd = config.getSSLKeyStorePathPassword(); + String trustStoreFilepwd = config.getSSLTrustStorePathPassword(); + String keyStoreType = config.getSSLKeyStoreType(); + String trustStoreType = config.getSSLTrustStoreType(); + authenticationType = config.getProperty(AUTHENTICATION_TYPE, "simple"); + try { + principal = SecureClientLogin.getPrincipal(config.getProperty(KERBEROS_PRINCIPAL, ""), localHostname); + } catch (IOException ignored) { // do nothing + } + keytab = config.getProperty(KERBEROS_KEYTAB, ""); + policyMgrUserName = config.getPolicyMgrUserName(); + nameRules = config.getProperty(NAME_RULE, "DEFAULT"); + ldapUgSyncClient = new RangerUgSyncRESTClient(policyMgrBaseUrl, keyStoreFile, keyStoreFilepwd, keyStoreType, trustStoreFile, trustStoreFilepwd, trustStoreType, authenticationType, principal, keytab, policyMgrUserName, config.getPolicyMgrPassword()); + + String userGroupRoles = config.getGroupRoleRules(); + if (userGroupRoles != null && !userGroupRoles.isEmpty()) { + getRoleForUserGroups(userGroupRoles, userMap, groupMap); + } + String whiteListUserRoles = config.getWhileListUserRoleRules(); + if (whiteListUserRoles != null && !whiteListUserRoles.isEmpty()) { + getRoleForUserGroups(whiteListUserRoles, whiteListUserMap, whiteListGroupMap); + } + String policyMgrUserRole = whiteListUserMap.get(policyMgrUserName); + if (!StringUtils.equalsIgnoreCase(policyMgrUserRole, "ROLE_SYS_ADMIN")) { + whiteListUserMap.put(policyMgrUserName, "ROLE_SYS_ADMIN"); + } + + LOG.debug("Entries in group role assignments: {}", groupMap); + LOG.debug("Entries in whitelist group role assignments: {}", whiteListGroupMap); + + buildUserGroupInfo(); + LOG.debug(String.format("PolicyMgrUserGroupBuilderOld.init()==> policyMgrBaseUrl: %s, KeyStore File: %s, TrustStore File: %s, Authentication Type: %s", policyMgrBaseUrl, keyStoreFile, trustStoreFile, authenticationType)); + } + + @Override + public void postUserGroupAuditInfo(UgsyncAuditInfo ugsyncAuditInfo) throws Throwable { + ugsyncAuditInfo.setNoOfNewUsers(Integer.toUnsignedLong(noOfNewUsers)); + ugsyncAuditInfo.setNoOfNewGroups(Integer.toUnsignedLong(noOfNewGroups)); + ugsyncAuditInfo.setNoOfModifiedUsers(Integer.toUnsignedLong(noOfModifiedUsers)); + ugsyncAuditInfo.setNoOfModifiedGroups(Integer.toUnsignedLong(noOfModifiedGroups)); + int noOfCachedUsers = userCache.size(); + int noOfCachedGroups = groupCache.size(); + switch (ugsyncAuditInfo.getSyncSource()) { + case "LDAP/AD": + ugsyncAuditInfo.getLdapSyncSourceInfo().setTotalUsersSynced(noOfCachedUsers); + ugsyncAuditInfo.getLdapSyncSourceInfo().setTotalGroupsSynced(noOfCachedGroups); + ugsyncAuditInfo.getLdapSyncSourceInfo().setTotalUsersDeleted(noOfDeletedUsers); + ugsyncAuditInfo.getLdapSyncSourceInfo().setTotalGroupsDeleted(noOfDeletedGroups); + break; + case "Unix": + ugsyncAuditInfo.getUnixSyncSourceInfo().setTotalUsersSynced(noOfCachedUsers); + ugsyncAuditInfo.getUnixSyncSourceInfo().setTotalGroupsSynced(noOfCachedGroups); + ugsyncAuditInfo.getUnixSyncSourceInfo().setTotalUsersDeleted(noOfDeletedUsers); + ugsyncAuditInfo.getUnixSyncSourceInfo().setTotalGroupsDeleted(noOfDeletedGroups); + break; + case "File": + ugsyncAuditInfo.getFileSyncSourceInfo().setTotalUsersSynced(noOfCachedUsers); + ugsyncAuditInfo.getFileSyncSourceInfo().setTotalGroupsSynced(noOfCachedGroups); + ugsyncAuditInfo.getFileSyncSourceInfo().setTotalUsersDeleted(noOfDeletedUsers); + ugsyncAuditInfo.getFileSyncSourceInfo().setTotalGroupsDeleted(noOfDeletedGroups); + break; + default: + break; + } + + if (!isMockRun) { + checkStatus(); + addUserGroupAuditInfo(ugsyncAuditInfo); + } + } + + @Override + public void addOrUpdateUsersGroups(Map> sourceGroups, Map> sourceUsers, Map> sourceGroupUsers, boolean computeDeletes) throws Throwable { + checkStatus(); + + noOfNewUsers = 0; + noOfNewGroups = 0; + noOfModifiedUsers = 0; + noOfModifiedGroups = 0; + computeRolesForUsers = new HashSet<>(); + + if (!isStartupFlag && computeDeletes) { + LOG.info("Computing deleted users/groups"); + if (MapUtils.isNotEmpty(sourceGroups)) { + updateDeletedGroups(sourceGroups); + } + if (MapUtils.isNotEmpty(sourceUsers)) { + updateDeletedUsers(sourceUsers); + } + + if (MapUtils.isNotEmpty(deletedGroups)) { + groupCache.putAll(deletedGroups); + } + + if (MapUtils.isNotEmpty(deletedUsers)) { + userCache.putAll(deletedUsers); + } + } + + if (MapUtils.isNotEmpty(sourceGroups)) { + addOrUpdateGroups(sourceGroups); + } + if (MapUtils.isNotEmpty(sourceUsers)) { + addOrUpdateUsers(sourceUsers); + } + + if (MapUtils.isNotEmpty(sourceGroupUsers)) { + addOrUpdateGroupUsers(sourceGroupUsers); + } + + if (isStartupFlag) { + // This is to handle any config changes for role assignments that might impact existing users in ranger db + if (MapUtils.isNotEmpty(whiteListUserMap)) { + LOG.debug("adding {} for computing roles during startup", whiteListUserMap.keySet()); + computeRolesForUsers.addAll(whiteListUserMap.keySet()); // Add all the user defined in the whitelist role assignment rules + } + if (MapUtils.isNotEmpty(whiteListGroupMap)) { + for (String groupName : whiteListGroupMap.keySet()) { + Set groupUsers = null; + if (CollectionUtils.isNotEmpty(groupUsersCache.get(groupName))) { + groupUsers = new HashSet<>(groupUsersCache.get(groupName)); + } else if (CollectionUtils.isNotEmpty(deltaGroupUsers.get(groupName))) { + groupUsers = new HashSet<>(deltaGroupUsers.get(groupName)); + } + if (groupUsers != null) { + LOG.debug("adding {} from {} for computing roles during startup", groupUsers, groupName); + computeRolesForUsers.addAll(groupUsers); + } + } + } + + if (MapUtils.isNotEmpty(userMap)) { + LOG.debug("adding {} for computing roles during startup", userMap.keySet()); + computeRolesForUsers.addAll(userMap.keySet()); // Add all the user defined in the role assignment rules + } + if (MapUtils.isNotEmpty(groupMap)) { + for (String groupName : groupMap.keySet()) { + Set groupUsers = null; + if (CollectionUtils.isNotEmpty(groupUsersCache.get(groupName))) { + groupUsers = new HashSet<>(groupUsersCache.get(groupName)); + } else if (CollectionUtils.isNotEmpty(deltaGroupUsers.get(groupName))) { + groupUsers = new HashSet<>(deltaGroupUsers.get(groupName)); + } + if (groupUsers != null) { + LOG.debug("adding {} from {} for computing roles during startup", groupUsers, groupName); + computeRolesForUsers.addAll(groupUsers); + } + } + } + } + + if (CollectionUtils.isNotEmpty(computeRolesForUsers)) { + updateUserRoles(); + } + isStartupFlag = false; + + LOG.debug("Update cache"); + if (MapUtils.isNotEmpty(deltaGroups)) { + groupCache.putAll(deltaGroups); + } + if (MapUtils.isNotEmpty(deltaUsers)) { + userCache.putAll(deltaUsers); + } + if (MapUtils.isNotEmpty(deltaGroupUsers)) { + groupUsersCache.putAll(deltaGroupUsers); + } + } + + protected String userNameTransform(String userName) { + if (userNameCaseConversionFlag) { + if (userNameLowerCaseFlag) { + userName = userName.toLowerCase(); + } else { + userName = userName.toUpperCase(); + } + } + + if (userNameRegExInst != null) { + userName = userNameRegExInst.transform(userName); + } + + return userName; + } + + protected String groupNameTransform(String groupName) { + if (groupNameCaseConversionFlag) { + if (groupNameLowerCaseFlag) { + groupName = groupName.toLowerCase(); + } else { + groupName = groupName.toUpperCase(); + } + } + + if (groupNameRegExInst != null) { + groupName = groupNameRegExInst.transform(groupName); + } + + return groupName; + } + + protected boolean isValidString(final String name) { + if (StringUtils.isBlank(name)) { + return false; + } + if (isUserSyncNameValidationEnabled) { + return USER_OR_GROUP_NAME_VALIDATION_REGEX.matcher(name).matches(); + } + return true; + } + + //Only for testing purpose + protected void setUserSyncNameValidationEnabled(String isNameValidationEnabled) { + config.setProperty(UserGroupSyncConfig.UGSYNC_NAME_VALIDATION_ENABLED, isNameValidationEnabled); + this.isUserSyncNameValidationEnabled = config.isUserSyncNameValidationEnabled(); + } + + private void buildUserGroupInfo() throws Throwable { + if (authenticationType != null && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) && SecureClientLogin.isKerberosCredentialExists(principal, keytab)) { + LOG.info(String.format("Using principal: %s and keytab: %s", principal, keytab)); + Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); + Boolean isInitDone = Subject.doAs(sub, new PrivilegedAction() { + @Override + public Boolean run() { + try { + buildGroupList(); + buildUserList(); + buildGroupUserLinkList(); + } catch (Throwable e) { + LOG.error("Failed to build Users and Groups from Ranger admin : ", e); + return false; + } + return true; + } + }); + if (isInitDone.booleanValue() == false) { + String msg = ("Failed to build Users and Groups from Ranger admin"); + LOG.error(msg); + throw new Exception(msg); + } + } else { + buildGroupList(); + buildUserList(); + buildGroupUserLinkList(); + } + } + + private void buildGroupList() throws Throwable { + LOG.debug("==> PolicyMgrUserGroupBuilder.buildGroupList()"); + int totalCount = 100; + int retrievedCount = 0; + + while (retrievedCount < totalCount) { + String response = null; + ClientResponse clientResp = null; + + Map queryParams = new HashMap<>(); + queryParams.put("pageSize", recordsToPullPerCall); + queryParams.put("startIndex", String.valueOf(retrievedCount)); + + if (isRangerCookieEnabled) { + response = cookieBasedGetEntity(PM_GROUP_LIST_URI, retrievedCount); + } else { + try { + clientResp = ldapUgSyncClient.get(PM_GROUP_LIST_URI, queryParams); + if (clientResp != null) { + response = clientResp.getEntity(String.class); + } + } catch (Exception e) { + LOG.error("Failed to get groups from Ranger, Error is : ", e.getMessage()); + throw e; + } + } + LOG.debug(String.format("REST response from %s : %s", PM_GROUP_LIST_URI, response)); + GetXGroupListResponse groupList = JsonUtils.jsonToObject(response, GetXGroupListResponse.class); + + totalCount = groupList.getTotalCount(); + + if (groupList.getXgroupInfoList() != null) { + for (XGroupInfo g : groupList.getXgroupInfoList()) { + LOG.debug(String.format("GROUP: Id: %s, Name: %s, Description: %s", g.getId(), g.getName(), g.getDescription())); + if (null != g.getOtherAttributes()) { + g.setOtherAttrsMap(JsonUtils.jsonToObject(g.getOtherAttributes(), Map.class)); + } + groupCache.put(g.getName(), g); + } + retrievedCount = groupCache.size(); + } + LOG.info("PolicyMgrUserGroupBuilder.buildGroupList(): No. of groups retrieved from ranger admin {}", retrievedCount); + } + + LOG.debug("<== PolicyMgrUserGroupBuilder.buildGroupList()"); + } + + private void buildUserList() throws Throwable { + LOG.debug("==> PolicyMgrUserGroupBuilder.buildUserList()"); + int totalCount = 100; + int retrievedCount = 0; + + while (retrievedCount < totalCount) { + String response = null; + ClientResponse clientResp = null; + + Map queryParams = new HashMap<>(); + queryParams.put("pageSize", recordsToPullPerCall); + queryParams.put("startIndex", String.valueOf(retrievedCount)); + + if (isRangerCookieEnabled) { + response = cookieBasedGetEntity(PM_USER_LIST_URI, retrievedCount); + } else { + try { + clientResp = ldapUgSyncClient.get(PM_USER_LIST_URI, queryParams); + if (clientResp != null) { + response = clientResp.getEntity(String.class); + } + } catch (Exception e) { + LOG.error("Failed to get users from Ranger admin, Error is : {}", e.getMessage()); + throw e; + } + } + + LOG.debug(String.format("REST response from %s : %s", PM_USER_LIST_URI, response)); + GetXUserListResponse userList = JsonUtils.jsonToObject(response, GetXUserListResponse.class); + totalCount = userList.getTotalCount(); + + if (userList.getXuserInfoList() != null) { + for (XUserInfo u : userList.getXuserInfoList()) { + LOG.debug(String.format("USER: Id: %s, Name: %s, Description: %s", u.getId(), u.getName(), u.getDescription())); + if (null != u.getOtherAttributes()) { + u.setOtherAttrsMap(JsonUtils.jsonToObject(u.getOtherAttributes(), Map.class)); + } + userCache.put(u.getName(), u); + } + retrievedCount = userCache.size(); + } + LOG.info("PolicyMgrUserGroupBuilder.buildUserList(): No. of users retrieved from ranger admin = {}", retrievedCount); + } + LOG.debug("<== PolicyMgrUserGroupBuilder.buildUserList()"); + } + + private void buildGroupUserLinkList() throws Throwable { + LOG.debug("==> PolicyMgrUserGroupBuilder.buildGroupUserLinkList()"); + + String response = null; + ClientResponse clientResp = null; + + if (isRangerCookieEnabled) { + response = cookieBasedGetEntity(PM_GET_ALL_GROUP_USER_MAP_LIST_URI, 0); + } else { + try { + clientResp = ldapUgSyncClient.get(PM_GET_ALL_GROUP_USER_MAP_LIST_URI, null); + if (clientResp != null) { + response = clientResp.getEntity(String.class); + } + } catch (Exception e) { + LOG.error("Failed to get response, group user mappings from Ranger admin. Error is : {}", e.getMessage()); + throw e; + } + } + LOG.debug(String.format("REST response from %s : %s", PM_GET_ALL_GROUP_USER_MAP_LIST_URI, response)); + groupUsersCache = JsonUtils.jsonToObject(response, Map.class); + if (MapUtils.isEmpty(groupUsersCache)) { + groupUsersCache = new HashMap<>(); + } + + LOG.debug("Group User List : {}", groupUsersCache.values()); + LOG.debug("<== PolicyMgrUserGroupBuilder.buildGroupUserLinkList()"); + } + + private void addOrUpdateUsers(Map> sourceUsers) throws Throwable { + computeUserDelta(sourceUsers); + if (MapUtils.isNotEmpty(deltaUsers)) { + if (addOrUpdateDeltaUsers() == 0) { + String msg = "Failed to addorUpdate users to ranger admin"; + LOG.error(msg); + throw new Exception(msg); + } + } + } + + private void addOrUpdateGroups(Map> sourceGroups) throws Throwable { + computeGroupDelta(sourceGroups); + if (MapUtils.isNotEmpty(deltaGroups)) { + if (addOrUpdateDeltaGroups() == 0) { + String msg = "Failed to addorUpdate groups to ranger admin"; + LOG.error(msg); + throw new Exception(msg); + } + } + } + + private void addOrUpdateGroupUsers(Map> sourceGroupUsers) throws Throwable { + List groupUserInfoList = computeGroupUsersDelta(sourceGroupUsers); + if (CollectionUtils.isNotEmpty(groupUserInfoList)) { + noOfModifiedGroups += groupUserInfoList.size(); + if (addOrUpdateDeltaGroupUsers(groupUserInfoList) == 0) { + String msg = "Failed to addorUpdate group memberships to ranger admin"; + LOG.error(msg); + throw new Exception(msg); + } + } + } + + private void updateUserRoles() throws Throwable { + UsersGroupRoleAssignments ugRoleAssignments = new UsersGroupRoleAssignments(); + List allUsers = new ArrayList<>(computeRolesForUsers); + + ugRoleAssignments.setUsers(allUsers); + ugRoleAssignments.setGroupRoleAssignments(groupMap); + ugRoleAssignments.setUserRoleAssignments(userMap); + ugRoleAssignments.setWhiteListUserRoleAssignments(whiteListUserMap); + ugRoleAssignments.setWhiteListGroupRoleAssignments(whiteListGroupMap); + ugRoleAssignments.setReset(isStartupFlag); + String updatedUsers = updateRoles(ugRoleAssignments); + if (updatedUsers == null) { + String msg = "Unable to update roles for " + allUsers; + LOG.error(msg); + throw new Exception(msg); + } + } + + private T setOtherAttributes(T uginfo, String syncSource, Map otherAttrsMap, String otherAttributes) { + if (uginfo instanceof XUserInfo) { + XUserInfo xUserInfo = ((XUserInfo) uginfo); + xUserInfo.setSyncSource(syncSource); + xUserInfo.setOtherAttrsMap(otherAttrsMap); + xUserInfo.setOtherAttributes(otherAttributes); + return ((T) xUserInfo); + } else if (uginfo instanceof XGroupInfo) { + XGroupInfo xGroupInfo = ((XGroupInfo) uginfo); + xGroupInfo.setSyncSource(syncSource); + xGroupInfo.setOtherAttrsMap(otherAttrsMap); + xGroupInfo.setOtherAttributes(otherAttributes); + return ((T) xGroupInfo); + } else { + return null; + } + } + + private void computeGroupDelta(Map> sourceGroups) { + LOG.debug("PolicyMgrUserGroupBuilder.computeGroupDelta({})", sourceGroups.keySet()); + deltaGroups = new HashMap<>(); + // Check if the group exists in cache. If not, mark as new group else check if other attributes are updated and mark as updated group + for (String groupDN : sourceGroups.keySet()) { + Map newGroupAttrs = sourceGroups.get(groupDN); + String newGroupAttrsStr = JsonUtils.objectToJson(newGroupAttrs); + String groupName = groupNameMap.get(groupDN); + if (StringUtils.isEmpty(groupName)) { + groupName = groupNameTransform(newGroupAttrs.get(UgsyncCommonConstants.ORIGINAL_NAME).trim()); + } + + if (!isValidString(groupName)) { + LOG.warn("Ignoring invalid group {} Full name = {}", groupName, groupDN); + continue; + } + + if (!groupCache.containsKey(groupName)) { + XGroupInfo newGroup = addXGroupInfo(groupName, newGroupAttrs, newGroupAttrsStr); + deltaGroups.put(groupName, newGroup); + noOfNewGroups++; + groupNameMap.put(groupDN, groupName); + } else { + XGroupInfo curGroup = groupCache.get(groupName); + String curSyncSource = curGroup.getSyncSource(); + String curGroupAttrsStr = curGroup.getOtherAttributes(); + Map curGroupAttrs = curGroup.getOtherAttrsMap(); + String curGroupDN = MapUtils.isEmpty(curGroupAttrs) ? groupName : curGroupAttrs.get(UgsyncCommonConstants.FULL_NAME); + String newSyncSource = newGroupAttrs.get(UgsyncCommonConstants.SYNC_SOURCE); + + if (isStartupFlag && !isSyncSourceValidationEnabled && (!StringUtils.equalsIgnoreCase(curSyncSource, newSyncSource))) { + LOG.debug("[{}]: SyncSource updated to {}, previous value: {}", groupName, newSyncSource, curSyncSource); + curGroup = setOtherAttributes(curGroup, newSyncSource, newGroupAttrs, newGroupAttrsStr); + deltaGroups.put(groupName, curGroup); + noOfModifiedGroups++; + groupNameMap.put(groupDN, groupName); + } else { + if (MapUtils.isNotEmpty(curGroupAttrs) && !StringUtils.equalsIgnoreCase(groupDN, curGroupDN)) { // skip update + LOG.debug("[{}]: SyncSource update skipped, current group DN = {} new user DN = {}", groupName, curGroupDN, groupDN); + + if (StringUtils.equalsIgnoreCase(curGroupAttrsStr, newGroupAttrsStr)) { + groupNameMap.put(groupDN, groupName); + } + continue; + } + + if (StringUtils.isEmpty(curSyncSource) || (!StringUtils.equalsIgnoreCase(curGroupAttrsStr, newGroupAttrsStr) && StringUtils.equalsIgnoreCase(curSyncSource, newSyncSource))) { // update + if (StringUtils.isEmpty(curSyncSource)) { + LOG.debug("[{}]: SyncSource updated to {}, previously empty", groupName, newSyncSource); + } else { + LOG.debug("[{}]: Other Attributes updated!", groupName); + } + curGroup = setOtherAttributes(curGroup, newSyncSource, newGroupAttrs, newGroupAttrsStr); + deltaGroups.put(groupName, curGroup); + noOfModifiedGroups++; + groupNameMap.put(groupDN, groupName); + } else { + if (!StringUtils.equalsIgnoreCase(curSyncSource, newSyncSource)) { + LOG.debug("[{}]: Different sync source exists, update skipped!", groupName); + } else { + LOG.debug("[{}]: No change, update skipped!", groupName); + } + } + + if (StringUtils.equalsIgnoreCase(curGroupAttrsStr, newGroupAttrsStr)) { + groupNameMap.put(groupDN, groupName); + } + } + } + } + LOG.debug("<== PolicyMgrUserGroupBuilder.computeGroupDelta({})", deltaGroups.keySet()); + } + + private void computeUserDelta(Map> sourceUsers) { + LOG.debug("==> PolicyMgrUserGroupBuilder.computeUserDelta({})", sourceUsers.keySet()); + deltaUsers = new HashMap<>(); + // Check if the user exists in cache. If not, mark as new user else check if other attributes are updated and mark as updated user + for (Map.Entry> sourceUser : sourceUsers.entrySet()) { + String userDN = sourceUser.getKey(); + Map newUserAttrs = sourceUser.getValue(); + String newUserAttrsStr = JsonUtils.objectToJson(newUserAttrs); + String userName = userNameMap.get(userDN); + if (StringUtils.isEmpty(userName)) { + userName = userNameTransform(newUserAttrs.get(UgsyncCommonConstants.ORIGINAL_NAME).trim()); + } + + if (!isValidString(userName)) { + LOG.warn("Ignoring invalid user {} Full name = {}", userName, userDN); + continue; + } + + if (!userCache.containsKey(userName)) { + XUserInfo newUser = addXUserInfo(userName, newUserAttrs, newUserAttrsStr); + deltaUsers.put(userName, newUser); + noOfNewUsers++; + userNameMap.put(userDN, userName); + } else { + // no updates allowed for rangerusersync and admin + if (StringUtils.equalsIgnoreCase(policyMgrUserName, userName) || StringUtils.equalsIgnoreCase("admin", userName)) { + LOG.debug("[{}]: SyncSource update skipped!", userName); + continue; + } + + XUserInfo curUser = userCache.get(userName); + String curSyncSource = curUser.getSyncSource(); + String curUserAttrsStr = curUser.getOtherAttributes(); + Map curUserAttrs = curUser.getOtherAttrsMap(); + String curUserDN = MapUtils.isEmpty(curUserAttrs) ? userName : curUserAttrs.get(UgsyncCommonConstants.FULL_NAME); + String newSyncSource = newUserAttrs.get(UgsyncCommonConstants.SYNC_SOURCE); + if (isStartupFlag && !isSyncSourceValidationEnabled && (!StringUtils.equalsIgnoreCase(curSyncSource, newSyncSource))) { + LOG.debug("[{}]: SyncSource updated to {}, previous value: {}", userName, newSyncSource, curSyncSource); + curUser = setOtherAttributes(curUser, newSyncSource, newUserAttrs, newUserAttrsStr); + curUser.setUserSource(SOURCE_EXTERNAL); + deltaUsers.put(userName, curUser); + noOfModifiedGroups++; + userNameMap.put(userDN, userName); + } else { + if (MapUtils.isNotEmpty(curUserAttrs) && !StringUtils.equalsIgnoreCase(userDN, curUserDN)) { // skip update + // Same username with different DN already exists + LOG.debug("[{}]: SyncSource update skipped, current user DN = {} new user DN = {}", userName, curUserDN, userDN); + + if (StringUtils.equalsIgnoreCase(curUserAttrsStr, newUserAttrsStr)) { + userNameMap.put(userDN, userName); + } + continue; + } + + if (StringUtils.isEmpty(curSyncSource) || (!StringUtils.equalsIgnoreCase(curUserAttrsStr, newUserAttrsStr) && StringUtils.equalsIgnoreCase(curSyncSource, newSyncSource))) { // update + if (StringUtils.isEmpty(curSyncSource)) { + LOG.debug("[{}]: SyncSource updated to {}, previously empty", userName, newSyncSource); + } else { + LOG.debug("[{}]: Other Attributes updated!", userName); + } + curUser = setOtherAttributes(curUser, newSyncSource, newUserAttrs, newUserAttrsStr); + curUser.setUserSource(SOURCE_EXTERNAL); + deltaUsers.put(userName, curUser); + noOfModifiedUsers++; + userNameMap.put(userDN, userName); + } else { + if (!StringUtils.equalsIgnoreCase(curSyncSource, newSyncSource)) { + LOG.debug("[{}]: Different sync source exists, update skipped!", userName); + } else { + LOG.debug("[{}]: No change, update skipped!", userName); + } + } + + if (StringUtils.equalsIgnoreCase(curUserAttrsStr, newUserAttrsStr)) { + userNameMap.put(userDN, userName); + } + } + } + } + LOG.debug("<== PolicyMgrUserGroupBuilder.computeUserDelta({})", deltaUsers.keySet()); + } + + private List computeGroupUsersDelta(Map> sourceGroupUsers) { + LOG.debug("==> PolicyMgrUserGroupBuilder.computeGroupUsersDelta({})", sourceGroupUsers.keySet()); + deltaGroupUsers = new HashMap<>(); + List deltaGroupUserInfoList = new ArrayList<>(); + for (String groupDN : sourceGroupUsers.keySet()) { + String groupName = groupNameMap.get(groupDN); + if (StringUtils.isEmpty(groupName)) { + LOG.debug("Ignoring group membership update for {}", groupDN); + continue; + } + Set oldUsers = new HashSet<>(); + Set newUsers = new HashSet<>(); + Set addUsers = new HashSet<>(); + Set delUsers = new HashSet<>(); + if (CollectionUtils.isNotEmpty(groupUsersCache.get(groupName))) { + oldUsers = new HashSet<>(groupUsersCache.get(groupName)); + } + + for (String userDN : sourceGroupUsers.get(groupDN)) { + String userName = userNameMap.get(userDN); + if (!StringUtils.isEmpty(userName)) { + newUsers.add(userName); + if (CollectionUtils.isEmpty(oldUsers) || !oldUsers.contains(userName)) { + addUsers.add(userName); + } + } + } + + if (CollectionUtils.isNotEmpty(oldUsers)) { + for (String userName : oldUsers) { + if (CollectionUtils.isEmpty(newUsers) || !newUsers.contains(userName)) { + delUsers.add(userName); + } + } + } + + if (CollectionUtils.isNotEmpty(addUsers) || CollectionUtils.isNotEmpty(delUsers)) { + GroupUserInfo groupUserInfo = new GroupUserInfo(); + groupUserInfo.setGroupName(groupName); + if (CollectionUtils.isNotEmpty(addUsers)) { + groupUserInfo.setAddUsers(addUsers); + if (groupMap.containsKey(groupName) || whiteListGroupMap.containsKey(groupName)) { + // Add users to the computeRole list only if there is a rule defined for the group. + computeRolesForUsers.addAll(addUsers); + } + } + if (CollectionUtils.isNotEmpty(delUsers)) { + groupUserInfo.setDelUsers(delUsers); + if (groupMap.containsKey(groupName) || whiteListGroupMap.containsKey(groupName)) { + // Add users to the computeRole list only if there is a rule defined for the group. + computeRolesForUsers.addAll(delUsers); + } + } + deltaGroupUserInfoList.add(groupUserInfo); + deltaGroupUsers.put(groupName, newUsers); + } + } + if (CollectionUtils.isNotEmpty(deltaGroupUserInfoList)) { + LOG.debug("<== PolicyMgrUserGroupBuilder.computeGroupUsersDelta({})", deltaGroupUserInfoList); + } else { + LOG.debug("<== PolicyMgrUserGroupBuilder.computeGroupUsersDelta(0)"); + } + + return deltaGroupUserInfoList; + } + + private XUserInfo addXUserInfo(String aUserName, Map otherAttrsMap, String otherAttributes) { + XUserInfo xuserInfo = new XUserInfo(); + xuserInfo.setName(aUserName); + xuserInfo.setFirstName(aUserName); + xuserInfo.setDescription(aUserName + " - add from Unix box"); + xuserInfo.setUserSource(SOURCE_EXTERNAL); + xuserInfo.setStatus(STATUS_ENABLED); + xuserInfo.setIsVisible(ISVISIBLE); + List roleList = new ArrayList(); + if (userMap.containsKey(aUserName)) { + roleList.add(userMap.get(aUserName)); + } else { + roleList.add("ROLE_USER"); + } + xuserInfo.setUserRoleList(roleList); + xuserInfo.setOtherAttributes(otherAttributes); + xuserInfo.setSyncSource(otherAttrsMap.get(UgsyncCommonConstants.SYNC_SOURCE)); + xuserInfo.setOtherAttrsMap(otherAttrsMap); + return xuserInfo; + } + + private XGroupInfo addXGroupInfo(String aGroupName, Map otherAttrsMap, String otherAttributes) { + XGroupInfo addGroup = new XGroupInfo(); + + addGroup.setName(aGroupName); + + addGroup.setDescription(aGroupName + " - add from Unix box"); + + addGroup.setGroupType("1"); + addGroup.setIsVisible(ISVISIBLE); + addGroup.setGroupSource(SOURCE_EXTERNAL); + addGroup.setOtherAttributes(otherAttributes); + addGroup.setSyncSource(otherAttrsMap.get(UgsyncCommonConstants.SYNC_SOURCE)); + addGroup.setOtherAttrsMap(otherAttrsMap); + + return addGroup; + } + + private int addOrUpdateDeltaUsers() throws Throwable { + LOG.debug("PolicyMgrUserGroupBuilder.addOrUpdateDeltaUsers({})", deltaUsers.keySet()); + int ret = 0; + + GetXUserListResponse xUserList = new GetXUserListResponse(); + xUserList.setTotalCount(deltaUsers.size()); + xUserList.setXuserInfoList(new ArrayList<>(deltaUsers.values())); + + if (authenticationType != null && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) && SecureClientLogin.isKerberosCredentialExists(principal, keytab)) { + try { + Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); + final GetXUserListResponse xUserListFinal = xUserList; + ret = Subject.doAs(sub, new PrivilegedAction() { + @Override + public Integer run() { + try { + return getUsers(xUserListFinal); + } catch (Throwable e) { + LOG.error("Failed to add or update Users : ", e); + } + return 0; + } + }); + } catch (Exception e) { + LOG.error("Failed to add or update Users : ", e); + throw e; + } + } else { + ret = getUsers(xUserList); + } + LOG.debug("PolicyMgrUserGroupBuilder.addOrUpdateDeltaUsers({})", deltaUsers.keySet()); + return ret; + } + + private int getUsers(GetXUserListResponse xUserList) throws Throwable { + LOG.debug("==> PolicyMgrUserGroupBuilder.getUsers()"); + int ret = 0; + int totalCount = xUserList.getTotalCount(); + int uploadedCount = 0; + int pageSize = Integer.parseInt(recordsToPullPerCall); + while (uploadedCount < totalCount) { + checkStatus(); + GetXUserListResponse pagedXUserList = new GetXUserListResponse(); + int pagedXUserListLen = uploadedCount + pageSize; + pagedXUserList.setXuserInfoList(xUserList.getXuserInfoList().subList(uploadedCount, + pagedXUserListLen > totalCount ? totalCount : pagedXUserListLen)); + pagedXUserList.setTotalCount(pageSize); + if (pagedXUserList.getXuserInfoList().isEmpty()) { + LOG.info("PolicyMgrUserGroupBuilder.getUsers() done updating users"); + return 1; + } + + String response = getDataFromLdap(PM_ADD_USERS_URI, pagedXUserList); + + if (StringUtils.isNotEmpty(response)) { + try { + ret = Integer.valueOf(response); + uploadedCount += pageSize; + } catch (NumberFormatException e) { + LOG.error("Failed to addOrUpdateUsers {}", uploadedCount, e); + throw e; + } + } else { + LOG.error("Failed to addOrUpdateUsers {}", uploadedCount); + throw new Exception("Failed to addOrUpdateUsers" + uploadedCount); + } + LOG.info(String.format("API returned: %s, No. of users uploaded to ranger admin = %s", ret, (uploadedCount > totalCount ? totalCount : uploadedCount))); + } + + LOG.debug("<== PolicyMgrUserGroupBuilder.getUsers()"); + return ret; + } + + private String getDataFromLdap(String uri, Object pagedList) throws Exception { + String response = null; + if (isRangerCookieEnabled) { + response = cookieBasedUploadEntity(pagedList, uri); + } else { + try { + ClientResponse clientRes = ldapUgSyncClient.post(uri, null, pagedList); + if (clientRes != null) { + response = clientRes.getEntity(String.class); + } + } catch (Throwable t) { + LOG.error("Failed to get response, Error is : ", t); + throw t; + } + } + LOG.debug(String.format("REST response from %s : %s", uri, response)); + return response; + } + + private int addOrUpdateDeltaGroups() throws Throwable { + LOG.debug("PolicyMgrUserGroupBuilder.addOrUpdateDeltaGroups({})", deltaGroups.keySet()); + int ret = 0; + + GetXGroupListResponse xGroupList = new GetXGroupListResponse(); + xGroupList.setTotalCount(deltaGroups.size()); + xGroupList.setXgroupInfoList(new ArrayList<>(deltaGroups.values())); + + if (authenticationType != null + && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) + && SecureClientLogin.isKerberosCredentialExists(principal, + keytab)) { + try { + Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); + final GetXGroupListResponse xGroupListFinal = xGroupList; + ret = Subject.doAs(sub, new PrivilegedAction() { + @Override + public Integer run() { + try { + return getGroups(xGroupListFinal); + } catch (Throwable e) { + LOG.error("Failed to add or update groups : ", e); + } + return 0; + } + }); + } catch (Exception e) { + LOG.error("Failed to add or update groups : ", e); + throw e; + } + } else { + ret = getGroups(xGroupList); + } + LOG.debug("PolicyMgrUserGroupBuilder.addOrUpdateDeltaGroups({})", deltaGroups.keySet()); + return ret; + } + + private int getGroups(GetXGroupListResponse xGroupList) throws Throwable { + LOG.debug("==> PolicyMgrUserGroupBuilder.getGroups()"); + int ret = 0; + int totalCount = xGroupList.getTotalCount(); + int uploadedCount = 0; + int pageSize = Integer.parseInt(recordsToPullPerCall); + while (uploadedCount < totalCount) { + checkStatus(); + GetXGroupListResponse pagedXGroupList = new GetXGroupListResponse(); + int pagedXGroupListLen = uploadedCount + pageSize; + pagedXGroupList.setXgroupInfoList(xGroupList.getXgroupInfoList().subList(uploadedCount, + pagedXGroupListLen > totalCount ? totalCount : pagedXGroupListLen)); + pagedXGroupList.setTotalCount(pageSize); + + String response = getDataFromLdap(PM_ADD_GROUPS_URI, pagedXGroupList); + if (StringUtils.isNotEmpty(response)) { + try { + ret = Integer.valueOf(response); + uploadedCount += pageSize; + } catch (NumberFormatException e) { + LOG.error("Failed to addOrUpdateGroups {}", uploadedCount, e); + throw e; + } + } else { + LOG.error("Failed to addOrUpdateGroups {}", uploadedCount); + throw new Exception("Failed to addOrUpdateGroups " + uploadedCount); + } + LOG.info(String.format("API returned: %s, No. of groups uploaded to ranger admin = %s", ret, (uploadedCount > totalCount ? totalCount : uploadedCount))); + } + + LOG.debug("<== PolicyMgrUserGroupBuilder.getGroups()"); + return ret; + } + + private int addOrUpdateDeltaGroupUsers(List groupUserInfoList) throws Throwable { + LOG.debug("==> PolicyMgrUserGroupBuilder.addOrUpdateDeltaGroupUsers({})", groupUserInfoList); + int ret = 0; + + if (authenticationType != null + && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) + && SecureClientLogin.isKerberosCredentialExists(principal, + keytab)) { + try { + Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); + final List groupUserInfoListFinal = groupUserInfoList; + ret = Subject.doAs(sub, new PrivilegedAction() { + @Override + public Integer run() { + try { + return getGroupUsers(groupUserInfoListFinal); + } catch (Throwable e) { + LOG.error("Failed to add or update group memberships : ", e); + } + return 0; + } + }); + } catch (Exception e) { + LOG.error("Failed to add or update group memberships : ", e); + throw e; + } + } else { + ret = getGroupUsers(groupUserInfoList); + } + LOG.debug("<== PolicyMgrUserGroupBuilder.addOrUpdateDeltaGroupUsers({})", groupUserInfoList); + return ret; + } + + private int getGroupUsers(List groupUserInfoList) throws Throwable { + LOG.debug("==> PolicyMgrUserGroupBuilder.getGroupUsers()"); + int ret = 0; + int totalCount = groupUserInfoList.size(); + int uploadedCount = 0; + int pageSize = Integer.parseInt(recordsToPullPerCall); + while (uploadedCount < totalCount) { + checkStatus(); + + int pagedGroupUserInfoListLen = uploadedCount + pageSize; + List pagedGroupUserInfoList = groupUserInfoList.subList(uploadedCount, pagedGroupUserInfoListLen > totalCount ? totalCount : pagedGroupUserInfoListLen); + + String response = getDataFromLdap(PM_ADD_GROUP_USER_LIST_URI, pagedGroupUserInfoList); + if (StringUtils.isNotEmpty(response)) { + try { + ret = Integer.valueOf(response); + uploadedCount += pageSize; + } catch (NumberFormatException e) { + LOG.error("Failed to addOrUpdateGroupUsers {}", uploadedCount, e); + throw e; + } + } else { + LOG.error("Failed to addOrUpdateGroupUsers {}", uploadedCount); + throw new Exception("Failed to addOrUpdateGroupUsers " + uploadedCount); + } + + LOG.info(String.format("API returned: %s, No. of group memberships uploaded to ranger admin = %s", ret, (uploadedCount > totalCount ? totalCount : uploadedCount))); + } + + LOG.debug("<== PolicyMgrUserGroupBuilder.getGroupUsers()"); + return ret; + } + + private String updateRoles(UsersGroupRoleAssignments ugRoleAssignments) { + LOG.debug("==> PolicyMgrUserGroupBuilder.updateUserRole({})", ugRoleAssignments.getUsers()); + + if (authenticationType != null && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) && SecureClientLogin.isKerberosCredentialExists(principal, keytab)) { + try { + Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); + final UsersGroupRoleAssignments result = ugRoleAssignments; + return Subject.doAs(sub, (PrivilegedAction) () -> { + try { + return updateUsersRoles(result); + } catch (Exception e) { + LOG.error("Failed to add User Group Info: ", e); + return null; + } + }); + } catch (Exception e) { + LOG.error("Failed to Authenticate Using given Principal and Keytab : ", e); + } + return null; + } else { + return updateUsersRoles(ugRoleAssignments); + } + } + + private String updateUsersRoles(UsersGroupRoleAssignments ugRoleAssignments) { + LOG.debug("==> PolicyMgrUserGroupBuilder.updateUserRoles({})", ugRoleAssignments.getUsers()); + String response = null; + try { + int totalCount = ugRoleAssignments.getUsers().size(); + int uploadedCount = 0; + int pageSize = Integer.parseInt(recordsToPullPerCall); + while (uploadedCount < totalCount) { + checkStatus(); + int pagedUgRoleAssignmentsListLen = uploadedCount + pageSize; + UsersGroupRoleAssignments pagedUgRoleAssignmentsList = new UsersGroupRoleAssignments(); + pagedUgRoleAssignmentsList.setUsers(ugRoleAssignments.getUsers().subList(uploadedCount, + pagedUgRoleAssignmentsListLen > totalCount ? totalCount : pagedUgRoleAssignmentsListLen)); + pagedUgRoleAssignmentsList.setGroupRoleAssignments(ugRoleAssignments.getGroupRoleAssignments()); + pagedUgRoleAssignmentsList.setUserRoleAssignments(ugRoleAssignments.getUserRoleAssignments()); + pagedUgRoleAssignmentsList.setWhiteListGroupRoleAssignments(ugRoleAssignments.getWhiteListGroupRoleAssignments()); + pagedUgRoleAssignmentsList.setWhiteListUserRoleAssignments(ugRoleAssignments.getWhiteListUserRoleAssignments()); + pagedUgRoleAssignmentsList.setReset(ugRoleAssignments.isReset()); + if ((uploadedCount + pageSize) >= totalCount) { // this is the last iteration of the loop + pagedUgRoleAssignmentsList.setLastPage(true); + } + ClientResponse clientRes; + String url = PM_UPDATE_USERS_ROLES_URI; + + String jsonString = JsonUtils.objectToJson(pagedUgRoleAssignmentsList); + LOG.debug(String.format("Paged RoleAssignments Request to %s: %s", url, jsonString)); + + if (isRangerCookieEnabled) { + response = cookieBasedUploadEntity(pagedUgRoleAssignmentsList, url); + } else { + try { + clientRes = ldapUgSyncClient.post(url, null, ugRoleAssignments); + if (clientRes != null) { + response = clientRes.getEntity(String.class); + } + } catch (Throwable t) { + LOG.error("Failed to get response: ", t); + } + } + + LOG.debug(String.format("REST response from %s : %s", url, response)); + + if (response == null) { + throw new RuntimeException("Failed to get a REST response!"); + } + + uploadedCount += pageSize; + } + } catch (Exception e) { + LOG.error("Unable to update roles for: {}", ugRoleAssignments.getUsers(), e); + response = null; + } + + LOG.debug("<== PolicyMgrUserGroupBuilder.updateUserRoles({})", response); + return response; + } + + private void addUserGroupAuditInfo(UgsyncAuditInfo auditInfo) throws Throwable { + LOG.debug(String.format("==> PolicyMgrUserGroupBuilder.addUserGroupAuditInfo(%s, %s, %s, %s, %s)", auditInfo.getNoOfNewUsers(), auditInfo.getNoOfNewGroups(), auditInfo.getNoOfModifiedUsers(), auditInfo.getNoOfModifiedGroups(), auditInfo.getSyncSource())); + + if (authenticationType != null && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) && SecureClientLogin.isKerberosCredentialExists(principal, keytab)) { + try { + Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); + final UgsyncAuditInfo auditInfoFinal = auditInfo; + Subject.doAs(sub, new PrivilegedAction() { + @Override + public Void run() { + try { + getUserGroupAuditInfo(auditInfoFinal); + } catch (Throwable e) { + LOG.error("Failed to add User : ", e); + } + return null; + } + }); + } catch (Exception e) { + LOG.error("Failed to Authenticate Using given Principal and Keytab : ", e); + } + } else { + getUserGroupAuditInfo(auditInfo); + } + } + + private void getUserGroupAuditInfo(UgsyncAuditInfo userInfo) throws Throwable { + LOG.debug("==> PolicyMgrUserGroupBuilder.getUserGroupAuditInfo()"); + checkStatus(); + String response = null; + ClientResponse clientRes = null; + + if (isRangerCookieEnabled) { + response = cookieBasedUploadEntity(userInfo, PM_AUDIT_INFO_URI); + } else { + try { + clientRes = ldapUgSyncClient.post(PM_AUDIT_INFO_URI, null, userInfo); + if (clientRes != null) { + response = clientRes.getEntity(String.class); + } + } catch (Throwable t) { + LOG.error("Failed to get response, Error is : ", t); + } + } + LOG.debug(String.format("REST response from %s : %s", PM_AUDIT_INFO_URI, response)); + JsonUtils.jsonToObject(response, UgsyncAuditInfo.class); + + LOG.debug("AuditInfo Creation successful "); + LOG.debug("<== PolicyMgrUserGroupBuilder.getUserGroupAuditInfo()"); + } + + private String cookieBasedUploadEntity(Object obj, String apiURL) { + LOG.debug("==> PolicyMgrUserGroupBuilder.cookieBasedUploadEntity()"); + String response = null; + if (sessionId != null && isValidRangerCookie) { + response = tryUploadEntityWithCookie(obj, apiURL); + } else { + response = tryUploadEntityWithCred(obj, apiURL); + } + LOG.debug("<== PolicyMgrUserGroupBuilder.cookieBasedUploadEntity()"); + return response; + } + + private String cookieBasedGetEntity(String apiURL, int retrievedCount) { + LOG.debug("==> PolicyMgrUserGroupBuilder.cookieBasedGetEntity()"); + String response = null; + if (sessionId != null && isValidRangerCookie) { + response = tryGetEntityWithCookie(apiURL, retrievedCount); + } else { + response = tryGetEntityWithCred(apiURL, retrievedCount); + } + LOG.debug("<== PolicyMgrUserGroupBuilder.cookieBasedGetEntity()"); + return response; + } + + private String tryUploadEntityWithCookie(Object obj, String apiURL) { + LOG.debug("==> PolicyMgrUserGroupBuilder.tryUploadEntityWithCookie()"); + String response = null; + ClientResponse clientResp = null; + try { + clientResp = ldapUgSyncClient.post(apiURL, null, obj, sessionId); + } catch (Throwable t) { + LOG.error("Failed to get response, Error is : ", t); + } + if (clientResp != null) { + if (!(clientResp.toString().contains(apiURL))) { + clientResp.setStatus(HttpServletResponse.SC_NOT_FOUND); + sessionId = null; + isValidRangerCookie = false; + } else if (clientResp.getStatus() == HttpServletResponse.SC_UNAUTHORIZED) { + sessionId = null; + isValidRangerCookie = false; + } else if (clientResp.getStatus() == HttpServletResponse.SC_NO_CONTENT || clientResp.getStatus() == HttpServletResponse.SC_OK) { + List respCookieList = clientResp.getCookies(); + for (NewCookie cookie : respCookieList) { + if (cookie.getName().equalsIgnoreCase(rangerCookieName)) { + if (!(sessionId.getValue().equalsIgnoreCase(cookie.toCookie().getValue()))) { + sessionId = cookie.toCookie(); + } + isValidRangerCookie = true; + break; + } + } + } + + if (clientResp.getStatus() != HttpServletResponse.SC_OK && clientResp.getStatus() != HttpServletResponse.SC_NO_CONTENT + && clientResp.getStatus() != HttpServletResponse.SC_BAD_REQUEST) { + sessionId = null; + isValidRangerCookie = false; + } + clientResp.bufferEntity(); + response = clientResp.getEntity(String.class); + } + LOG.debug("<== PolicyMgrUserGroupBuilder.tryUploadEntityWithCookie()"); + return response; + } + + private String tryUploadEntityWithCred(Object obj, String apiURL) { + LOG.debug("==> PolicyMgrUserGroupBuilder.tryUploadEntityInfoWithCred()"); + String response = null; + ClientResponse clientResp = null; + + String jsonString = JsonUtils.objectToJson(obj); + LOG.debug(String.format("User Group Mapping: %s", jsonString)); + + try { + clientResp = ldapUgSyncClient.post(apiURL, null, obj); + } catch (Throwable t) { + LOG.error("Failed to get response, Error is : ", t); + } + if (clientResp != null) { + if (!(clientResp.toString().contains(apiURL))) { + clientResp.setStatus(HttpServletResponse.SC_NOT_FOUND); + } else if (clientResp.getStatus() == HttpServletResponse.SC_UNAUTHORIZED) { + LOG.warn("Credentials response from ranger is 401."); + } else if (clientResp.getStatus() == HttpServletResponse.SC_OK || clientResp.getStatus() == HttpServletResponse.SC_NO_CONTENT) { + cookieList = clientResp.getCookies(); + for (NewCookie cookie : cookieList) { + if (cookie.getName().equalsIgnoreCase(rangerCookieName)) { + sessionId = cookie.toCookie(); + isValidRangerCookie = true; + LOG.info("valid cookie saved "); + break; + } + } + } + if (clientResp.getStatus() != HttpServletResponse.SC_OK && clientResp.getStatus() != HttpServletResponse.SC_NO_CONTENT + && clientResp.getStatus() != HttpServletResponse.SC_BAD_REQUEST) { + sessionId = null; + isValidRangerCookie = false; + } + clientResp.bufferEntity(); + response = clientResp.getEntity(String.class); + } + + LOG.debug("<== PolicyMgrUserGroupBuilder.tryUploadEntityInfoWithCred()"); + return response; + } + + private String tryGetEntityWithCred(String apiURL, int retrievedCount) { + LOG.debug("==> PolicyMgrUserGroupBuilder.tryGetEntityWithCred()"); + String response = null; + ClientResponse clientResp = null; + + Map queryParams = new HashMap(); + queryParams.put("pageSize", recordsToPullPerCall); + queryParams.put("startIndex", String.valueOf(retrievedCount)); + try { + clientResp = ldapUgSyncClient.get(apiURL, queryParams); + } catch (Throwable t) { + LOG.error("Failed to get response, Error is : ", t); + } + if (clientResp != null) { + if (!(clientResp.toString().contains(apiURL))) { + clientResp.setStatus(HttpServletResponse.SC_NOT_FOUND); + } else if (clientResp.getStatus() == HttpServletResponse.SC_UNAUTHORIZED) { + LOG.warn("Credentials response from ranger is 401."); + } else if (clientResp.getStatus() == HttpServletResponse.SC_OK || clientResp.getStatus() == HttpServletResponse.SC_NO_CONTENT) { + cookieList = clientResp.getCookies(); + for (NewCookie cookie : cookieList) { + if (cookie.getName().equalsIgnoreCase(rangerCookieName)) { + sessionId = cookie.toCookie(); + isValidRangerCookie = true; + LOG.info("valid cookie saved "); + break; + } + } + } + if (clientResp.getStatus() != HttpServletResponse.SC_OK && clientResp.getStatus() != HttpServletResponse.SC_NO_CONTENT + && clientResp.getStatus() != HttpServletResponse.SC_BAD_REQUEST) { + sessionId = null; + isValidRangerCookie = false; + } + clientResp.bufferEntity(); + response = clientResp.getEntity(String.class); + } + + LOG.debug("<== PolicyMgrUserGroupBuilder.tryGetEntityWithCred()"); + return response; + } + + private String tryGetEntityWithCookie(String apiURL, int retrievedCount) { + LOG.debug("==> PolicyMgrUserGroupBuilder.tryGetEntityWithCookie()"); + String response = null; + ClientResponse clientResp = null; + + Map queryParams = new HashMap(); + queryParams.put("pageSize", recordsToPullPerCall); + queryParams.put("startIndex", String.valueOf(retrievedCount)); + try { + clientResp = ldapUgSyncClient.get(apiURL, queryParams, sessionId); + } catch (Throwable t) { + LOG.error("Failed to get response, Error is : ", t); + } + if (clientResp != null) { + if (!(clientResp.toString().contains(apiURL))) { + clientResp.setStatus(HttpServletResponse.SC_NOT_FOUND); + sessionId = null; + isValidRangerCookie = false; + } else if (clientResp.getStatus() == HttpServletResponse.SC_UNAUTHORIZED) { + sessionId = null; + isValidRangerCookie = false; + } else if (clientResp.getStatus() == HttpServletResponse.SC_NO_CONTENT || clientResp.getStatus() == HttpServletResponse.SC_OK) { + List respCookieList = clientResp.getCookies(); + for (NewCookie cookie : respCookieList) { + if (cookie.getName().equalsIgnoreCase(rangerCookieName)) { + if (!(sessionId.getValue().equalsIgnoreCase(cookie.toCookie().getValue()))) { + sessionId = cookie.toCookie(); + } + isValidRangerCookie = true; + break; + } + } + } + + if (clientResp.getStatus() != HttpServletResponse.SC_OK && clientResp.getStatus() != HttpServletResponse.SC_NO_CONTENT && clientResp.getStatus() != HttpServletResponse.SC_BAD_REQUEST) { + sessionId = null; + isValidRangerCookie = false; + } + clientResp.bufferEntity(); + response = clientResp.getEntity(String.class); + } + LOG.debug("<== PolicyMgrUserGroupBuilder.tryGetEntityWithCookie()"); + return response; + } + + private void getRoleForUserGroups(String userGroupRolesData, Map userMap, Map groupMap) { + LOG.debug("==>> getRoleForUserGroups({})", userGroupRolesData); + Map reverseGroupMap = new LinkedHashMap<>(); + String roleDelimiter = config.getRoleDelimiter(); + String userGroupDelimiter = config.getUserGroupDelimiter(); + String userNameDelimiter = config.getUserGroupNameDelimiter(); + roleDelimiter = StringUtils.isEmpty(roleDelimiter) ? "&" : roleDelimiter; + userGroupDelimiter = StringUtils.isEmpty(userGroupDelimiter) ? ":" : userGroupDelimiter; + userNameDelimiter = StringUtils.isEmpty(userNameDelimiter) ? "," : userNameDelimiter; + StringTokenizer str = new StringTokenizer(userGroupRolesData, roleDelimiter); + int flag = 0; + String userGroupCheck = null; + String roleName = null; + while (str.hasMoreTokens()) { + flag = 0; + String tokens = str.nextToken(); + if (StringUtils.isNotEmpty(tokens)) { + StringTokenizer userGroupRoles = new StringTokenizer(tokens, userGroupDelimiter); + if (userGroupRoles != null) { + while (userGroupRoles.hasMoreElements()) { + String userGroupRolesTokens = userGroupRoles.nextToken(); + if (StringUtils.isNotEmpty(userGroupRolesTokens)) { + flag++; + switch (flag) { + case 1: + roleName = userGroupRolesTokens; + break; + case 2: + userGroupCheck = userGroupRolesTokens; + break; + case 3: + StringTokenizer userGroupNames = new StringTokenizer(userGroupRolesTokens, userNameDelimiter); + if (userGroupNames != null) { + while (userGroupNames.hasMoreElements()) { + String userGroup = userGroupNames.nextToken(); + if (StringUtils.isNotEmpty(userGroup)) { + if (userGroupCheck.trim().equalsIgnoreCase("u")) { + userMap.put(userGroup.trim(), roleName.trim()); + } else if (userGroupCheck.trim().equalsIgnoreCase("g")) { + reverseGroupMap.put(userGroup.trim(), + roleName.trim()); + } + } + } + } + break; + default: + userMap.clear(); + reverseGroupMap.clear(); + break; + } + } + } + } + } + } + // Reversing the order of group keys so that the last role specified in the configuration rules is applied when a user belongs to multiple groups with different roles + if (MapUtils.isNotEmpty(reverseGroupMap)) { + List groupNames = new ArrayList<>(reverseGroupMap.keySet()); + Collections.reverse(groupNames); + groupNames.forEach(group -> groupMap.put(group, reverseGroupMap.get(group))); + } + } + + private void updateDeletedGroups(Map> sourceGroups) throws Throwable { + computeDeletedGroups(sourceGroups); + if (MapUtils.isNotEmpty(deletedGroups)) { + if (updateDeletedGroups() == 0) { + String msg = "Failed to update deleted groups to ranger admin"; + LOG.error(msg); + throw new Exception(msg); + } + } + LOG.info("No. of groups marked for delete = {}", deletedGroups.size()); + noOfDeletedGroups += deletedGroups.size(); + } + + private void computeDeletedGroups(Map> sourceGroups) { + LOG.debug("PolicyMgrUserGroupBuilder.computeDeletedGroups({})", sourceGroups.keySet()); + deletedGroups = new HashMap<>(); + // Check if the group from cache exists in the sourceGroups. If not, mark as deleted group. + for (XGroupInfo groupInfo : groupCache.values()) { + Map groupOtherAttrs = groupInfo.getOtherAttrsMap(); + String groupDN = groupOtherAttrs != null ? groupOtherAttrs.get(UgsyncCommonConstants.FULL_NAME) : null; + if (StringUtils.isNotEmpty(groupDN) && !sourceGroups.containsKey(groupDN) + && StringUtils.equalsIgnoreCase(groupOtherAttrs.get(UgsyncCommonConstants.SYNC_SOURCE), currentSyncSource) && + StringUtils.equalsIgnoreCase(groupOtherAttrs.get(UgsyncCommonConstants.LDAP_URL), ldapUrl)) { + if (ISHIDDEN.equals(groupInfo.getIsVisible())) { + groupInfo.setIsVisible(ISHIDDEN); + deletedGroups.put(groupInfo.getName(), groupInfo); + } else { + LOG.info("group {} already marked for delete", groupInfo.getName()); + } + } + } + LOG.debug("<== PolicyMgrUserGroupBuilder.computeDeletedGroups({})", deletedGroups); + } + + private int updateDeletedGroups() throws Throwable { + LOG.debug("==> PolicyMgrUserGroupBuilder.updateDeletedGroups({})", deletedGroups); + int ret = 0; + + if (authenticationType != null + && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) + && SecureClientLogin.isKerberosCredentialExists(principal, + keytab)) { + try { + Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); + ret = Subject.doAs(sub, new PrivilegedAction() { + @Override + public Integer run() { + try { + return getDeletedGroups(); + } catch (Throwable e) { + LOG.error("Failed to add or update deleted groups : ", e); + } + return 0; + } + }); + } catch (Exception e) { + LOG.error("Failed to add or update deleted groups : ", e); + throw e; + } + } else { + ret = getDeletedGroups(); + } + LOG.debug("<== PolicyMgrUserGroupBuilder.updateDeletedGroups({})", deletedGroups); + return ret; + } + + private int getDeletedGroups() throws Throwable { + LOG.debug("==> PolicyMgrUserGroupBuilder.getDeletedGroups()"); + checkStatus(); + int ret = 0; + String response = null; + ClientResponse clientRes = null; + + if (isRangerCookieEnabled) { + response = cookieBasedUploadEntity(deletedGroups.keySet(), PM_UPDATE_DELETED_GROUPS_URI); + } else { + try { + clientRes = ldapUgSyncClient.post(PM_UPDATE_DELETED_GROUPS_URI, null, deletedGroups.keySet()); + if (clientRes != null) { + response = clientRes.getEntity(String.class); + } + } catch (Throwable t) { + LOG.error("Failed to get response, Error is : ", t); + } + } + LOG.debug(String.format("REST response from %s : %s", PM_UPDATE_DELETED_GROUPS_URI, response)); + if (response != null) { + try { + ret = Integer.valueOf(response); + } catch (NumberFormatException e) { + LOG.error("Failed to update deleted groups", e); + throw e; + } + } else { + LOG.error("Failed to update deleted groups "); + throw new Exception("Failed to update deleted groups "); + } + + LOG.debug("<== PolicyMgrUserGroupBuilder.getDeletedGroups({})", ret); + return ret; + } + + private void updateDeletedUsers(Map> sourceUsers) throws Throwable { + computeDeletedUsers(sourceUsers); + if (MapUtils.isNotEmpty(deletedUsers)) { + if (updateDeletedUsers() == 0) { + String msg = "Failed to update deleted users to ranger admin"; + LOG.error(msg); + throw new Exception(msg); + } + } + LOG.info("No. of users marked for delete = {}", deletedUsers.size()); + noOfDeletedUsers += deletedUsers.size(); + } + + private void computeDeletedUsers(Map> sourceUsers) { + LOG.debug("PolicyMgrUserGroupBuilder.computeDeletedUsers({})", sourceUsers.keySet()); + deletedUsers = new HashMap<>(); + // Check if the group from cache exists in the sourceGroups. If not, mark as deleted group. + for (XUserInfo userInfo : userCache.values()) { + Map userOtherAttrs = userInfo.getOtherAttrsMap(); + String userDN = userOtherAttrs != null ? userOtherAttrs.get(UgsyncCommonConstants.FULL_NAME) : null; + if (StringUtils.isNotEmpty(userDN) && !sourceUsers.containsKey(userDN) + && StringUtils.equalsIgnoreCase(userOtherAttrs.get(UgsyncCommonConstants.SYNC_SOURCE), currentSyncSource) + && StringUtils.equalsIgnoreCase(userOtherAttrs.get(UgsyncCommonConstants.LDAP_URL), ldapUrl)) { + if (userInfo.getIsVisible() != ISHIDDEN) { + userInfo.setIsVisible(ISHIDDEN); + deletedUsers.put(userInfo.getName(), userInfo); + } else { + LOG.info("user {} already marked for delete", userInfo.getName()); + } + } + } + LOG.debug("<== PolicyMgrUserGroupBuilder.computeDeletedUsers({})", deletedUsers); + } + + private int updateDeletedUsers() throws Throwable { + LOG.debug("==> PolicyMgrUserGroupBuilder.updateDeletedUsers({})", deletedUsers); + int ret = 0; + + if (authenticationType != null + && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) + && SecureClientLogin.isKerberosCredentialExists(principal, + keytab)) { + try { + Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); + ret = Subject.doAs(sub, new PrivilegedAction() { + @Override + public Integer run() { + try { + return getDeletedUsers(); + } catch (Throwable e) { + LOG.error("Failed to add or update deleted users : ", e); + } + return 0; + } + }); + } catch (Exception e) { + LOG.error("Failed to add or update deleted users : ", e); + throw e; + } + } else { + ret = getDeletedUsers(); + } + LOG.debug("<== PolicyMgrUserGroupBuilder.updateDeletedUsers({})", deletedUsers); + return ret; + } + + private int getDeletedUsers() throws Throwable { + LOG.debug("==> PolicyMgrUserGroupBuilder.getDeletedUsers()"); + checkStatus(); + int ret = 0; + String response = null; + ClientResponse clientRes = null; + + if (isRangerCookieEnabled) { + response = cookieBasedUploadEntity(deletedUsers.keySet(), PM_UPDATE_DELETED_USERS_URI); + } else { + try { + clientRes = ldapUgSyncClient.post(PM_UPDATE_DELETED_USERS_URI, null, deletedUsers.keySet()); + if (clientRes != null) { + response = clientRes.getEntity(String.class); + } + } catch (Throwable t) { + LOG.error("Failed to get response, Error is : ", t); + } + } + LOG.debug(String.format("REST response from %s : %s", PM_UPDATE_DELETED_USERS_URI, response)); + if (response != null) { + try { + ret = Integer.valueOf(response); + } catch (NumberFormatException e) { + LOG.error("Failed to update deleted users", e); + throw e; + } + } else { + LOG.error("Failed to update deleted users "); + throw new Exception("Failed to update deleted users "); + } + + LOG.debug("<== PolicyMgrUserGroupBuilder.getDeletedUsers({})", ret); + return ret; + } + + // This will throw RuntimeException if Server is not Active + private void checkStatus() throws Exception { + if (!UserGroupSyncConfig.isUgsyncServiceActive()) { + LOG.error(errMsgForInactiveServer); + throw new RuntimeException(errMsgForInactiveServer); + } + } + + static { + try { + localHostname = java.net.InetAddress.getLocalHost().getCanonicalHostName(); + } catch (UnknownHostException e) { + localHostname = "unknown"; + } + } } diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/RangerUgSyncRESTClient.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/RangerUgSyncRESTClient.java index 4ead5a9ad2..a44b95e53b 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/RangerUgSyncRESTClient.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/RangerUgSyncRESTClient.java @@ -19,63 +19,57 @@ package org.apache.ranger.unixusersync.process; -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.KeyManager; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSession; -import javax.net.ssl.TrustManager; - import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.security.SecureClientLogin; -import org.apache.ranger.plugin.util.RangerRESTClient; -import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; - - import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.config.ClientConfig; import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter; import com.sun.jersey.client.urlconnection.HTTPSProperties; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.security.SecureClientLogin; +import org.apache.ranger.plugin.util.RangerRESTClient; +import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; -public class RangerUgSyncRESTClient extends RangerRESTClient { - private String AUTH_KERBEROS = "kerberos"; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.KeyManager; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.TrustManager; - public RangerUgSyncRESTClient(String policyMgrBaseUrls, String ugKeyStoreFile, String ugKeyStoreFilepwd, - String ugKeyStoreType, String ugTrustStoreFile, String ugTrustStoreFilepwd, String ugTrustStoreType, - String authenticationType, String principal, String keytab, String polMgrUsername, String polMgrPassword) { +public class RangerUgSyncRESTClient extends RangerRESTClient { + private String authKerberos = "kerberos"; - super(policyMgrBaseUrls, "", UserGroupSyncConfig.getInstance().getConfig()); - if (!(authenticationType != null && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) - && SecureClientLogin.isKerberosCredentialExists(principal, keytab))) { - setBasicAuthInfo(polMgrUsername, polMgrPassword); - } + public RangerUgSyncRESTClient(String policyMgrBaseUrls, String ugKeyStoreFile, String ugKeyStoreFilepwd, String ugKeyStoreType, String ugTrustStoreFile, String ugTrustStoreFilepwd, String ugTrustStoreType, String authenticationType, String principal, String keytab, String polMgrUsername, String polMgrPassword) { + super(policyMgrBaseUrls, "", UserGroupSyncConfig.getInstance().getConfig()); + if (!(authenticationType != null && authKerberos.equalsIgnoreCase(authenticationType) && SecureClientLogin.isKerberosCredentialExists(principal, keytab))) { + setBasicAuthInfo(polMgrUsername, polMgrPassword); + } - if (isSSL()) { - setKeyStoreType(ugKeyStoreType); - setTrustStoreType(ugTrustStoreType); - KeyManager[] kmList = getKeyManagers(ugKeyStoreFile, ugKeyStoreFilepwd); - TrustManager[] tmList = getTrustManagers(ugTrustStoreFile, ugTrustStoreFilepwd); - SSLContext sslContext = getSSLContext(kmList, tmList); - ClientConfig config = new DefaultClientConfig(); + if (isSSL()) { + setKeyStoreType(ugKeyStoreType); + setTrustStoreType(ugTrustStoreType); + KeyManager[] kmList = getKeyManagers(ugKeyStoreFile, ugKeyStoreFilepwd); + TrustManager[] tmList = getTrustManagers(ugTrustStoreFile, ugTrustStoreFilepwd); + SSLContext sslContext = getSSLContext(kmList, tmList); + ClientConfig config = new DefaultClientConfig(); - config.getClasses().add(JacksonJsonProvider.class); // to handle List<> unmarshalling - HostnameVerifier hv = new HostnameVerifier() { - public boolean verify(String urlHostName, SSLSession session) { - return session.getPeerHost().equals(urlHostName); - } - }; - config.getProperties().put(HTTPSProperties.PROPERTY_HTTPS_PROPERTIES, new HTTPSProperties(hv, sslContext)); + config.getClasses().add(JacksonJsonProvider.class); // to handle List<> unmarshalling + HostnameVerifier hv = new HostnameVerifier() { + public boolean verify(String urlHostName, SSLSession session) { + return session.getPeerHost().equals(urlHostName); + } + }; + config.getProperties().put(HTTPSProperties.PROPERTY_HTTPS_PROPERTIES, new HTTPSProperties(hv, sslContext)); - setClient(Client.create(config)); - if (StringUtils.isNotEmpty(getUsername()) && StringUtils.isNotEmpty(getPassword())) { - getClient().addFilter(new HTTPBasicAuthFilter(getUsername(), getPassword())); - } - } + setClient(Client.create(config)); + if (StringUtils.isNotEmpty(getUsername()) && StringUtils.isNotEmpty(getPassword())) { + getClient().addFilter(new HTTPBasicAuthFilter(getUsername(), getPassword())); + } + } - UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); + UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); - super.setMaxRetryAttempts(config.getPolicyMgrMaxRetryAttempts()); - super.setRetryIntervalMs(config.getPolicyMgrRetryIntervalMs()); - } + super.setMaxRetryAttempts(config.getPolicyMgrMaxRetryAttempts()); + super.setRetryIntervalMs(config.getPolicyMgrRetryIntervalMs()); + } } diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java index 23fcb879aa..6a256812e6 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java @@ -19,21 +19,6 @@ package org.apache.ranger.unixusersync.process; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Map; -import java.util.Set; -import java.util.HashSet; -import java.util.HashMap; -import java.util.Date; -import java.util.Arrays; -import java.util.Iterator; - import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.collect.HashBasedTable; import org.apache.hadoop.thirdparty.com.google.common.collect.Table; @@ -46,537 +31,521 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class UnixUserGroupBuilder implements UserGroupSource { - - private static final Logger LOG = LoggerFactory.getLogger(UnixUserGroupBuilder.class); - private final static String OS = System.getProperty("os.name"); - - // kept for legacy support - //public static final String UNIX_USER_PASSWORD_FILE = "/etc/passwd"; - //public static final String UNIX_GROUP_FILE = "/etc/group"; - - /** Shell commands to get users and groups */ - static final String LINUX_GET_ALL_USERS_CMD = "getent passwd"; - static final String LINUX_GET_ALL_GROUPS_CMD = "getent group"; - static final String LINUX_GET_GROUP_CMD = "getent group %s"; - - // mainly for testing purposes, there might be a better way - static final String MAC_GET_ALL_USERS_CMD = "dscl . -readall /Users UniqueID PrimaryGroupID | " + - "awk 'BEGIN { OFS = \":\"; ORS=\"\\n\"; i=0;}" + - "/RecordName: / {name = $2;i = 0;}/PrimaryGroupID: / {gid = $2;}" + - "/^ / {if (i == 0) { i++; name = $1;}}" + - "/UniqueID: / {uid = $2;print name, \"*\", gid, uid;}'"; - static final String MAC_GET_ALL_GROUPS_CMD = "dscl . -list /Groups PrimaryGroupID | " + - "awk -v OFS=\":\" '{print $1, \"*\", $2, \"\"}'"; - static final String MAC_GET_GROUP_CMD = "dscl . -read /Groups/%1$s | paste -d, -s - | sed -e 's/:/|/g' | " + - "awk -v OFS=\":\" -v ORS=\"\\n\" -F, '{print \"%1$s\",\"*\",$6,$4}' | " + - "sed -e 's/:[^:]*| /:/g' | sed -e 's/ /,/g'"; - - static final String BACKEND_PASSWD = "passwd"; - - private boolean isUpdateSinkSucc = true; - private boolean enumerateGroupMembers = false; - private boolean useNss = false; - - private long lastUpdateTime = 0; // Last time maps were updated - private long timeout = 0; - - private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); - private Map groupId2groupNameMap; - private Map> sourceUsers; // Stores username and attr name & value pairs - private Map> sourceGroups; // Stores groupname and attr name & value pairs - private Map> sourceGroupUsers; - private Table groupUserTable; // groupname, username, group id - private int minimumUserId = 0; - private int minimumGroupId = 0; - private String unixPasswordFile; - private String unixGroupFile; - - private long passwordFileModifiedAt = 0; - private long groupFileModifiedAt = 0; - private UgsyncAuditInfo ugsyncAuditInfo; - private UnixSyncSourceInfo unixSyncSourceInfo; - private boolean isStartupFlag = false; - private int deleteCycles; - private String currentSyncSource; - private boolean computeDeletes = false; - Set allGroups = new HashSet<>(); - - - public static void main(String[] args) throws Throwable { - UnixUserGroupBuilder ugbuilder = new UnixUserGroupBuilder(); - ugbuilder.init(); - if (LOG.isDebugEnabled()) { - ugbuilder.print(); - } - } - - public UnixUserGroupBuilder() { - isStartupFlag = true; - minimumUserId = Integer.parseInt(config.getMinUserId()); - minimumGroupId = Integer.parseInt(config.getMinGroupId()); - unixPasswordFile = config.getUnixPasswordFile(); - unixGroupFile = config.getUnixGroupFile(); - - if (LOG.isDebugEnabled()) { - LOG.debug("Minimum UserId: " + minimumUserId + ", minimum GroupId: " + minimumGroupId); - } - - timeout = config.getUpdateMillisMin(); - enumerateGroupMembers = config.isGroupEnumerateEnabled(); - - } - - @Override - public void init() throws Throwable { - deleteCycles = 1; - - currentSyncSource = config.getCurrentSyncSource(); - - ugsyncAuditInfo = new UgsyncAuditInfo(); - unixSyncSourceInfo = new UnixSyncSourceInfo(); - ugsyncAuditInfo.setSyncSource(currentSyncSource); - ugsyncAuditInfo.setUnixSyncSourceInfo(unixSyncSourceInfo); - unixSyncSourceInfo.setFileName(unixPasswordFile); - unixSyncSourceInfo.setMinUserId(config.getMinUserId()); - unixSyncSourceInfo.setMinGroupId(config.getMinGroupId()); - if (!config.getUnixBackend().equalsIgnoreCase(BACKEND_PASSWD)) { - useNss = true; - unixSyncSourceInfo.setUnixBackend("nss"); - } else { - LOG.warn("DEPRECATED: Unix backend is configured to use /etc/passwd and /etc/group files directly " + - "instead of standard system mechanisms."); - unixSyncSourceInfo.setUnixBackend(BACKEND_PASSWD); - } - buildUserGroupInfo(); - } - - @Override - public boolean isChanged() { - computeDeletes = false; - // If previous update to Ranger admin fails, - // we want to retry the sync process even if there are no changes to the sync files - if (!isUpdateSinkSucc) { - LOG.info("Previous updateSink failed and hence retry!!"); - isUpdateSinkSucc = true; - return true; - } - try { - if (config.isUserSyncDeletesEnabled() && deleteCycles >= config.getUserSyncDeletesFrequency()) { - deleteCycles = 1; - computeDeletes = true; - if (LOG.isDebugEnabled()) { - LOG.debug("Compute deleted users/groups is enabled for this sync cycle"); - } - return true; - } - } catch (Throwable t) { - LOG.error("Failed to get information about usersync delete frequency", t); - } - if (config.isUserSyncDeletesEnabled()) { - deleteCycles++; - } - - if (useNss) - return System.currentTimeMillis() - lastUpdateTime > timeout; - - long TempPasswordFileModifiedAt = new File(unixPasswordFile).lastModified(); - if (passwordFileModifiedAt != TempPasswordFileModifiedAt) { - return true; - } - - long TempGroupFileModifiedAt = new File(unixGroupFile).lastModified(); - return groupFileModifiedAt != TempGroupFileModifiedAt; - } - - - @Override - public void updateSink(UserGroupSink sink) throws Throwable { - DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - Date lastModifiedTime = new Date(passwordFileModifiedAt); - Date syncTime = new Date(System.currentTimeMillis()); - unixSyncSourceInfo.setLastModified(formatter.format(lastModifiedTime)); - unixSyncSourceInfo.setSyncTime(formatter.format(syncTime)); - if (isChanged() || isStartupFlag) { - buildUserGroupInfo(); - if (LOG.isDebugEnabled()) { - LOG.debug("Users = " + sourceUsers.keySet()); - LOG.debug("Groups = " + sourceGroups.keySet()); - LOG.debug("GroupUsers = " + sourceGroupUsers.keySet()); - } - - try { - sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers, sourceGroupUsers, computeDeletes); - } catch (Throwable t) { - LOG.error("Failed to update ranger admin. Will retry in next sync cycle!!", t); - isUpdateSinkSucc = false; - } - } - try { - sink.postUserGroupAuditInfo(ugsyncAuditInfo); - } catch (Throwable t) { - LOG.error("sink.postUserGroupAuditInfo failed with exception: ", t); - } - isStartupFlag = false; - } - - - private void buildUserGroupInfo() throws Throwable { - groupId2groupNameMap = new HashMap<>(); - sourceUsers = new HashMap<>(); - sourceGroups = new HashMap<>(); - sourceGroupUsers = new HashMap<>(); - groupUserTable = HashBasedTable.create(); - allGroups = new HashSet<>(); - - if (OS.startsWith("Mac")) { - buildUnixGroupList(MAC_GET_ALL_GROUPS_CMD, MAC_GET_GROUP_CMD, false); - buildUnixUserList(MAC_GET_ALL_USERS_CMD); - } else { - if (!OS.startsWith("Linux")) { - LOG.warn("Platform not recognized assuming Linux compatible"); - } - buildUnixGroupList(LINUX_GET_ALL_GROUPS_CMD, LINUX_GET_GROUP_CMD, true); - buildUnixUserList(LINUX_GET_ALL_USERS_CMD); - } - - Iterator groupUserTableIterator = groupUserTable.rowKeySet().iterator(); - while (groupUserTableIterator.hasNext()) { - String groupName = groupUserTableIterator.next(); - Map groupUsersMap = groupUserTable.row(groupName); - Set userSet = new HashSet<>(); - - for (String userName : groupUsersMap.keySet()) { - if (sourceUsers.containsKey(userName)) { - userSet.add(userName); - } - } - sourceGroupUsers.put(groupName, userSet); - } - - lastUpdateTime = System.currentTimeMillis(); - - if (LOG.isDebugEnabled()) { - print(); - } - } - - private void print() { - for(String user : sourceUsers.keySet()) { - LOG.debug("USER:" + user); - Set groups = groupUserTable.column(user).keySet(); - for(String group : groups) { - LOG.debug("\tGROUP: " + group); - } - } - } - - private void buildUnixUserList(String command) throws Throwable { - BufferedReader reader = null; - Map userName2uid = new HashMap<>(); - - try { - if (!useNss) { - File file = new File(unixPasswordFile); - passwordFileModifiedAt = file.lastModified(); - FileInputStream fis = new FileInputStream(file); - reader = new BufferedReader(new InputStreamReader(fis, StandardCharsets.UTF_8)); - } else { - Process process = Runtime.getRuntime().exec(new String[]{"bash", "-c", command}); - reader = new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8)); - } - - String line; - while ((line = reader.readLine()) != null) { - if (line.trim().isEmpty()) - continue; - - String[] tokens = line.split(":"); - - int len = tokens.length; - - if (len < 4) { - LOG.warn("Unable to parse: " + line); - continue; - } - - String userName; - String userId; - String groupId; - - try { - userName = tokens[0]; - userId = tokens[2]; - groupId = tokens[3]; - } - catch(ArrayIndexOutOfBoundsException aiobe) { - LOG.warn("Ignoring line - [" + line + "]: Unable to parse line for getting user information", aiobe); - continue; - } - - int numUserId; - try { - numUserId = Integer.parseInt(userId); - } catch (NumberFormatException nfe) { - LOG.warn("Unix UserId: [" + userId + "]: can not be parsed as valid int. considering as -1.", nfe); - numUserId = -1; - } - - if (numUserId >= minimumUserId) { - userName2uid.put(userName, userId); - String groupName = groupId2groupNameMap.get(groupId); - if (groupName != null) { - Map userAttrMap = new HashMap<>(); - userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName); - userAttrMap.put(UgsyncCommonConstants.FULL_NAME, userName); - userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); - sourceUsers.put(userName, userAttrMap); - groupUserTable.put(groupName, userName, groupId); - } else { - // we are ignoring the possibility that this user was present in getent group. - LOG.warn("Group Name could not be found for group id: [" + groupId + "]. Skipping adding user [" + userName + "] with id [" + userId + "]."); - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("Skipping user [" + userName + "] since its userid [" + userId + "] is less than minuserid limit [" + minimumUserId + "]."); - } - } - } - } finally { - if (reader != null) - reader.close(); - } - - if (!useNss) - return; - - // this does a reverse check as not all users might be listed in getent passwd - if (enumerateGroupMembers) { - String line; - Table tempGroupUserTable = HashBasedTable.create(); - - if (LOG.isDebugEnabled()) { - LOG.debug("Start drill down group members"); - } - for (String userName : groupUserTable.columnKeySet()) { - - if (sourceUsers.containsKey(userName))// skip users we already now about - continue; - - if (LOG.isDebugEnabled()) { - LOG.debug("Enumerating user " + userName); - } - - // "id" is same across Linux / BSD / MacOSX - // gids are used as id might return groups with spaces, ie "domain users" - Process process = Runtime.getRuntime().exec(new String[]{"bash", "-c", "id -G " + userName}); - - try { - reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - line = reader.readLine(); - } finally { - reader.close(); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("id -G returned " + line); - } - - if (line == null || line.trim().isEmpty()) { - LOG.warn("User " + userName + " could not be resolved"); - continue; - } - - // check if all groups returned by id are visible to ranger - for (String gid : line.split(" ")) { - if (Integer.parseInt(gid) >= minimumGroupId) { - String groupName = groupId2groupNameMap.get(gid); - if (groupName != null) { - if (LOG.isDebugEnabled()) { - LOG.debug("New group user mapping found : " + groupName + " " + userName); - } - tempGroupUserTable.put(groupName, userName, gid); - } - } - } - - Map userAttrMap = new HashMap<>(); - userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName); - userAttrMap.put(UgsyncCommonConstants.FULL_NAME, userName); - userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); - sourceUsers.put(userName, userAttrMap); - } - groupUserTable.putAll(tempGroupUserTable); - if (LOG.isDebugEnabled()) { - LOG.debug("End drill down group members"); - } - } - } - - private void parseMembers(String line) { - if (line == null || line.isEmpty()) - return; - - String[] tokens = line.split(":"); - - if (tokens.length < 3) - return; - - String groupName = tokens[0]; - String groupId = tokens[2]; - String groupMembers = null; - - if (tokens.length > 3) - groupMembers = tokens[3]; - - groupId2groupNameMap.remove(groupId); - - int numGroupId = Integer.parseInt(groupId); - if (numGroupId < minimumGroupId) - return; - - groupId2groupNameMap.put(groupId, groupName); - Map groupAttrMap = new HashMap<>(); - groupAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, groupName); - groupAttrMap.put(UgsyncCommonConstants.FULL_NAME, groupName); - groupAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); - sourceGroups.put(groupName, groupAttrMap); - - if (groupMembers != null && !groupMembers.trim().isEmpty()) { - for (String user : groupMembers.split(",")) { - groupUserTable.put(groupName, user, groupId); - } - } else { - sourceGroupUsers.put(groupName, new HashSet()); - } - } - - private void buildUnixGroupList(String allGroupsCmd, String groupCmd, boolean useGid) throws Throwable { - if (LOG.isDebugEnabled()) { - LOG.debug("Start enumerating groups"); - } - BufferedReader reader = null; - - try { - if (!useNss) { - File file = new File(unixGroupFile); - groupFileModifiedAt = file.lastModified(); - FileInputStream fis = new FileInputStream(file); - reader = new BufferedReader(new InputStreamReader(fis, StandardCharsets.UTF_8)); - } else { - Process process = Runtime.getRuntime().exec(new String[]{"bash", "-c", allGroupsCmd}); - reader = new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8)); - } - - String line; - while ((line = reader.readLine()) != null) { - if (!line.trim().isEmpty()) - parseMembers(line); - } - } finally { - if (reader != null) - reader.close(); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("End enumerating group"); - } - - if (!useNss) - return; - - if (enumerateGroupMembers) { - if (LOG.isDebugEnabled()) { - LOG.debug("Start enumerating group members"); - } - String line = null; - Map copy = new HashMap(groupId2groupNameMap); - - for (Map.Entry group : copy.entrySet()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Enumerating group: " + group.getValue() + " GID(" + group.getKey() + ")"); - } - - String command; - if (useGid) { - command = String.format(groupCmd, group.getKey()); - } else { - command = String.format(groupCmd, group.getValue()); - } - - String[] cmd = new String[]{"bash", "-c", command + " " + group.getKey()}; - if (LOG.isDebugEnabled()) { - LOG.debug("Executing: " + Arrays.toString(cmd)); - } - - try { - Process process = Runtime.getRuntime().exec(cmd); - reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - line = reader.readLine(); - } finally { - if (reader != null) - reader.close(); - } - if (LOG.isDebugEnabled()) { - LOG.debug("bash -c " + command + " for group " + group + " returned " + line); - } - - parseMembers(line); - } - if (LOG.isDebugEnabled()) { - LOG.debug("End enumerating group members"); - } - } - - if (config.getEnumerateGroups() != null) { - String line; - String[] groups = config.getEnumerateGroups().split(","); - - if (LOG.isDebugEnabled()) { - LOG.debug("Adding extra groups"); - } - - for (String group : groups) { - String command = String.format(groupCmd, group); - String[] cmd = new String[]{"bash", "-c", command}; - if (LOG.isDebugEnabled()) { - LOG.debug("Executing: " + Arrays.toString(cmd)); - } - - try { - Process process = Runtime.getRuntime().exec(cmd); - reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - line = reader.readLine(); - } finally { - if (reader != null) - reader.close(); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("bash -c " + command + " for group " + group + " returned " + line); - } - - parseMembers(line); - } - if (LOG.isDebugEnabled()) { - LOG.debug("Done adding extra groups"); - } - } - } - - @VisibleForTesting - Map> getGroupUserListMap() { - return sourceGroupUsers; - } - - @VisibleForTesting - Map getGroupId2groupNameMap() { - return groupId2groupNameMap; - } - - @VisibleForTesting - Set getUsers() { - return sourceUsers.keySet(); - } +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +public class UnixUserGroupBuilder implements UserGroupSource { + private static final Logger LOG = LoggerFactory.getLogger(UnixUserGroupBuilder.class); + private static final String OS = System.getProperty("os.name"); + /** + * Shell commands to get users and groups + */ + static final String LINUX_GET_ALL_USERS_CMD = "getent passwd"; + static final String LINUX_GET_ALL_GROUPS_CMD = "getent group"; + static final String LINUX_GET_GROUP_CMD = "getent group %s"; + + // mainly for testing purposes, there might be a better way + static final String MAC_GET_ALL_USERS_CMD = "dscl . -readall /Users UniqueID PrimaryGroupID | " + + "awk 'BEGIN { OFS = \":\"; ORS=\"\\n\"; i=0;}" + + "/RecordName: / {name = $2;i = 0;}/PrimaryGroupID: / {gid = $2;}" + + "/^ / {if (i == 0) { i++; name = $1;}}" + + "/UniqueID: / {uid = $2;print name, \"*\", gid, uid;}'"; + static final String MAC_GET_ALL_GROUPS_CMD = "dscl . -list /Groups PrimaryGroupID | " + + "awk -v OFS=\":\" '{print $1, \"*\", $2, \"\"}'"; + static final String MAC_GET_GROUP_CMD = "dscl . -read /Groups/%1$s | paste -d, -s - | sed -e 's/:/|/g' | " + + "awk -v OFS=\":\" -v ORS=\"\\n\" -F, '{print \"%1$s\",\"*\",$6,$4}' | " + + "sed -e 's/:[^:]*| /:/g' | sed -e 's/ /,/g'"; + static final String BACKEND_PASSWD = "passwd"; + Set allGroups = new HashSet<>(); + private boolean useNss; + private boolean enumerateGroupMembers; + private boolean isUpdateSinkSucc = true; + private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); + private Map groupId2groupNameMap; + private Map> sourceUsers; // Stores username and attr name & value pairs + private Map> sourceGroups; // Stores groupname and attr name & value pairs + private Map> sourceGroupUsers; + private Table groupUserTable; // groupname, username, group id + private String unixPasswordFile; + private String unixGroupFile; + private String currentSyncSource; + private int deleteCycles; + private int minimumUserId; + private int minimumGroupId; + private long lastUpdateTime; + private long timeout; + private long passwordFileModifiedAt; + private long groupFileModifiedAt; + private UgsyncAuditInfo ugsyncAuditInfo; + private UnixSyncSourceInfo unixSyncSourceInfo; + private boolean isStartupFlag; + private boolean computeDeletes; + + public UnixUserGroupBuilder() { + isStartupFlag = true; + minimumUserId = Integer.parseInt(config.getMinUserId()); + minimumGroupId = Integer.parseInt(config.getMinGroupId()); + unixPasswordFile = config.getUnixPasswordFile(); + unixGroupFile = config.getUnixGroupFile(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Minimum UserId: {}, minimum GroupId: {}", minimumUserId, minimumGroupId); + } + + timeout = config.getUpdateMillisMin(); + enumerateGroupMembers = config.isGroupEnumerateEnabled(); + } + + public static void main(String[] args) throws Throwable { + UnixUserGroupBuilder ugbuilder = new UnixUserGroupBuilder(); + ugbuilder.init(); + if (LOG.isDebugEnabled()) { + ugbuilder.print(); + } + } + + @Override + public void init() throws Throwable { + deleteCycles = 1; + + currentSyncSource = config.getCurrentSyncSource(); + + ugsyncAuditInfo = new UgsyncAuditInfo(); + unixSyncSourceInfo = new UnixSyncSourceInfo(); + ugsyncAuditInfo.setSyncSource(currentSyncSource); + ugsyncAuditInfo.setUnixSyncSourceInfo(unixSyncSourceInfo); + unixSyncSourceInfo.setFileName(unixPasswordFile); + unixSyncSourceInfo.setMinUserId(config.getMinUserId()); + unixSyncSourceInfo.setMinGroupId(config.getMinGroupId()); + if (!config.getUnixBackend().equalsIgnoreCase(BACKEND_PASSWD)) { + useNss = true; + unixSyncSourceInfo.setUnixBackend("nss"); + } else { + LOG.warn("DEPRECATED: Unix backend is configured to use /etc/passwd and /etc/group files directly instead of standard system mechanisms"); + unixSyncSourceInfo.setUnixBackend(BACKEND_PASSWD); + } + buildUserGroupInfo(); + } + + @Override + public boolean isChanged() { + computeDeletes = false; + // If previous update to Ranger admin fails, we want to retry the sync process even if there are no changes to the sync files + if (!isUpdateSinkSucc) { + LOG.info("Previous updateSink failed and hence retry!!"); + isUpdateSinkSucc = true; + return true; + } + try { + if (config.isUserSyncDeletesEnabled() && deleteCycles >= config.getUserSyncDeletesFrequency()) { + deleteCycles = 1; + computeDeletes = true; + LOG.debug("Compute deleted users/groups is enabled for this sync cycle"); + return true; + } + } catch (Throwable t) { + LOG.error("Failed to get information about usersync delete frequency", t); + } + if (config.isUserSyncDeletesEnabled()) { + deleteCycles++; + } + + if (useNss) { + return System.currentTimeMillis() - lastUpdateTime > timeout; + } + + long tempPasswordFileModifiedAt = new File(unixPasswordFile).lastModified(); + if (passwordFileModifiedAt != tempPasswordFileModifiedAt) { + return true; + } + + long tempGroupFileModifiedAt = new File(unixGroupFile).lastModified(); + return groupFileModifiedAt != tempGroupFileModifiedAt; + } + + @Override + public void updateSink(UserGroupSink sink) throws Throwable { + DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + Date lastModifiedTime = new Date(passwordFileModifiedAt); + Date syncTime = new Date(System.currentTimeMillis()); + unixSyncSourceInfo.setLastModified(formatter.format(lastModifiedTime)); + unixSyncSourceInfo.setSyncTime(formatter.format(syncTime)); + if (isChanged() || isStartupFlag) { + buildUserGroupInfo(); + LOG.debug("Users = " + sourceUsers.keySet()); + LOG.debug("Groups = " + sourceGroups.keySet()); + LOG.debug("GroupUsers = " + sourceGroupUsers.keySet()); + + try { + sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers, sourceGroupUsers, computeDeletes); + } catch (Throwable t) { + LOG.error("Failed to update ranger admin. Will retry in next sync cycle!!", t); + isUpdateSinkSucc = false; + } + } + try { + sink.postUserGroupAuditInfo(ugsyncAuditInfo); + } catch (Throwable t) { + LOG.error("sink.postUserGroupAuditInfo failed with exception: ", t); + } + isStartupFlag = false; + } + + @VisibleForTesting + Map> getGroupUserListMap() { + return sourceGroupUsers; + } + + @VisibleForTesting + Map getGroupId2groupNameMap() { + return groupId2groupNameMap; + } + + @VisibleForTesting + Set getUsers() { + return sourceUsers.keySet(); + } + + private void buildUserGroupInfo() throws Throwable { + groupId2groupNameMap = new HashMap<>(); + sourceUsers = new HashMap<>(); + sourceGroups = new HashMap<>(); + sourceGroupUsers = new HashMap<>(); + groupUserTable = HashBasedTable.create(); + allGroups = new HashSet<>(); + + if (OS.startsWith("Mac")) { + buildUnixGroupList(MAC_GET_ALL_GROUPS_CMD, MAC_GET_GROUP_CMD, false); + buildUnixUserList(MAC_GET_ALL_USERS_CMD); + } else { + if (!OS.startsWith("Linux")) { + LOG.warn("Platform not recognized assuming Linux compatible"); + } + buildUnixGroupList(LINUX_GET_ALL_GROUPS_CMD, LINUX_GET_GROUP_CMD, true); + buildUnixUserList(LINUX_GET_ALL_USERS_CMD); + } + + Iterator groupUserTableIterator = groupUserTable.rowKeySet().iterator(); + while (groupUserTableIterator.hasNext()) { + String groupName = groupUserTableIterator.next(); + Map groupUsersMap = groupUserTable.row(groupName); + Set userSet = new HashSet<>(); + + for (String userName : groupUsersMap.keySet()) { + if (sourceUsers.containsKey(userName)) { + userSet.add(userName); + } + } + sourceGroupUsers.put(groupName, userSet); + } + + lastUpdateTime = System.currentTimeMillis(); + + if (LOG.isDebugEnabled()) { + print(); + } + } + + private void print() { + for (String user : sourceUsers.keySet()) { + LOG.debug("USER:" + user); + Set groups = groupUserTable.column(user).keySet(); + for (String group : groups) { + LOG.debug("\tGROUP: " + group); + } + } + } + + private void buildUnixUserList(String command) throws Throwable { + BufferedReader reader = null; + Map userName2uid = new HashMap<>(); + + try { + if (!useNss) { + File file = new File(unixPasswordFile); + passwordFileModifiedAt = file.lastModified(); + FileInputStream fis = new FileInputStream(file); + reader = new BufferedReader(new InputStreamReader(fis, StandardCharsets.UTF_8)); + } else { + Process process = Runtime.getRuntime().exec(new String[] {"bash", "-c", command}); + reader = new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8)); + } + + String line; + while ((line = reader.readLine()) != null) { + if (line.trim().isEmpty()) { + continue; + } + + String[] tokens = line.split(":"); + + int len = tokens.length; + + if (len < 4) { + LOG.warn("Unable to parse: " + line); + continue; + } + + String userName; + String userId; + String groupId; + + try { + userName = tokens[0]; + userId = tokens[2]; + groupId = tokens[3]; + } catch (ArrayIndexOutOfBoundsException aiobe) { + LOG.warn("Ignoring line - [" + line + "]: Unable to parse line for getting user information", aiobe); + continue; + } + + int numUserId; + try { + numUserId = Integer.parseInt(userId); + } catch (NumberFormatException nfe) { + LOG.warn("Unix UserId: [" + userId + "]: can not be parsed as valid int. considering as -1.", nfe); + numUserId = -1; + } + + if (numUserId >= minimumUserId) { + userName2uid.put(userName, userId); + String groupName = groupId2groupNameMap.get(groupId); + if (groupName != null) { + Map userAttrMap = new HashMap<>(); + userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName); + userAttrMap.put(UgsyncCommonConstants.FULL_NAME, userName); + userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); + sourceUsers.put(userName, userAttrMap); + groupUserTable.put(groupName, userName, groupId); + } else { + // we are ignoring the possibility that this user was present in getent group. + LOG.warn("Group Name could not be found for group id: [{}]. Skipping adding user [{}] with id [{}]", groupId, userName, userId); + } + } else { + LOG.debug("Skipping user [{}] since its userid [{}] is less than minuserid limit [{}]", userName, userId, minimumUserId); + } + } + } finally { + if (reader != null) { + reader.close(); + } + } + + if (!useNss) { + return; + } + + // this does a reverse check as not all users might be listed in getent passwd + if (enumerateGroupMembers) { + String line; + Table tempGroupUserTable = HashBasedTable.create(); + + LOG.debug("Start drill down group members"); + for (String userName : groupUserTable.columnKeySet()) { + if (sourceUsers.containsKey(userName)) { // skip users we already now about + continue; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Enumerating user {}", userName); + } + + // "id" is same across Linux / BSD / MacOSX + // gids are used as id might return groups with spaces, ie "domain users" + Process process = Runtime.getRuntime().exec(new String[] {"bash", "-c", "id -G " + userName}); + + try { + reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + line = reader.readLine(); + } finally { + reader.close(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("id -G returned " + line); + } + + if (line == null || line.trim().isEmpty()) { + LOG.warn("User " + userName + " could not be resolved"); + continue; + } + + // check if all groups returned by id are visible to ranger + for (String gid : line.split(" ")) { + if (Integer.parseInt(gid) >= minimumGroupId) { + String groupName = groupId2groupNameMap.get(gid); + if (groupName != null) { + LOG.debug("New group user mapping found : {} {}", groupName, userName); + tempGroupUserTable.put(groupName, userName, gid); + } + } + } + + Map userAttrMap = new HashMap<>(); + userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName); + userAttrMap.put(UgsyncCommonConstants.FULL_NAME, userName); + userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); + sourceUsers.put(userName, userAttrMap); + } + groupUserTable.putAll(tempGroupUserTable); + if (LOG.isDebugEnabled()) { + LOG.debug("End drill down group members"); + } + } + } + + private void parseMembers(String line) { + if (line == null || line.isEmpty()) { + return; + } + + String[] tokens = line.split(":"); + + if (tokens.length < 3) { + return; + } + + String groupName = tokens[0]; + String groupId = tokens[2]; + String groupMembers = null; + + if (tokens.length > 3) { + groupMembers = tokens[3]; + } + + groupId2groupNameMap.remove(groupId); + + int numGroupId = Integer.parseInt(groupId); + if (numGroupId < minimumGroupId) { + return; + } + + groupId2groupNameMap.put(groupId, groupName); + Map groupAttrMap = new HashMap<>(); + groupAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, groupName); + groupAttrMap.put(UgsyncCommonConstants.FULL_NAME, groupName); + groupAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); + sourceGroups.put(groupName, groupAttrMap); + + if (groupMembers != null && !groupMembers.trim().isEmpty()) { + for (String user : groupMembers.split(",")) { + groupUserTable.put(groupName, user, groupId); + } + } else { + sourceGroupUsers.put(groupName, new HashSet()); + } + } + + private void buildUnixGroupList(String allGroupsCmd, String groupCmd, boolean useGid) throws Throwable { + LOG.debug("Start enumerating groups"); + + BufferedReader reader = null; + + try { + if (!useNss) { + File file = new File(unixGroupFile); + groupFileModifiedAt = file.lastModified(); + FileInputStream fis = new FileInputStream(file); + reader = new BufferedReader(new InputStreamReader(fis, StandardCharsets.UTF_8)); + } else { + Process process = Runtime.getRuntime().exec(new String[] {"bash", "-c", allGroupsCmd}); + reader = new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8)); + } + + String line; + while ((line = reader.readLine()) != null) { + if (!line.trim().isEmpty()) { + parseMembers(line); + } + } + } finally { + if (reader != null) { + reader.close(); + } + } + + LOG.debug("End enumerating group"); + + if (!useNss) { + return; + } + + if (enumerateGroupMembers) { + LOG.debug("Start enumerating group members"); + String line = null; + Map copy = new HashMap(groupId2groupNameMap); + + for (Map.Entry group : copy.entrySet()) { + LOG.debug("Enumerating group: {} GID({})", group.getValue(), group.getKey()); + + String command; + if (useGid) { + command = String.format(groupCmd, group.getKey()); + } else { + command = String.format(groupCmd, group.getValue()); + } + + String[] cmd = new String[] {"bash", "-c", command + " " + group.getKey()}; + if (LOG.isDebugEnabled()) { + LOG.debug("Executing: {}", Arrays.toString(cmd)); + } + + try { + Process process = Runtime.getRuntime().exec(cmd); + reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + line = reader.readLine(); + } finally { + if (reader != null) { + reader.close(); + } + } + + LOG.debug("bash -c {} for group {} returned {}", command, group, line); + + parseMembers(line); + } + LOG.debug("End enumerating group members"); + } + + if (config.getEnumerateGroups() != null) { + String line; + String[] groups = config.getEnumerateGroups().split(","); + + LOG.debug("Adding extra groups"); + + for (String group : groups) { + String command = String.format(groupCmd, group); + String[] cmd = new String[] {"bash", "-c", command}; + if (LOG.isDebugEnabled()) { + LOG.debug("Executing: {}", Arrays.toString(cmd)); + } + + try { + Process process = Runtime.getRuntime().exec(cmd); + reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + line = reader.readLine(); + } finally { + if (reader != null) { + reader.close(); + } + } + + LOG.debug("bash -c {} for group {} returned {}", command, group, line); + parseMembers(line); + } + LOG.debug("Done adding extra groups"); + } + } } diff --git a/ugsync/src/main/java/org/apache/ranger/usergroupsync/AbstractMapper.java b/ugsync/src/main/java/org/apache/ranger/usergroupsync/AbstractMapper.java index bbbc3c4d13..e090175fb3 100644 --- a/ugsync/src/main/java/org/apache/ranger/usergroupsync/AbstractMapper.java +++ b/ugsync/src/main/java/org/apache/ranger/usergroupsync/AbstractMapper.java @@ -23,19 +23,16 @@ import org.slf4j.LoggerFactory; public abstract class AbstractMapper implements Mapper { - - protected static final Logger logger = LoggerFactory.getLogger(AbstractMapper.class); + protected static final Logger logger = LoggerFactory.getLogger(AbstractMapper.class); - @Override - public void init(String baseProperty) { - // TODO Auto-generated method stub - - } - - @Override - public String transform(String attrValue) { - // TODO Auto-generated method stub - return null; - } + @Override + public void init(String baseProperty) { + // TODO Auto-generated method stub + } + @Override + public String transform(String attrValue) { + // TODO Auto-generated method stub + return null; + } } diff --git a/ugsync/src/main/java/org/apache/ranger/usergroupsync/AbstractUserGroupSource.java b/ugsync/src/main/java/org/apache/ranger/usergroupsync/AbstractUserGroupSource.java index 87ea787df9..2fcf210b70 100644 --- a/ugsync/src/main/java/org/apache/ranger/usergroupsync/AbstractUserGroupSource.java +++ b/ugsync/src/main/java/org/apache/ranger/usergroupsync/AbstractUserGroupSource.java @@ -18,25 +18,22 @@ */ package org.apache.ranger.usergroupsync; +import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; public abstract class AbstractUserGroupSource { - private static final Logger LOG = LoggerFactory.getLogger(AbstractUserGroupSource.class); protected UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); - - protected Mapper userNameRegExInst = null; - protected Mapper groupNameRegExInst = null; - + protected Mapper userNameRegExInst; + protected Mapper groupNameRegExInst; public AbstractUserGroupSource() { String mappingUserNameHandler = config.getUserSyncMappingUserNameHandler(); try { if (mappingUserNameHandler != null) { - Class regExClass = (Class)Class.forName(mappingUserNameHandler); + Class regExClass = (Class) Class.forName(mappingUserNameHandler); userNameRegExInst = regExClass.newInstance(); if (userNameRegExInst != null) { userNameRegExInst.init(UserGroupSyncConfig.SYNC_MAPPING_USERNAME); @@ -45,15 +42,15 @@ public AbstractUserGroupSource() { } } } catch (ClassNotFoundException cne) { - LOG.error("Failed to load " + mappingUserNameHandler + " " + cne); + LOG.error("Failed to load {}: {}", mappingUserNameHandler, cne); } catch (Throwable te) { - LOG.error("Failed to instantiate " + mappingUserNameHandler + " " + te); + LOG.error("Failed to instantiate {}: {}", mappingUserNameHandler, te); } String mappingGroupNameHandler = config.getUserSyncMappingGroupNameHandler(); try { if (mappingGroupNameHandler != null) { - Class regExClass = (Class)Class.forName(mappingGroupNameHandler); + Class regExClass = (Class) Class.forName(mappingGroupNameHandler); groupNameRegExInst = regExClass.newInstance(); if (groupNameRegExInst != null) { groupNameRegExInst.init(UserGroupSyncConfig.SYNC_MAPPING_GROUPNAME); @@ -62,10 +59,9 @@ public AbstractUserGroupSource() { } } } catch (ClassNotFoundException cne) { - LOG.error("Failed to load " + mappingGroupNameHandler + " " + cne); + LOG.error("Failed to load {}: {}", mappingGroupNameHandler, cne); } catch (Throwable te) { - LOG.error("Failed to instantiate " + mappingGroupNameHandler + " " + te); + LOG.error("Failed to instantiate {}: {}", mappingGroupNameHandler, te); } } - } diff --git a/ugsync/src/main/java/org/apache/ranger/usergroupsync/Mapper.java b/ugsync/src/main/java/org/apache/ranger/usergroupsync/Mapper.java index 696c665304..a1daba8c2f 100644 --- a/ugsync/src/main/java/org/apache/ranger/usergroupsync/Mapper.java +++ b/ugsync/src/main/java/org/apache/ranger/usergroupsync/Mapper.java @@ -20,7 +20,7 @@ package org.apache.ranger.usergroupsync; public interface Mapper { - void init(String baseProperty); + void init(String baseProperty); String transform(String attrValue); } diff --git a/ugsync/src/main/java/org/apache/ranger/usergroupsync/RegEx.java b/ugsync/src/main/java/org/apache/ranger/usergroupsync/RegEx.java index 9e5ca4ca58..4e97c7e097 100644 --- a/ugsync/src/main/java/org/apache/ranger/usergroupsync/RegEx.java +++ b/ugsync/src/main/java/org/apache/ranger/usergroupsync/RegEx.java @@ -19,71 +19,69 @@ package org.apache.ranger.usergroupsync; +import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; + import java.util.LinkedHashMap; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; - public class RegEx extends AbstractMapper { - private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); - private LinkedHashMap replacementPattern; + private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); + private LinkedHashMap replacementPattern; - public LinkedHashMap getReplacementPattern() { - return replacementPattern; - } + public LinkedHashMap getReplacementPattern() { + return replacementPattern; + } - @Override - public void init (String baseProperty) { - logger.info("Initializing for " + baseProperty); - try { - List regexPatterns = config.getAllRegexPatterns(baseProperty); - String regexSeparator = config.getRegexSeparator(); - populateReplacementPatterns(baseProperty, regexPatterns, regexSeparator); - } catch (Throwable t) { - logger.error("Failed to initialize " + baseProperty, t.fillInStackTrace()); - } - } + @Override + public void init(String baseProperty) { + logger.info("Initializing for {}", baseProperty); + try { + List regexPatterns = config.getAllRegexPatterns(baseProperty); + String regexSeparator = config.getRegexSeparator(); + populateReplacementPatterns(baseProperty, regexPatterns, regexSeparator); + } catch (Throwable t) { + logger.error("Failed to initialize {}", baseProperty, t.fillInStackTrace()); + } + } - protected void populateReplacementPatterns(String baseProperty, List regexPatterns, String regexSeparator) throws Throwable { - replacementPattern = new LinkedHashMap(); - String regex = String.format("s%s([^%s]*)%s([^%s]*)%s(g)?", regexSeparator, regexSeparator, regexSeparator, regexSeparator, regexSeparator); - Pattern p = Pattern.compile(regex); - for (String regexPattern : regexPatterns) { - Matcher m = p.matcher(regexPattern); - if (!m.matches()) { - logger.warn("Invalid RegEx " + regexPattern + " and hence skipping this regex property"); - } - m = m.reset(); - while (m.find()) { - String matchPattern = m.group(1); - String replacement = m.group(2); - if (matchPattern != null && !matchPattern.isEmpty() && replacement != null) { - replacementPattern.put(matchPattern, replacement); - if (logger.isDebugEnabled()) { - logger.debug(baseProperty + " match pattern = " + matchPattern + " and replacement string = " + replacement); - } - } - } - } - } + @Override + public String transform(String attrValue) { + String result = attrValue; + try { + if (replacementPattern != null && !replacementPattern.isEmpty()) { + for (String matchPattern : replacementPattern.keySet()) { + String replacement = replacementPattern.get(matchPattern); + Pattern p = Pattern.compile(matchPattern); + Matcher m = p.matcher(result); + result = m.replaceAll(replacement); + } + } + } catch (Throwable t) { + logger.error("Failed to transform {}", attrValue, t.fillInStackTrace()); + } + return result; + } - @Override - public String transform (String attrValue) { - String result = attrValue; - try { - if (replacementPattern != null && !replacementPattern.isEmpty()) { - for (String matchPattern: replacementPattern.keySet()) { - String replacement = replacementPattern.get(matchPattern); - Pattern p = Pattern.compile(matchPattern); - Matcher m = p.matcher(result); - result = m.replaceAll(replacement); - } - } - } catch (Throwable t) { - logger.error("Failed to transform " + attrValue, t.fillInStackTrace()); - } - return result; - } + protected void populateReplacementPatterns(String baseProperty, List regexPatterns, String regexSeparator) throws Throwable { + replacementPattern = new LinkedHashMap(); + String regex = String.format("s%s([^%s]*)%s([^%s]*)%s(g)?", regexSeparator, regexSeparator, regexSeparator, regexSeparator, regexSeparator); + Pattern p = Pattern.compile(regex); + for (String regexPattern : regexPatterns) { + Matcher m = p.matcher(regexPattern); + if (!m.matches()) { + logger.warn("Invalid RegEx {} and hence skipping this regex property", regexPattern); + } + m = m.reset(); + while (m.find()) { + String matchPattern = m.group(1); + String replacement = m.group(2); + if (matchPattern != null && !matchPattern.isEmpty() && replacement != null) { + replacementPattern.put(matchPattern, replacement); + logger.debug("{} match pattern = {} and replacement string = {}", baseProperty, matchPattern, replacement); + } + } + } + } } diff --git a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSink.java b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSink.java index e79fcf758b..5dad3af5f8 100644 --- a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSink.java +++ b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSink.java @@ -17,7 +17,7 @@ * under the License. */ - package org.apache.ranger.usergroupsync; +package org.apache.ranger.usergroupsync; import org.apache.ranger.ugsyncutil.model.UgsyncAuditInfo; @@ -25,13 +25,12 @@ import java.util.Set; public interface UserGroupSink { - void init() throws Throwable; + void init() throws Throwable; - void postUserGroupAuditInfo(UgsyncAuditInfo ugsyncAuditInfo) throws Throwable; - - void addOrUpdateUsersGroups(Map> sourceGroups, - Map> sourceUsers, - Map> sourceGroupUsers, - boolean computeDeletes) throws Throwable; + void postUserGroupAuditInfo(UgsyncAuditInfo ugsyncAuditInfo) throws Throwable; + void addOrUpdateUsersGroups(Map> sourceGroups, + Map> sourceUsers, + Map> sourceGroupUsers, + boolean computeDeletes) throws Throwable; } diff --git a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSource.java b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSource.java index 81a546ec9c..a09bf7c446 100644 --- a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSource.java +++ b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSource.java @@ -17,13 +17,12 @@ * under the License. */ - package org.apache.ranger.usergroupsync; - +package org.apache.ranger.usergroupsync; public interface UserGroupSource { - void init() throws Throwable; + void init() throws Throwable; - boolean isChanged(); + boolean isChanged(); - void updateSink(UserGroupSink sink) throws Throwable; + void updateSink(UserGroupSink sink) throws Throwable; } diff --git a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java index 361842c2ef..87bf6566a0 100644 --- a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java +++ b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java @@ -19,102 +19,97 @@ package org.apache.ranger.usergroupsync; +import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; public class UserGroupSync implements Runnable { - - private static final Logger LOG = LoggerFactory.getLogger(UserGroupSync.class); - - private UserGroupSink ugSink; - private UserGroupSource ugSource; - - public static void main(String[] args) { - UserGroupSync userGroupSync = new UserGroupSync(); - userGroupSync.run(); - } - - @Override - public void run() { - try { - long sleepTimeBetweenCycleInMillis = UserGroupSyncConfig.getInstance().getSleepTimeInMillisBetweenCycle(); - long initSleepTimeBetweenCycleInMillis = UserGroupSyncConfig.getInstance().getInitSleepTimeInMillisBetweenCycle(); - boolean initPending = true; - - while (initPending) { - try { - if (UserGroupSyncConfig.isUgsyncServiceActive()) { - ugSink = UserGroupSyncConfig.getInstance().getUserGroupSink(); - LOG.info("initializing sink: " + ugSink.getClass().getName()); - ugSink.init(); - - ugSource = UserGroupSyncConfig.getInstance().getUserGroupSource(); - LOG.info("initializing source: " + ugSource.getClass().getName()); - ugSource.init(); - - LOG.info("Begin: initial load of user/group from source==>sink"); - syncUserGroup(); - LOG.info("End: initial load of user/group from source==>sink"); - - initPending = false; - LOG.info("Done initializing user/group source and sink"); - }else { - if (LOG.isDebugEnabled()){ - LOG.debug("Sleeping for [" + initSleepTimeBetweenCycleInMillis + "] milliSeconds as this server is running in passive mode"); - } - Thread.sleep(initSleepTimeBetweenCycleInMillis); - } - } catch (Throwable t) { - LOG.error("Failed to initialize UserGroup source/sink. Will retry after " + sleepTimeBetweenCycleInMillis + " milliseconds. Error details: ", t); - try { - if (LOG.isDebugEnabled()){ - LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); - } - Thread.sleep(sleepTimeBetweenCycleInMillis); - } catch (Exception e) { - LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to initialize UserGroup source/sink", e); - } - } - } - - while (true) { - try { - if (LOG.isDebugEnabled()){ - LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); - } - Thread.sleep(sleepTimeBetweenCycleInMillis); - } catch (InterruptedException e) { - LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to synchronize UserGroup information", e); - } - - try { - if (UserGroupSyncConfig.isUgsyncServiceActive()) { - LOG.info("Begin: update user/group from source==>sink"); - syncUserGroup(); - LOG.info("End: update user/group from source==>sink"); - } else { - LOG.info("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds as this server is running in passive mode"); - } - } catch (Throwable t) { - LOG.error("Failed to synchronize UserGroup information. Error details: ", t); - } - } - - } catch (Throwable t) { - LOG.error("UserGroupSync thread got an error", t); - } finally { - LOG.info("Shutting down the UserGroupSync thread"); - } - } - - private void syncUserGroup() throws Throwable { - UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); - - if (config.isUserSyncEnabled()) { - ugSource.updateSink(ugSink); - } - - } - + private static final Logger LOG = LoggerFactory.getLogger(UserGroupSync.class); + private UserGroupSink ugSink; + private UserGroupSource ugSource; + + public static void main(String[] args) { + UserGroupSync userGroupSync = new UserGroupSync(); + userGroupSync.run(); + } + + @Override + public void run() { + try { + long sleepTimeBetweenCycleInMillis = UserGroupSyncConfig.getInstance().getSleepTimeInMillisBetweenCycle(); + long initSleepTimeBetweenCycleInMillis = UserGroupSyncConfig.getInstance().getInitSleepTimeInMillisBetweenCycle(); + boolean initPending = true; + + while (initPending) { + try { + if (UserGroupSyncConfig.isUgsyncServiceActive()) { + ugSink = UserGroupSyncConfig.getInstance().getUserGroupSink(); + LOG.info("initializing sink: " + ugSink.getClass().getName()); + ugSink.init(); + + ugSource = UserGroupSyncConfig.getInstance().getUserGroupSource(); + LOG.info("initializing source: " + ugSource.getClass().getName()); + ugSource.init(); + + LOG.info("Begin: initial load of user/group from source==>sink"); + syncUserGroup(); + LOG.info("End: initial load of user/group from source==>sink"); + + initPending = false; + LOG.info("Done initializing user/group source and sink"); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Sleeping for [" + initSleepTimeBetweenCycleInMillis + "] milliSeconds as this server is running in passive mode"); + } + Thread.sleep(initSleepTimeBetweenCycleInMillis); + } + } catch (Throwable t) { + LOG.error("Failed to initialize UserGroup source/sink. Will retry after " + sleepTimeBetweenCycleInMillis + " milliseconds. Error details: ", t); + try { + if (LOG.isDebugEnabled()) { + LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); + } + Thread.sleep(sleepTimeBetweenCycleInMillis); + } catch (Exception e) { + LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to initialize UserGroup source/sink", e); + } + } + } + + while (true) { + try { + if (LOG.isDebugEnabled()) { + LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); + } + Thread.sleep(sleepTimeBetweenCycleInMillis); + } catch (InterruptedException e) { + LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to synchronize UserGroup information", e); + } + + try { + if (UserGroupSyncConfig.isUgsyncServiceActive()) { + LOG.info("Begin: update user/group from source==>sink"); + syncUserGroup(); + LOG.info("End: update user/group from source==>sink"); + } else { + LOG.info("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds as this server is running in passive mode"); + } + } catch (Throwable t) { + LOG.error("Failed to synchronize UserGroup information. Error details: ", t); + } + } + } catch (Throwable t) { + LOG.error("UserGroupSync thread got an error", t); + } finally { + LOG.info("Shutting down the UserGroupSync thread"); + } + } + + private void syncUserGroup() throws Throwable { + UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); + + if (config.isUserSyncEnabled()) { + ugSource.updateSink(ugSink); + } + } } diff --git a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserSyncMetricsProducer.java b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserSyncMetricsProducer.java index 1e80c5da87..e3e028a7ac 100644 --- a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserSyncMetricsProducer.java +++ b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserSyncMetricsProducer.java @@ -19,85 +19,79 @@ package org.apache.ranger.usergroupsync; -import java.io.File; - import org.apache.hadoop.conf.Configuration; import org.apache.ranger.plugin.util.RangerMetricsUtil; import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class UserSyncMetricsProducer implements Runnable { - - private static final Logger LOG = LoggerFactory.getLogger(UserSyncMetricsProducer.class); - private boolean shutdownFlag = false; +import java.io.File; - public static void main(String[] args) { - UserSyncMetricsProducer userSyncMetrics = new UserSyncMetricsProducer(); - userSyncMetrics.run(); - /* - * try { userSyncMetrics.writeJVMMetrics(); } catch (Throwable e) { // TODO - * Auto-generated catch block e.printStackTrace(); } - */ - } +public class UserSyncMetricsProducer implements Runnable { + private static final Logger LOG = LoggerFactory.getLogger(UserSyncMetricsProducer.class); + private boolean shutdownFlag; - @Override - public void run() { - try { - UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); - long sleepTimeBetweenCycleInMillis = config.getUserSyncMetricsFrequency(); - String logFileNameWithPath = config.getUserSyncMetricsFileName(); - LOG.info("user sync metrics frequency : " + sleepTimeBetweenCycleInMillis + " and metrics file : " - + logFileNameWithPath); - if (null != logFileNameWithPath) { - while (!shutdownFlag) { - try { - if (LOG.isDebugEnabled()) { - LOG.debug("Sleeping metrics for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); - } - Thread.sleep(sleepTimeBetweenCycleInMillis); - } catch (InterruptedException e) { - LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis - + "] milliseconds before attempting to fetch userSync metrics information", e); - } - try { - writeJVMMetrics(logFileNameWithPath); - } catch (Throwable t) { - LOG.error("Failed to write user sync metrics into file. Error details: ", t); - } - } - } else { - LOG.info("No file directory found for usersync metrics log "); - } - } catch (Throwable t) { - LOG.error("Failed to start user sync metrics. Error details: ", t); - } finally { - LOG.info("Shutting down the User Sync metrics producer thread"); - } + public static void main(String[] args) { + UserSyncMetricsProducer userSyncMetrics = new UserSyncMetricsProducer(); + userSyncMetrics.run(); + /* TODO + * try { userSyncMetrics.writeJVMMetrics(); } catch (Throwable e) { + * Auto-generated catch block e.printStackTrace(); } + */ + } - } + @Override + public void run() { + try { + UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); + long sleepTimeBetweenCycleInMillis = config.getUserSyncMetricsFrequency(); + String logFileNameWithPath = config.getUserSyncMetricsFileName(); + LOG.info("User Sync metrics frequency : {} and metrics file : {}", sleepTimeBetweenCycleInMillis, logFileNameWithPath); + if (logFileNameWithPath != null) { + while (!shutdownFlag) { + try { + LOG.debug("Sleeping metrics for {} milliSeconds", sleepTimeBetweenCycleInMillis); + Thread.sleep(sleepTimeBetweenCycleInMillis); + } catch (InterruptedException e) { + LOG.error("Failed to wait for [{}] milliseconds before attempting to fetch userSync metrics information", sleepTimeBetweenCycleInMillis, e); + } + try { + writeJVMMetrics(logFileNameWithPath); + } catch (Throwable t) { + LOG.error("Failed to write user sync metrics into file. Error details: ", t); + } + } + } else { + LOG.info("No file directory found for usersync metrics log "); + } + } catch (Throwable t) { + LOG.error("Failed to start user sync metrics. Error details: ", t); + } finally { + LOG.info("Shutting down the User Sync metrics producer thread"); + } + } - private void writeJVMMetrics(String logFileNameWithPath) throws Throwable { - try { - File userMetricFile = null; - UserGroupSyncConfig userConfig = UserGroupSyncConfig.getInstance(); - Configuration config = userConfig.getUserGroupConfig(); - userMetricFile = new File(logFileNameWithPath); - if (!userMetricFile.exists()) { - userMetricFile.createNewFile(); - } - RangerMetricsUtil rangerMetricsUtil = new RangerMetricsUtil(); - if (config.getBoolean(UserGroupSyncConfig.UGSYNC_SERVER_HA_ENABLED_PARAM, false)) { - if(userConfig.isUgsyncServiceActive()){ - rangerMetricsUtil.setIsRoleActive(1); - }else{ - rangerMetricsUtil.setIsRoleActive(0); - } - } - rangerMetricsUtil.writeMetricsToFile(userMetricFile); - } catch (Throwable t) { - LOG.error("UserSyncMetricsProducer.writeJVMMetrics() failed to write metrics into file. Error details: ", t); - throw t; - } - } + private void writeJVMMetrics(String logFileNameWithPath) throws Throwable { + try { + File userMetricFile = null; + UserGroupSyncConfig userConfig = UserGroupSyncConfig.getInstance(); + Configuration config = userConfig.getUserGroupConfig(); + userMetricFile = new File(logFileNameWithPath); + if (!userMetricFile.exists()) { + userMetricFile.createNewFile(); + } + RangerMetricsUtil rangerMetricsUtil = new RangerMetricsUtil(); + if (config.getBoolean(UserGroupSyncConfig.UGSYNC_SERVER_HA_ENABLED_PARAM, false)) { + if (userConfig.isUgsyncServiceActive()) { + rangerMetricsUtil.setIsRoleActive(1); + } else { + rangerMetricsUtil.setIsRoleActive(0); + } + } + rangerMetricsUtil.writeMetricsToFile(userMetricFile); + } catch (Throwable t) { + LOG.error("UserSyncMetricsProducer.writeJVMMetrics() failed to write metrics into file. Error details: ", t); + throw t; + } + } } diff --git a/ugsync/src/test/java/org/apache/ranger/unixusersync/process/TestFileSourceUserGroupBuilder.java b/ugsync/src/test/java/org/apache/ranger/unixusersync/process/TestFileSourceUserGroupBuilder.java index 6b01ba7ad0..dcedbb183d 100644 --- a/ugsync/src/test/java/org/apache/ranger/unixusersync/process/TestFileSourceUserGroupBuilder.java +++ b/ugsync/src/test/java/org/apache/ranger/unixusersync/process/TestFileSourceUserGroupBuilder.java @@ -18,15 +18,14 @@ */ package org.apache.ranger.unixusersync.process; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertEquals; - import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; import org.apache.ranger.usergroupsync.PolicyMgrUserGroupBuilderTest; import org.junit.Test; -public class TestFileSourceUserGroupBuilder { +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +public class TestFileSourceUserGroupBuilder { private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); @Test diff --git a/ugsync/src/test/java/org/apache/ranger/unixusersync/process/TestUnixUserGroupBuilder.java b/ugsync/src/test/java/org/apache/ranger/unixusersync/process/TestUnixUserGroupBuilder.java index 526efa0b54..5d379b6cb2 100644 --- a/ugsync/src/test/java/org/apache/ranger/unixusersync/process/TestUnixUserGroupBuilder.java +++ b/ugsync/src/test/java/org/apache/ranger/unixusersync/process/TestUnixUserGroupBuilder.java @@ -27,14 +27,14 @@ import java.util.Set; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertFalse; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasValue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; public class TestUnixUserGroupBuilder { private UserGroupSyncConfig config; @@ -50,21 +50,19 @@ public void setUp() { public void testBuilderPasswd() throws Throwable { config.setProperty("ranger.usersync.unix.backend", "passwd"); config.setProperty(UserGroupSyncConfig.UGSYNC_UNIX_PASSWORD_FILE, "/etc/passwd"); - config.setProperty(UserGroupSyncConfig.UGSYNC_UNIX_GROUP_FILE, "/etc/group"); - + config.setProperty(UserGroupSyncConfig.UGSYNC_UNIX_GROUP_FILE, "/etc/group"); UnixUserGroupBuilder builder = new UnixUserGroupBuilder(); builder.init(); Map groups = builder.getGroupId2groupNameMap(); - String name = groups.get("0"); + String name = groups.get("0"); assertThat(name, anyOf(equalTo("wheel"), equalTo("root"))); Map> groupUsers = builder.getGroupUserListMap(); - Set users = groupUsers.get(name); + Set users = groupUsers.get(name); assertNotNull(users); assertThat(users, anyOf(hasItem("wheel"), hasItem("root"))); - } @Test @@ -75,11 +73,11 @@ public void testBuilderNss() throws Throwable { builder.init(); Map groups = builder.getGroupId2groupNameMap(); - String name = groups.get("0"); + String name = groups.get("0"); assertThat(name, anyOf(equalTo("wheel"), equalTo("root"))); Map> groupUsers = builder.getGroupUserListMap(); - Set users = groupUsers.get(name); + Set users = groupUsers.get(name); assertNotNull(users); assertThat(users, anyOf(hasItem("wheel"), hasItem("root"))); } @@ -114,25 +112,23 @@ public void testMinUidGid() throws Throwable { Map> groupUsers = builder.getGroupUserListMap(); assertNull(groupUsers.get("wheel")); } - + @Test public void testUnixPasswdAndGroupFile() throws Throwable { - config.setProperty("ranger.usersync.unix.backend", "passwd"); - config.setProperty(UserGroupSyncConfig.UGSYNC_UNIX_PASSWORD_FILE, "src/test/resources/passwordFile.txt"); - config.setProperty(UserGroupSyncConfig.UGSYNC_UNIX_GROUP_FILE, "src/test/resources/groupFile.txt"); + config.setProperty("ranger.usersync.unix.backend", "passwd"); + config.setProperty(UserGroupSyncConfig.UGSYNC_UNIX_PASSWORD_FILE, "src/test/resources/passwordFile.txt"); + config.setProperty(UserGroupSyncConfig.UGSYNC_UNIX_GROUP_FILE, "src/test/resources/groupFile.txt"); UnixUserGroupBuilder builder = new UnixUserGroupBuilder(); builder.init(); Map groups = builder.getGroupId2groupNameMap(); - String name = groups.get("1028"); + String name = groups.get("1028"); assertThat(name, anyOf(equalTo("wheel"), equalTo("sam"))); Map> groupUsers = builder.getGroupUserListMap(); - Set users = groupUsers.get("sam"); + Set users = groupUsers.get("sam"); assertNotNull(groupUsers); assertThat(users, anyOf(hasItem("wheel"), hasItem("sam"))); - } - } diff --git a/ugsync/src/test/java/org/apache/ranger/usergroupsync/PolicyMgrUserGroupBuilderTest.java b/ugsync/src/test/java/org/apache/ranger/usergroupsync/PolicyMgrUserGroupBuilderTest.java index fd8181be93..7897439050 100644 --- a/ugsync/src/test/java/org/apache/ranger/usergroupsync/PolicyMgrUserGroupBuilderTest.java +++ b/ugsync/src/test/java/org/apache/ranger/usergroupsync/PolicyMgrUserGroupBuilderTest.java @@ -19,101 +19,92 @@ package org.apache.ranger.usergroupsync; +import org.apache.commons.collections.CollectionUtils; +import org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder; + import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; -import org.apache.commons.collections.CollectionUtils; -import org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder; public class PolicyMgrUserGroupBuilderTest extends PolicyMgrUserGroupBuilder { - private Set allGroups; - private Set allUsers; - private Map> groupUsers; - private Set invalidGroups; - private Set invalidUsers; - - public PolicyMgrUserGroupBuilderTest() { - super(); + private Set allGroups; + private Set allUsers; + private Map> groupUsers; + private Set invalidGroups; + private Set invalidUsers; + + public PolicyMgrUserGroupBuilderTest() { + super(); + } + + @Override + public void init() throws Throwable { + allGroups = new HashSet<>(); + allUsers = new HashSet<>(); + groupUsers = new HashMap<>(); + invalidGroups = new HashSet<>(); + invalidUsers = new HashSet<>(); + } + + @Override + public void addOrUpdateUsersGroups(Map> sourceGroups, Map> sourceUsers, Map> sourceGroupUsers, boolean computeDeletes) throws Throwable { + for (String userdn : sourceUsers.keySet()) { + String username = userNameTransform(sourceUsers.get(userdn).get("original_name")); + allUsers.add(username); + if (!isValidString(username)) { + invalidUsers.add(username); + } } - - @Override - public void init() throws Throwable { - allGroups = new HashSet<>(); - allUsers = new HashSet<>(); - groupUsers = new HashMap<>(); - invalidGroups = new HashSet<>(); - invalidUsers = new HashSet<>(); - } - - public int getTotalUsers() { - return allUsers.size(); + for (String groupdn : sourceGroups.keySet()) { + String groupname = groupNameTransform(sourceGroups.get(groupdn).get("original_name")); + allGroups.add(groupname); + if (!isValidString(groupname)) { + invalidGroups.add(groupname); + } } + groupUsers = sourceGroupUsers; + } - public int getTotalGroups() { - //System.out.println("Groups = " + allGroups); - return allGroups.size(); - } + public int getTotalUsers() { + return allUsers.size(); + } - public int getTotalGroupUsers() { - int totalGroupUsers = 0; - for (String group : groupUsers.keySet()) { - totalGroupUsers += groupUsers.get(group).size(); - } - return totalGroupUsers; - } + public int getTotalGroups() { + return allGroups.size(); + } - public Set getAllGroups() { - return allGroups; + public int getTotalGroupUsers() { + int totalGroupUsers = 0; + for (String group : groupUsers.keySet()) { + totalGroupUsers += groupUsers.get(group).size(); } - - public Set getAllUsers() { - return allUsers; + return totalGroupUsers; + } + + public Set getAllGroups() { + return allGroups; + } + + public Set getAllUsers() { + return allUsers; + } + + public int getTotalInvalidGroups() { + return invalidGroups.size(); + } + + public int getTotalInvalidUsers() { + return invalidUsers.size(); + } + + public int getGroupsWithNoUsers() { + int groupsWithNoUsers = 0; + for (String group : groupUsers.keySet()) { + if (CollectionUtils.isEmpty(groupUsers.get(group))) { + groupsWithNoUsers++; + } } - - public int getTotalInvalidGroups() { - return invalidGroups.size(); - } - - public int getTotalInvalidUsers() { - return invalidUsers.size(); - } - - public int getGroupsWithNoUsers() { - int groupsWithNoUsers = 0; - for (String group : groupUsers.keySet()) { - if (CollectionUtils.isEmpty(groupUsers.get(group))) { - groupsWithNoUsers++; - } - } - return groupsWithNoUsers; - } - - @Override - public void addOrUpdateUsersGroups(Map> sourceGroups, - Map> sourceUsers, - Map> sourceGroupUsers, - boolean computeDeletes) throws Throwable { - - for (String userdn : sourceUsers.keySet()) { - //System.out.println("Username: " + sourceUsers.get(userdn).get("original_name")); - String username = userNameTransform(sourceUsers.get(userdn).get("original_name")); - allUsers.add(username); - if (!isValidString(username)) { - invalidUsers.add(username); - } - } - for (String groupdn : sourceGroups.keySet()) { - //System.out.println("Groupname: " + sourceGroups.get(groupdn).get("original_name")); - String groupname = groupNameTransform(sourceGroups.get(groupdn).get("original_name")); - allGroups.add(groupname); - if (!isValidString(groupname)) { - invalidGroups.add(groupname); - } - } - groupUsers = sourceGroupUsers; - //System.out.println("Username: " + user + " and associated groups: " + groups); - } - - -} \ No newline at end of file + return groupsWithNoUsers; + } +} diff --git a/ugsync/src/test/java/org/apache/ranger/usergroupsync/TestLdapUserGroup.java b/ugsync/src/test/java/org/apache/ranger/usergroupsync/TestLdapUserGroup.java index 2011b5b75b..894117482f 100644 --- a/ugsync/src/test/java/org/apache/ranger/usergroupsync/TestLdapUserGroup.java +++ b/ugsync/src/test/java/org/apache/ranger/usergroupsync/TestLdapUserGroup.java @@ -19,13 +19,11 @@ package org.apache.ranger.usergroupsync; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - import org.apache.directory.server.annotations.CreateLdapConnectionPool; import org.apache.directory.server.core.annotations.ApplyLdifFiles; import org.apache.directory.server.core.annotations.ContextEntry; import org.apache.directory.server.core.annotations.CreateDS; +import org.apache.directory.server.core.annotations.CreateIndex; import org.apache.directory.server.core.annotations.CreatePartition; import org.apache.directory.server.core.integ.AbstractLdapTestUnit; import org.apache.directory.server.core.integ.FrameworkRunner; @@ -39,440 +37,422 @@ import org.junit.Test; import org.junit.runner.RunWith; -import org.apache.directory.server.core.annotations.CreateIndex; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; @RunWith(FrameworkRunner.class) -@CreateDS(name = "classDS", -partitions = -{ - @CreatePartition( - name = "AD", - suffix = "DC=ranger,DC=qe,DC=hortonworks,DC=com", - contextEntry = @ContextEntry( - entryLdif = - "dn: DC=ranger,DC=qe,DC=hortonworks,DC=com\n" + - "objectClass: domain\n" + - "objectClass: top\n" + - "dc: example\n\n" - ), - indexes = - { - @CreateIndex(attribute = "objectClass"), - @CreateIndex(attribute = "dc"), - @CreateIndex(attribute = "ou") - } - ) -} - ) -@CreateLdapConnectionPool( - maxActive = 1, - maxWait = 5000 ) -@ApplyLdifFiles( { - "ADSchema.ldif" -} - ) -public class TestLdapUserGroup extends AbstractLdapTestUnit{ - private UserGroupSyncConfig config; - private UserGroupSource ldapBuilder; - private PolicyMgrUserGroupBuilderTest sink; +@CreateDS(name = "classDS", partitions = { + @CreatePartition( + name = "AD", suffix = "DC=ranger,DC=qe,DC=hortonworks,DC=com", + contextEntry = @ContextEntry(entryLdif = "dn: DC=ranger,DC=qe,DC=hortonworks,DC=com\nobjectClass: domain\nobjectClass: top\ndc: example\n\n"), + indexes = { + @CreateIndex(attribute = "objectClass"), + @CreateIndex(attribute = "dc"), + @CreateIndex(attribute = "ou") + })}) +@CreateLdapConnectionPool(maxActive = 1, maxWait = 5000) +@ApplyLdifFiles("ADSchema.ldif") + +public class TestLdapUserGroup extends AbstractLdapTestUnit { + private UserGroupSyncConfig config; + private UserGroupSource ldapBuilder; + private PolicyMgrUserGroupBuilderTest sink; + + @Before + public void setup() throws Exception { + LdapServer ldapServer = new LdapServer(); + ldapServer.setSaslHost("127.0.0.1"); + ldapServer.setSearchBaseDn("DC=ranger,DC=qe,DC=hortonworks,DC=com"); + String ldapPort = System.getProperty("ldap.port"); + Assert.assertNotNull("Property 'ldap.port' null", ldapPort); + ldapServer.setTransports(new TcpTransport("127.0.0.1", Integer.parseInt(ldapPort))); + ldapServer.setDirectoryService(getService()); + ldapServer.setMaxSizeLimit(LdapServer.NO_SIZE_LIMIT); + setLdapServer(ldapServer); + getService().startup(); + getLdapServer().start(); + config = UserGroupSyncConfig.getInstance(); + ldapBuilder = new LdapUserGroupBuilder(); + sink = new PolicyMgrUserGroupBuilderTest(); + } - @Before - public void setup() throws Exception { - LdapServer ldapServer = new LdapServer(); - ldapServer.setSaslHost("127.0.0.1"); - ldapServer.setSearchBaseDn("DC=ranger,DC=qe,DC=hortonworks,DC=com"); - String ldapPort = System.getProperty("ldap.port"); - Assert.assertNotNull("Property 'ldap.port' null", ldapPort); - ldapServer.setTransports(new TcpTransport("127.0.0.1", Integer.parseInt(ldapPort))); - ldapServer.setDirectoryService(getService()); - ldapServer.setMaxSizeLimit( LdapServer.NO_SIZE_LIMIT ); - setLdapServer(ldapServer); - getService().startup(); - getLdapServer().start(); - config = UserGroupSyncConfig.getInstance(); - ldapBuilder = new LdapUserGroupBuilder(); - sink = new PolicyMgrUserGroupBuilderTest(); - } + @Test + public void testUserSearchFilterWithWildcards() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase("DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserSearchFilter(""); + config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(true); + config.setPagedResultsEnabled(true); + config.setGroupnames("memberof=cn=Group2*"); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(10, sink.getTotalUsers()); + } - @Test - public void testUserSearchFilterWithWildcards() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase("DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserSearchFilter(""); - config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(true); - config.setPagedResultsEnabled(true); - config.setGroupnames("memberof=cn=Group2*"); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(10, sink.getTotalUsers()); - } + @Test + public void testUserSearchFilterWithShortname() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase("DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserSearchFilter(""); + config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(true); + config.setPagedResultsEnabled(true); + config.setGroupnames("memberof=CN=Group20"); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(10, sink.getTotalUsers()); + } - @Test - public void testUserSearchFilterWithShortname() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase("DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserSearchFilter(""); - config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(true); - config.setPagedResultsEnabled(true); - config.setGroupnames("memberof=CN=Group20"); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(10, sink.getTotalUsers()); - } + @Test + public void testUserSearchFilterWithMultipleShortname() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase("DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserSearchFilter(""); + config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(true); + config.setPagedResultsEnabled(true); + config.setGroupnames("memberof=CN=Group20;CN=Group19"); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(21, sink.getTotalUsers()); + } - @Test - public void testUserSearchFilterWithMultipleShortname() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase("DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserSearchFilter(""); - config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(true); - config.setPagedResultsEnabled(true); - config.setGroupnames("memberof=CN=Group20;CN=Group19"); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(21, sink.getTotalUsers()); - } + @Test + public void testUserSearchFilterWithMultipleWildcards() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase("DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserSearchFilter(""); + config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(true); + config.setPagedResultsEnabled(true); + config.setGroupnames("memberof=CN=Group2*;memberof=CN=Group1*"); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(111, sink.getTotalUsers()); + } - @Test - public void testUserSearchFilterWithMultipleWildcards() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase("DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserSearchFilter(""); - config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(true); - config.setPagedResultsEnabled(true); - config.setGroupnames("memberof=CN=Group2*;memberof=CN=Group1*"); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(111, sink.getTotalUsers()); - } + @Test + public void testUserSearchFilterWithMultipleDNs() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase("DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserSearchFilter(""); + config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(true); + config.setPagedResultsEnabled(true); + config.setGroupnames("memberof=CN=Group14,OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com;memberof=CN=Group20,OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(21, sink.getTotalUsers()); + } - @Test - public void testUserSearchFilterWithMultipleDNs() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase("DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserSearchFilter(""); - config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(true); - config.setPagedResultsEnabled(true); - config.setGroupnames("memberof=CN=Group14,OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com;memberof=CN=Group20,OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(21, sink.getTotalUsers()); - } + @Test + public void testUserSearchFilterWithInvalidDNs() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase("DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserSearchFilter(""); + config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(true); + config.setPagedResultsEnabled(true); + config.setGroupnames("uid=Group14,OU=Groups,DC=ranger;memberuid=Group20,OU=Groups,DC=ranger,DC=qe"); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(111, sink.getTotalUsers()); + } - @Test - public void testUserSearchFilterWithInvalidDNs() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase("DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserSearchFilter(""); - config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(true); - config.setPagedResultsEnabled(true); - config.setGroupnames("uid=Group14,OU=Groups,DC=ranger;memberuid=Group20,OU=Groups,DC=ranger,DC=qe"); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(111, sink.getTotalUsers()); - } + @Test + public void testUpdateSinkTotalUsers() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserSearchFilter(""); + config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(false); + config.setPagedResultsEnabled(true); + config.setGroupSearchFirstEnabled(false); + ldapBuilder.init(); - @Test - public void testUpdateSinkTotalUsers() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserSearchFilter(""); - config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(false); - config.setPagedResultsEnabled(true); - config.setGroupSearchFirstEnabled(false); - //config.setGroupHierarchyLevel(0); - ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(109, sink.getTotalUsers()); + } - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(109, sink.getTotalUsers()); - } + @Test + public void testUpdateSinkWithoutPagedResults() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserSearchFilter(""); + config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(false); + config.setPagedResultsEnabled(false); + config.setGroupSearchFirstEnabled(false); + config.setGroupnames(""); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(109, sink.getTotalUsers()); + } - @Test - public void testUpdateSinkWithoutPagedResults() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserSearchFilter(""); - config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(false); - config.setPagedResultsEnabled(false); - config.setGroupSearchFirstEnabled(false); - config.setGroupnames(""); - //config.setGroupHierarchyLevel(0); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(109, sink.getTotalUsers()); - } + @Test + public void testUpdateSinkUserFilter() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserSearchFilter("(|(memberof=CN=Group10,OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com)(memberof=CN=Group11,OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com))"); + config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(false); + config.setGroupSearchFirstEnabled(false); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(12, sink.getTotalUsers()); + } - @Test - public void testUpdateSinkUserFilter() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserSearchFilter("(|(memberof=CN=Group10,OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com)(memberof=CN=Group11,OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com))"); - config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(false); - config.setGroupSearchFirstEnabled(false); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(12, sink.getTotalUsers()); - } + @Test + public void testUpdateSinkTotalGroups() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserSearchFilter(""); + config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setGroupSearchFilter(""); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(true); + config.setGroupSearchFirstEnabled(false); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(12, sink.getTotalGroups()); + } - @Test - public void testUpdateSinkTotalGroups() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserSearchFilter(""); - config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setGroupSearchFilter(""); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(true); - config.setGroupSearchFirstEnabled(false); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(12, sink.getTotalGroups()); - } + @Test + public void testUpdateSinkGroupFilter() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserSearchFilter(""); + config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setGroupSearchFilter("cn=Group19"); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(true); + config.setGroupSearchFirstEnabled(false); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(1, sink.getTotalGroups()); + } - @Test - public void testUpdateSinkGroupFilter() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserSearchFilter(""); - config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setGroupSearchFilter("cn=Group19"); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(true); - config.setGroupSearchFirstEnabled(false); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(1, sink.getTotalGroups()); - } + @Test + public void testUpdateSinkMultipleOUs() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com;ou=HadoopUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com;ou=BusinessUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserSearchFilter("cn=*"); + config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setGroupSearchFilter("cn=*Group10"); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(true); + config.setGroupSearchFirstEnabled(false); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(111, sink.getTotalUsers()); + assertEquals(1, sink.getTotalGroups()); + } - @Test - public void testUpdateSinkMultipleOUs() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com;ou=HadoopUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com;ou=BusinessUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserSearchFilter("cn=*"); - config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setGroupSearchFilter("cn=*Group10"); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(true); - config.setGroupSearchFirstEnabled(false); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(111, sink.getTotalUsers()); - assertEquals(1, sink.getTotalGroups()); - } + @Test + public void testMultipleOUGroupsWithGroupSearch() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com;ou=HadoopUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com;ou=BusinessUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserSearchFilter("cn=*"); + config.setGroupSearchBase("OU=HdpGroups,OU=HadoopUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com;OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setGroupSearchFilter("cn=*"); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(true); + config.setGroupSearchFirstEnabled(false); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(111, sink.getTotalUsers()); + assertEquals(13, sink.getTotalGroups()); + } - @Test - public void testMultipleOUGroupsWithGroupSearch() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com;ou=HadoopUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com;ou=BusinessUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserSearchFilter("cn=*"); - config.setGroupSearchBase("OU=HdpGroups,OU=HadoopUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com;OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setGroupSearchFilter("cn=*"); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(true); - config.setGroupSearchFirstEnabled(false); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(111, sink.getTotalUsers()); - assertEquals(13, sink.getTotalGroups()); - } + @Test + public void testUpdateSinkMultipleOUGroups() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com;ou=HadoopUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com;ou=BusinessUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserSearchFilter("cn=*"); + config.setGroupSearchBase("OU=HdpGroups,OU=HadoopUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com;OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setGroupSearchFilter("cn=*Group10"); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(true); + config.setGroupSearchFirstEnabled(false); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(111, sink.getTotalUsers()); + assertEquals(2, sink.getTotalGroups()); + } - @Test - public void testUpdateSinkMultipleOUGroups() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com;ou=HadoopUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com;ou=BusinessUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserSearchFilter("cn=*"); - config.setGroupSearchBase("OU=HdpGroups,OU=HadoopUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com;OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setGroupSearchFilter("cn=*Group10"); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(true); - config.setGroupSearchFirstEnabled(false); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(111, sink.getTotalUsers()); - assertEquals(2, sink.getTotalGroups()); - } + @Test + public void testUpdateSinkWithEmptyUserSearchBase() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase(""); + config.setUserSearchFilter(""); + config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(false); + config.setPagedResultsEnabled(true); + config.setGroupSearchFirstEnabled(false); + config.setGroupnames(""); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(111, sink.getTotalUsers()); + } - @Test - public void testUpdateSinkWithEmptyUserSearchBase() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase(""); - config.setUserSearchFilter(""); - config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(false); - config.setPagedResultsEnabled(true); - config.setGroupSearchFirstEnabled(false); - config.setGroupnames(""); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(111, sink.getTotalUsers()); - } + @Test + public void testUpdateSinkShortUserName() throws Throwable { + config.setUserNameAttribute("cn"); + config.setUserSearchBase("ou=people,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserSearchFilter("uid=*"); + config.setUserObjectClass("posixAccount"); + config.setGroupSearchBase("OU=pGroups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setGroupSearchFilter("cn=*"); + config.setGroupSearchEnabled(true); + config.setGroupSearchFirstEnabled(false); + config.setUserGroupMemberAttributeName("memberuid"); + config.setGroupObjectClass("posixGroup"); + config.setUserSearchEnabled(false); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(1, sink.getTotalUsers()); + assertEquals(3, sink.getTotalGroups()); + assertEquals(3, sink.getTotalGroupUsers()); + } - @Test - public void testUpdateSinkShortUserName() throws Throwable { - config.setUserNameAttribute("cn"); - config.setUserSearchBase("ou=people,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserSearchFilter("uid=*"); - config.setUserObjectClass("posixAccount"); - config.setGroupSearchBase("OU=pGroups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setGroupSearchFilter("cn=*"); - config.setGroupSearchEnabled(true); - config.setGroupSearchFirstEnabled(false); - config.setUserGroupMemberAttributeName("memberuid"); - config.setGroupObjectClass("posixGroup"); - config.setUserSearchEnabled(false); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(1, sink.getTotalUsers()); - assertEquals(3, sink.getTotalGroups()); - assertEquals(3, sink.getTotalGroupUsers()); - } + @Test + public void testUpdateSinkWithUserGroupMapping() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserSearchFilter(""); + config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setGroupSearchFilter(""); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(true); + config.setGroupSearchFirstEnabled(false); - @Test - public void testUpdateSinkWithUserGroupMapping() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserSearchFilter(""); - config.setGroupSearchBase("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setGroupSearchFilter(""); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(true); - config.setGroupSearchFirstEnabled(false); + config.setProperty(UserGroupSyncConfig.SYNC_MAPPING_USERNAME, "s/[=]/_/g"); + config.setProperty(UserGroupSyncConfig.SYNC_MAPPING_GROUPNAME, "s/[=]/_/g"); + sink = new PolicyMgrUserGroupBuilderTest(); - config.setProperty(UserGroupSyncConfig.SYNC_MAPPING_USERNAME, "s/[=]/_/g"); - config.setProperty(UserGroupSyncConfig.SYNC_MAPPING_GROUPNAME, "s/[=]/_/g"); - sink = new PolicyMgrUserGroupBuilderTest(); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(109, sink.getTotalUsers()); + assertEquals(12, sink.getTotalGroups()); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(109, sink.getTotalUsers()); - assertEquals(12, sink.getTotalGroups()); + // no user should have an = character because of the mapping + for (String user : sink.getAllUsers()) { + assertFalse(user.contains("=")); + } - // no user should have an = character because of the mapping - for (String user : sink.getAllUsers()) { - assertFalse(user.contains("=")); - } + // no group should have an = character because of the mapping + for (String group : sink.getAllGroups()) { + assertFalse(group.contains("=")); + } + } - // no group should have an = character because of the mapping - for (String group : sink.getAllGroups()) { - assertFalse(group.contains("=")); - } - } - - @Test - public void testMultipleOUInvalidOU() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com;ou=HadoopUsers1,DC=ranger,DC=qe,DC=hortonworks,DC=com;ou=BusinessUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setUserSearchFilter("cn=*"); - config.setGroupSearchBase("OU=HdpGroups,OU=HadoopUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com;OU=Groups1,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setGroupSearchFilter("cn=*"); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(true); - config.setGroupSearchFirstEnabled(false); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(110, sink.getTotalUsers()); - assertEquals(1, sink.getTotalGroups()); - } + @Test + public void testMultipleOUInvalidOU() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase("cn=users,DC=ranger,DC=qe,DC=hortonworks,DC=com;ou=HadoopUsers1,DC=ranger,DC=qe,DC=hortonworks,DC=com;ou=BusinessUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setUserSearchFilter("cn=*"); + config.setGroupSearchBase("OU=HdpGroups,OU=HadoopUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com;OU=Groups1,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setGroupSearchFilter("cn=*"); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(true); + config.setGroupSearchFirstEnabled(false); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(110, sink.getTotalUsers()); + assertEquals(1, sink.getTotalGroups()); + } - @Test - public void testGroupsWithNoUsers() throws Throwable { - config.setUserNameAttribute("sAMAccountName"); - config.setUserSearchBase("DC=ranger,DC=qe,DC=hortonworks,DC=com;"); - config.setUserSearchFilter("cn=*"); - config.setGroupSearchBase("OU=HdpGroups,OU=HadoopUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com;OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); - config.setGroupSearchFilter("cn=Group2*"); - config.setUserGroupMemberAttributeName("member"); - config.setUserObjectClass("organizationalPerson"); - config.setGroupObjectClass("groupOfNames"); - config.setGroupSearchEnabled(true); - config.setGroupSearchFirstEnabled(true); - config.setUserSearchEnabled(true); - config.setDeltaSync(true); - ldapBuilder = config.getUserGroupSource(); - ldapBuilder.init(); - sink.init(); - ldapBuilder.updateSink(sink); - assertEquals(2, sink.getGroupsWithNoUsers()); - } + @Test + public void testGroupsWithNoUsers() throws Throwable { + config.setUserNameAttribute("sAMAccountName"); + config.setUserSearchBase("DC=ranger,DC=qe,DC=hortonworks,DC=com;"); + config.setUserSearchFilter("cn=*"); + config.setGroupSearchBase("OU=HdpGroups,OU=HadoopUsers,DC=ranger,DC=qe,DC=hortonworks,DC=com;OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"); + config.setGroupSearchFilter("cn=Group2*"); + config.setUserGroupMemberAttributeName("member"); + config.setUserObjectClass("organizationalPerson"); + config.setGroupObjectClass("groupOfNames"); + config.setGroupSearchEnabled(true); + config.setGroupSearchFirstEnabled(true); + config.setUserSearchEnabled(true); + config.setDeltaSync(true); + ldapBuilder = config.getUserGroupSource(); + ldapBuilder.init(); + sink.init(); + ldapBuilder.updateSink(sink); + assertEquals(2, sink.getGroupsWithNoUsers()); + } - @After - public void shutdown() throws Exception { - if (getService().isStarted()) { - getService().shutdown(); - } - if (getLdapServer().isStarted()) { - getLdapServer().stop(); - } - } + @After + public void shutdown() throws Exception { + if (getService().isStarted()) { + getService().shutdown(); + } + if (getLdapServer().isStarted()) { + getLdapServer().stop(); + } + } } diff --git a/ugsync/src/test/java/org/apache/ranger/usergroupsync/TestRegEx.java b/ugsync/src/test/java/org/apache/ranger/usergroupsync/TestRegEx.java index 1be5fc4abc..560615a46d 100644 --- a/ugsync/src/test/java/org/apache/ranger/usergroupsync/TestRegEx.java +++ b/ugsync/src/test/java/org/apache/ranger/usergroupsync/TestRegEx.java @@ -19,35 +19,33 @@ package org.apache.ranger.usergroupsync; -import static org.junit.Assert.*; +import org.junit.Before; +import org.junit.Test; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assert.assertEquals; public class TestRegEx { - protected String userNameBaseProperty = "ranger.usersync.mapping.username.regex"; + protected String userNameBaseProperty = "ranger.usersync.mapping.username.regex"; protected String groupNameBaseProperty = "ranger.usersync.mapping.groupname.regex"; - protected String mappingSeparator = "/"; - protected RegEx userNameRegEx = null; - protected RegEx groupNameRegEx = null; - List userRegexPatterns = null; - List groupRegexPatterns = null; + protected String mappingSeparator = "/"; + protected RegEx userNameRegEx; + protected RegEx groupNameRegEx; + List userRegexPatterns; + List groupRegexPatterns; - @Before - public void setUp() throws Exception { - userNameRegEx = new RegEx(); - //userNameRegEx.init(userNameBaseProperty); - userRegexPatterns = new ArrayList(); - groupNameRegEx = new RegEx(); - //groupNameRegEx.init(groupNameBaseProperty); + @Before + public void setUp() throws Exception { + userNameRegEx = new RegEx(); + groupNameRegEx = new RegEx(); + userRegexPatterns = new ArrayList(); groupRegexPatterns = new ArrayList(); - } + } - @Test + @Test public void testUserNameTransform() throws Throwable { userRegexPatterns.add("s/\\s/_/"); userNameRegEx.populateReplacementPatterns(userNameBaseProperty, userRegexPatterns, mappingSeparator); @@ -123,5 +121,4 @@ public void testUsernameSuffix() throws Throwable { assertEquals("mew_ty", userNameRegEx.transform("mew")); assertEquals("onix_ty", userNameRegEx.transform("onix")); } - } diff --git a/unixauthclient/pom.xml b/unixauthclient/pom.xml index c59321b2bb..e037a87501 100644 --- a/unixauthclient/pom.xml +++ b/unixauthclient/pom.xml @@ -27,6 +27,9 @@ jar Unix Authentication Client Unix authentication client + + true + com.google.code.gson diff --git a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/ConsolePromptCallbackHandler.java b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/ConsolePromptCallbackHandler.java index ad1b6c649c..2fcbbdc1d6 100644 --- a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/ConsolePromptCallbackHandler.java +++ b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/ConsolePromptCallbackHandler.java @@ -17,11 +17,7 @@ * under the License. */ - package org.apache.ranger.authentication.unix.jaas; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; +package org.apache.ranger.authentication.unix.jaas; import javax.security.auth.callback.Callback; import javax.security.auth.callback.CallbackHandler; @@ -29,41 +25,39 @@ import javax.security.auth.callback.PasswordCallback; import javax.security.auth.callback.UnsupportedCallbackException; -public class ConsolePromptCallbackHandler implements CallbackHandler { - - - @Override - public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException { - - BufferedReader reader = new BufferedReader(new InputStreamReader(System.in)); - - for(Callback cb : callbacks) { - if (cb instanceof NameCallback) { - NameCallback nc = (NameCallback)cb; - System.out.print(nc.getPrompt()); - System.out.flush(); - String line = null; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; - while (line == null) { +public class ConsolePromptCallbackHandler implements CallbackHandler { + @Override + public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException { + BufferedReader reader = new BufferedReader(new InputStreamReader(System.in)); + + for (Callback cb : callbacks) { + if (cb instanceof NameCallback) { + NameCallback nc = (NameCallback) cb; + System.out.print(nc.getPrompt()); + System.out.flush(); + String line = null; + + while (line == null) { line = reader.readLine(); - } - nc.setName(line); - } - else if (cb instanceof PasswordCallback) { - PasswordCallback pc = (PasswordCallback)cb; - System.out.print(pc.getPrompt()); - System.out.flush(); - - String line = null; - while (line == null) { + } + nc.setName(line); + } else if (cb instanceof PasswordCallback) { + PasswordCallback pc = (PasswordCallback) cb; + System.out.print(pc.getPrompt()); + System.out.flush(); + + String line = null; + while (line == null) { line = reader.readLine(); - } - pc.setPassword(line.toCharArray()); - } - else { - System.out.println("Unknown callback [" + cb.getClass().getName() + "]"); - } - } - } - + } + pc.setPassword(line.toCharArray()); + } else { + System.out.println("Unknown callback [" + cb.getClass().getName() + "]"); + } + } + } } diff --git a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamLoginModule.java b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamLoginModule.java index 8ff5b2399f..7b33823787 100644 --- a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamLoginModule.java +++ b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamLoginModule.java @@ -24,10 +24,15 @@ import org.jvnet.libpam.UnixUser; import javax.security.auth.Subject; -import javax.security.auth.callback.*; +import javax.security.auth.callback.Callback; +import javax.security.auth.callback.CallbackHandler; +import javax.security.auth.callback.NameCallback; +import javax.security.auth.callback.PasswordCallback; +import javax.security.auth.callback.UnsupportedCallbackException; import javax.security.auth.login.FailedLoginException; import javax.security.auth.login.LoginException; import javax.security.auth.spi.LoginModule; + import java.io.IOException; import java.security.Principal; import java.util.Arrays; @@ -35,193 +40,160 @@ import java.util.Map; import java.util.Set; -public class PamLoginModule implements LoginModule -{ +public class PamLoginModule implements LoginModule { public static final String SERVICE_KEY = "ranger.pam.service"; + private PAM pam; + private Subject subject; + private CallbackHandler callbackHandler; + private Map options; - private PAM _pam; - private Subject _subject; - private CallbackHandler _callbackHandler; - private Map _options; - - private String _username; - private char[] _passwordchar; + private String username; + private char[] passwordChar; - private boolean _authSucceeded; - private PamPrincipal _principal; + private boolean authSucceeded; + private PamPrincipal principal; - public PamLoginModule() - { + public PamLoginModule() { super(); - _authSucceeded = false; + authSucceeded = false; } @Override - public void initialize(Subject subject, CallbackHandler callbackHandler, Map sharedState, Map options) - { - _subject = subject; - _callbackHandler = callbackHandler; - _options = new HashMap<>(options); + public void initialize(Subject subject, CallbackHandler callbackHandler, Map sharedState, Map options) { + subject = subject; + callbackHandler = callbackHandler; + options = new HashMap<>(options); } @Override - public boolean login() throws LoginException - { + public boolean login() throws LoginException { initializePam(); obtainUserAndPassword(); return performLogin(); } - private void initializePam() throws LoginException - { - String service = (String)_options.get(SERVICE_KEY); - if (service == null) - { + @Override + public boolean commit() throws LoginException { + if (!authSucceeded) { + return false; + } + + if (subject.isReadOnly()) { + cleanup(); + throw new LoginException("Subject is read-only"); + } + + Set principals = subject.getPrincipals(); + if (!principals.contains(principal)) { + principals.add(principal); + } + + return true; + } + + @Override + public boolean abort() throws LoginException { + if (!authSucceeded) { + return false; + } + + cleanup(); + return true; + } + + @Override + public boolean logout() throws LoginException { + if (subject.isReadOnly()) { + cleanup(); + throw new LoginException("Subject is read-only"); + } + + subject.getPrincipals().remove(principal); + + cleanup(); + return true; + } + + private void initializePam() throws LoginException { + String service = (String) options.get(SERVICE_KEY); + if (service == null) { throw new LoginException("Error: PAM service was not defined"); } createPam(service); } - private void createPam(String service) throws LoginException - { - try - { - _pam = new PAM(service); - } - catch (PAMException ex) - { + private void createPam(String service) throws LoginException { + try { + pam = new PAM(service); + } catch (PAMException ex) { LoginException le = new LoginException("Error initializing PAM"); le.initCause(ex); throw le; } } - private void obtainUserAndPassword() throws LoginException - { - if (_callbackHandler == null) - { + private void obtainUserAndPassword() throws LoginException { + if (callbackHandler == null) { throw new LoginException("Error: no CallbackHandler available to gather authentication information from the user"); } - try - { - NameCallback nameCallback = new NameCallback("username"); + try { + NameCallback nameCallback = new NameCallback("username"); PasswordCallback passwordCallback = new PasswordCallback("password", false); invokeCallbackHandler(nameCallback, passwordCallback); initUserName(nameCallback); initPassword(passwordCallback); - } - catch (IOException | UnsupportedCallbackException ex) - { + } catch (IOException | UnsupportedCallbackException ex) { LoginException le = new LoginException("Error in callbacks"); le.initCause(ex); throw le; } } - private void invokeCallbackHandler(NameCallback nameCallback, PasswordCallback passwordCallback) throws IOException, UnsupportedCallbackException - { + private void invokeCallbackHandler(NameCallback nameCallback, PasswordCallback passwordCallback) throws IOException, UnsupportedCallbackException { Callback[] callbacks = new Callback[2]; callbacks[0] = nameCallback; callbacks[1] = passwordCallback; - _callbackHandler.handle(callbacks); + callbackHandler.handle(callbacks); } - private void initUserName(NameCallback nameCallback) - { - _username = nameCallback.getName(); + private void initUserName(NameCallback nameCallback) { + username = nameCallback.getName(); } - private void initPassword(PasswordCallback passwordCallback) - { + private void initPassword(PasswordCallback passwordCallback) { char[] password = passwordCallback.getPassword(); if (password != null) { - _passwordchar = Arrays.copyOf(password, password.length); + passwordChar = Arrays.copyOf(password, password.length); } passwordCallback.clearPassword(); } - private boolean performLogin() throws LoginException - { - try - { - if (_passwordchar != null) { - UnixUser user = _pam.authenticate(_username, String.valueOf(_passwordchar)); - _principal = new PamPrincipal(user); - _authSucceeded = true; - return true; - } else { - throw new PAMException("Password is Null or Empty!!!"); - } - } - catch (PAMException ex) - { + private boolean performLogin() throws LoginException { + try { + if (passwordChar != null) { + UnixUser user = pam.authenticate(username, String.valueOf(passwordChar)); + principal = new PamPrincipal(user); + authSucceeded = true; + return true; + } else { + throw new PAMException("Password is Null or Empty!!!"); + } + } catch (PAMException ex) { LoginException le = new FailedLoginException("Invalid username or password"); le.initCause(ex); throw le; } } - @Override - public boolean commit() throws LoginException - { - if (_authSucceeded == false) - { - return false; - } - - if (_subject.isReadOnly()) - { - cleanup(); - throw new LoginException("Subject is read-only"); - } - - Set principals = _subject.getPrincipals(); - if (principals.contains(_principal) == false) - { - principals.add(_principal); - } - - return true; - } - - @Override - public boolean abort() throws LoginException - { - if (_authSucceeded == false) - { - return false; - } - - cleanup(); - return true; - } - - @Override - public boolean logout() throws LoginException - { - if (_subject.isReadOnly()) - { - cleanup(); - throw new LoginException("Subject is read-only"); - } - - _subject.getPrincipals().remove(_principal); - - cleanup(); - return true; - } - - private void cleanup() - { - _authSucceeded = false; - _username = null; - Arrays.fill(_passwordchar, ' '); - _principal = null; - _pam.dispose(); + private void cleanup() { + authSucceeded = false; + username = null; + Arrays.fill(passwordChar, ' '); + principal = null; + pam.dispose(); } } - diff --git a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamPrincipal.java b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamPrincipal.java index 235e22b937..969323b035 100644 --- a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamPrincipal.java +++ b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamPrincipal.java @@ -25,62 +25,52 @@ import java.util.Collections; import java.util.Set; - public class PamPrincipal implements Principal { - private String _userName; - private String _gecos; - private String _homeDir; - private String _shell; - private int _uid; - private int _gid; - private Set _groups; + private String userName; + private String gecos; + private String homeDir; + private String shell; + private int uid; + private int gid; + private Set groups; - public PamPrincipal(UnixUser user) - { + public PamPrincipal(UnixUser user) { super(); - _userName = user.getUserName(); - _gecos = user.getGecos(); - _homeDir = user.getDir(); - _shell = user.getShell(); - _uid = user.getUID(); - _gid = user.getGID(); - _groups = Collections.unmodifiableSet(user.getGroups()); + userName = user.getUserName(); + gecos = user.getGecos(); + homeDir = user.getDir(); + shell = user.getShell(); + uid = user.getUID(); + gid = user.getGID(); + groups = Collections.unmodifiableSet(user.getGroups()); } @Override - public String getName() - { - return _userName; + public String getName() { + return userName; } - public String getGecos() - { - return _gecos; + public String getGecos() { + return gecos; } - public String getHomeDir() - { - return _homeDir; + public String getHomeDir() { + return homeDir; } - public String getShell() - { - return _shell; + public String getShell() { + return shell; } - public int getUid() - { - return _uid; + public int getUid() { + return uid; } - public int getGid() - { - return _gid; + public int getGid() { + return gid; } - public Set getGroups() - { - return _groups; + public Set getGroups() { + return groups; } - } diff --git a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/RemoteUnixLoginModule.java b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/RemoteUnixLoginModule.java index cce8a31a69..57bb32412b 100644 --- a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/RemoteUnixLoginModule.java +++ b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/RemoteUnixLoginModule.java @@ -19,6 +19,22 @@ package org.apache.ranger.authentication.unix.jaas; +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; +import javax.security.auth.Subject; +import javax.security.auth.callback.Callback; +import javax.security.auth.callback.CallbackHandler; +import javax.security.auth.callback.NameCallback; +import javax.security.auth.callback.PasswordCallback; +import javax.security.auth.callback.UnsupportedCallbackException; +import javax.security.auth.login.LoginException; +import javax.security.auth.spi.LoginModule; + import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -36,419 +52,371 @@ import java.util.Map; import java.util.Properties; -import javax.net.ssl.KeyManager; -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSocketFactory; -import javax.net.ssl.TrustManager; -import javax.net.ssl.TrustManagerFactory; -import javax.net.ssl.X509TrustManager; -import javax.security.auth.Subject; -import javax.security.auth.callback.Callback; -import javax.security.auth.callback.CallbackHandler; -import javax.security.auth.callback.NameCallback; -import javax.security.auth.callback.PasswordCallback; -import javax.security.auth.callback.UnsupportedCallbackException; -import javax.security.auth.login.LoginException; -import javax.security.auth.spi.LoginModule; - public class RemoteUnixLoginModule implements LoginModule { - - private static final String DEBUG_PARAM = "ranger.unixauth.debug"; - private static final String REMOTE_LOGIN_HOST_PARAM = "ranger.unixauth.service.hostname"; - private static final String REMOTE_LOGIN_AUTH_SERVICE_PORT_PARAM = "ranger.unixauth.service.port"; - private static final String SSL_KEYSTORE_PATH_PARAM = "ranger.unixauth.keystore"; - private static final String SSL_KEYSTORE_PATH_PASSWORD_PARAM = "ranger.unixauth.keystore.password"; - private static final String SSL_TRUSTSTORE_PATH_PARAM = "ranger.unixauth.truststore"; - private static final String SSL_TRUSTSTORE_PATH_PASSWORD_PARAM = "ranger.unixauth.truststore.password"; - private static final String SSL_ENABLED_PARAM = "ranger.unixauth.ssl.enabled"; - private static final String SERVER_CERT_VALIDATION_PARAM = "ranger.unixauth.server.cert.validation"; - - private static final String JAAS_ENABLED_PARAM = "ranger.unixauth.remote.login.enabled"; - - private static final String SSL_ALGORITHM = "TLSv1.2"; - - private String userName; - private char[] password; - private Subject subject; - private CallbackHandler callbackHandler; - private boolean debug = true; - - private String remoteHostName; - private int remoteHostAuthServicePort; - - private boolean loginSuccessful = false; - private String loginGroups = null; - - private String keyStorePath; - private String keyStorePathPassword; - private String trustStorePath; - private String trustStorePathPassword; - - private boolean SSLEnabled = false; - - private boolean serverCertValidation = true; - - private boolean remoteLoginEnabled = true; - - public RemoteUnixLoginModule() { - log("Created RemoteUnixLoginModule"); - } - - @Override - public boolean abort() throws LoginException { - log("RemoteUnixLoginModule::abort() has been called."); - loginSuccessful = false; - return true; - } - - @Override - public boolean commit() throws LoginException { - log("RemoteUnixLoginModule::commit() has been called. -> isLoginSuccess [" + loginSuccessful + "]"); - if (loginSuccessful) { - if (subject != null) { - subject.getPrincipals().add(new UnixUserPrincipal(userName.trim())); - if (loginGroups != null) { - loginGroups = loginGroups.trim(); - for (String group : loginGroups.split(",")) { - subject.getPrincipals().add(new UnixGroupPrincipal(group.trim())); - } - } - } - } else { - if (subject != null) { - subject.getPrincipals().clear(); - } - } - return loginSuccessful; - } - - @Override - public void initialize(Subject subject, CallbackHandler callbackHandler, Map sharedState, Map options) { - this.subject = subject; - this.callbackHandler = callbackHandler; - - log("RemoteUnixLoginModule::initialize() has been called with callbackhandler: " + this.callbackHandler); - - if (this.callbackHandler == null) { - this.callbackHandler = new ConsolePromptCallbackHandler(); - } - - Properties config = new Properties(); - config.putAll(options); - initParams(config); - - } - - public void initParams(Properties options) { - - String val = (String) options.get(JAAS_ENABLED_PARAM); - - if (val != null) { - remoteLoginEnabled = val.trim().equalsIgnoreCase("true"); - if (! remoteLoginEnabled) { - log("Skipping RemoteLogin - [" + JAAS_ENABLED_PARAM + "] => [" + val + "]"); - return; - } - } - else { - remoteLoginEnabled = true; - } - - val = (String) options.get(DEBUG_PARAM); - if (val != null && (!val.equalsIgnoreCase("false"))) { - debug = true; - } - else { - debug = false; - } - - remoteHostName = (String) options.get(REMOTE_LOGIN_HOST_PARAM); - log("RemoteHostName:" + remoteHostName); - - val = (String) options.get(REMOTE_LOGIN_AUTH_SERVICE_PORT_PARAM); - if (val != null) { - remoteHostAuthServicePort = Integer.parseInt(val.trim()); - } - log("remoteHostAuthServicePort:" + remoteHostAuthServicePort); - - - val = (String)options.get(SSL_ENABLED_PARAM); - SSLEnabled = (val != null) && val.trim().equalsIgnoreCase("true"); - log("SSLEnabled:" + SSLEnabled); - - if (SSLEnabled) { - trustStorePath = (String) options.get(SSL_TRUSTSTORE_PATH_PARAM); - log("trustStorePath:" + trustStorePath); - - if (trustStorePath != null) { - trustStorePathPassword = (String) options.get(SSL_TRUSTSTORE_PATH_PASSWORD_PARAM); - if (trustStorePathPassword == null) { - trustStorePathPassword = ""; - } - log("trustStorePathPassword:*****"); - } - - keyStorePath = (String) options.get(SSL_KEYSTORE_PATH_PARAM); - log("keyStorePath:" + keyStorePath); - if (keyStorePath != null) { - keyStorePathPassword = (String) options.get(SSL_KEYSTORE_PATH_PASSWORD_PARAM); - if (keyStorePathPassword == null) { - keyStorePathPassword = ""; - } - log("keyStorePathPassword:*****"); - } - - String certValidationFlag = (String) options.get(SERVER_CERT_VALIDATION_PARAM); - serverCertValidation = (! (certValidationFlag != null && ("false".equalsIgnoreCase(certValidationFlag.trim())))); - log("Server Cert Validation : " + serverCertValidation); - } - - } - - @Override - public boolean login() throws LoginException { - - if (remoteLoginEnabled && this.callbackHandler != null) { - NameCallback nameCallback = new NameCallback("UserName:"); - PasswordCallback passwordCallback = new PasswordCallback("Password:", false); - try { - callbackHandler.handle(new Callback[] { nameCallback, passwordCallback }); - } catch (IOException e) { - throw new LoginException("Unable to get username/password due to exception: " + e); - } catch (UnsupportedCallbackException e) { - throw new LoginException("Unable to get username/password due to exception: " + e); - } - - userName = nameCallback.getName(); - - String modifiedUserName = userName; - - if (userName != null) { - int atStartsAt = userName.indexOf("@"); - if ( atStartsAt > -1) { - modifiedUserName = userName.substring(0, atStartsAt); - } - } - - password = passwordCallback.getPassword(); - - log("userName:" + userName); - log("modified UserName:" + modifiedUserName); - - char modifiedPasschar[]; - if (password != null) { - modifiedPasschar = Arrays.copyOf(password,password.length); - } else { - modifiedPasschar = new char[0]; - } - - doLogin(modifiedUserName, modifiedPasschar); - - Arrays.fill(password, ' '); - Arrays.fill(modifiedPasschar, ' '); - loginSuccessful = true; - } - - return loginSuccessful; - } - - @Override - public boolean logout() throws LoginException { - if (subject != null) { - subject.getPrincipals().clear(); - } - return true; - } - - public void doLogin(String aUserName, char[] modifiedPasschar) throws LoginException { - - // POSSIBLE values - // null - // OK: group1, group2, group3 - // FAILED: Invalid Password - - String ret = getLoginReplyFromAuthService(aUserName, modifiedPasschar); - - if (ret == null) { - throw new LoginException("FAILED: unable to authenticate to AuthenticationService: " + remoteHostName + ":" + remoteHostAuthServicePort); - } else if (ret.startsWith("OK:")) { - loginSuccessful = true; - if (ret.length() > 3) { - this.loginGroups = ret.substring(3); - } - } else if (ret.startsWith("FAILED:")) { - loginSuccessful = false; - throw new LoginException("FAILED: unable to authenticate to AuthenticationService: " + remoteHostName + ":" + remoteHostAuthServicePort); - } else { - throw new LoginException("FAILED: unable to authenticate to AuthenticationService: " + remoteHostName + ":" + remoteHostAuthServicePort + ", msg:" + ret); - } - } - - private String getLoginReplyFromAuthService(String aUserName, char[] modifiedPasschar) throws LoginException { - String ret = null; - - Socket sslsocket = null; - - char prefix[]=new String("LOGIN:"+aUserName+" ").toCharArray(); - char tail[]=new String("\n").toCharArray(); - char loginData[]=new char[prefix.length+modifiedPasschar.length+tail.length]; - System.arraycopy(prefix, 0, loginData, 0, prefix.length); - System.arraycopy(modifiedPasschar, 0, loginData, prefix.length,modifiedPasschar.length); - System.arraycopy(tail, 0, loginData, prefix.length+modifiedPasschar.length, tail.length); - try { - try { - if (SSLEnabled) { - - SSLContext context = SSLContext.getInstance(SSL_ALGORITHM); - - KeyManager[] km = null; - - if (keyStorePath != null) { - KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType()); - - InputStream in = null; - - in = getFileInputStream(keyStorePath); - - try { - ks.load(in, keyStorePathPassword.toCharArray()); - } finally { - if (in != null) { - in.close(); - } - } - - KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - kmf.init(ks, keyStorePathPassword.toCharArray()); - km = kmf.getKeyManagers(); - } - - TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - - TrustManager[] tm = null; - - if (serverCertValidation) { - - KeyStore trustStoreKeyStore = null; - - if (trustStorePath != null) { - trustStoreKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); - - InputStream in = null; - - in = getFileInputStream(trustStorePath); - - try { - trustStoreKeyStore.load(in, trustStorePathPassword.toCharArray()); - - trustManagerFactory.init(trustStoreKeyStore); - - tm = trustManagerFactory.getTrustManagers(); - - } finally { - if (in != null) { - in.close(); - } - } - } - } - else { - TrustManager ignoreValidationTM = new X509TrustManager() { - public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { - // Ignore Server Certificate Validation - } - - public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[0]; - } - - public void checkServerTrusted(X509Certificate[] chain, - String authType) - throws CertificateException { - // Ignore Server Certificate Validation - } - }; - - tm = new TrustManager[] {ignoreValidationTM}; - } - - SecureRandom random = new SecureRandom(); - - context.init(km, tm, random); - - SSLSocketFactory sf = context.getSocketFactory(); - - sslsocket = sf.createSocket(remoteHostName, remoteHostAuthServicePort); - - } else { - sslsocket = new Socket(remoteHostName, remoteHostAuthServicePort); - } - - OutputStreamWriter writer = new OutputStreamWriter(sslsocket.getOutputStream()); - - writer.write(loginData); - - writer.flush(); - - BufferedReader reader = new BufferedReader(new InputStreamReader(sslsocket.getInputStream())); - - ret = reader.readLine(); - - reader.close(); - - writer.close(); - - } finally { - if (sslsocket != null) { - sslsocket.close(); - } - } - } catch (Throwable t) { - throw new LoginException("FAILED: unable to authenticate to AuthenticationService: " + remoteHostName + ":" + remoteHostAuthServicePort + ", Exception: [" + t + "]"); - } finally { - log("Login of user String: {" + aUserName + "}, return from AuthServer: {" + ret + "}"); - Arrays.fill(loginData,' '); - Arrays.fill(modifiedPasschar,' '); - } - - return ret; - } - - private InputStream getFileInputStream(String path) throws FileNotFoundException { - - InputStream ret = null; - - File f = new File(path); - - if (f.exists()) { - ret = new FileInputStream(f); - } else { - ret = getClass().getResourceAsStream(path); - - if (ret == null) { - if (! path.startsWith("/")) { - ret = getClass().getResourceAsStream("/" + path); - } - } - - if (ret == null) { - ret = ClassLoader.getSystemClassLoader().getResourceAsStream(path); - if (ret == null) { - if (! path.startsWith("/")) { - ret = ClassLoader.getSystemResourceAsStream("/" + path); - } - } - } - } - - return ret; - } - - private void log(String msg) { - if (debug) { - System.err.println("RemoteUnixLoginModule: " + msg); - } - } - + private static final String DEBUG_PARAM = "ranger.unixauth.debug"; + private static final String REMOTE_LOGIN_HOST_PARAM = "ranger.unixauth.service.hostname"; + private static final String REMOTE_LOGIN_AUTH_SERVICE_PORT_PARAM = "ranger.unixauth.service.port"; + private static final String SSL_KEYSTORE_PATH_PARAM = "ranger.unixauth.keystore"; + private static final String SSL_KEYSTORE_PATH_PASSWORD_PARAM = "ranger.unixauth.keystore.password"; + private static final String SSL_TRUSTSTORE_PATH_PARAM = "ranger.unixauth.truststore"; + private static final String SSL_TRUSTSTORE_PATH_PASSWORD_PARAM = "ranger.unixauth.truststore.password"; + private static final String SSL_ENABLED_PARAM = "ranger.unixauth.ssl.enabled"; + private static final String SERVER_CERT_VALIDATION_PARAM = "ranger.unixauth.server.cert.validation"; + private static final String JAAS_ENABLED_PARAM = "ranger.unixauth.remote.login.enabled"; + private static final String SSL_ALGORITHM = "TLSv1.2"; + private String userName; + private String remoteHostName; + private String loginGroups; + private String keyStorePath; + private String keyStorePathPassword; + private String trustStorePath; + private String trustStorePathPassword; + private char[] password; + private int remoteHostAuthServicePort; + private Subject subject; + private CallbackHandler callbackHandler; + private boolean ssEnabled; + private boolean loginSuccessful; + private boolean debug = true; + private boolean serverCertValidation = true; + private boolean remoteLoginEnabled = true; + + public RemoteUnixLoginModule() { + log("Created RemoteUnixLoginModule"); + } + + @Override + public void initialize(Subject subject, CallbackHandler callbackHandler, Map sharedState, Map options) { + this.subject = subject; + this.callbackHandler = callbackHandler; + + log("RemoteUnixLoginModule::initialize() has been called with callbackhandler: " + this.callbackHandler); + + if (this.callbackHandler == null) { + this.callbackHandler = new ConsolePromptCallbackHandler(); + } + + Properties config = new Properties(); + config.putAll(options); + initParams(config); + } + + @Override + public boolean login() throws LoginException { + if (remoteLoginEnabled && this.callbackHandler != null) { + NameCallback nameCallback = new NameCallback("UserName:"); + PasswordCallback passwordCallback = new PasswordCallback("Password:", false); + try { + callbackHandler.handle(new Callback[] {nameCallback, passwordCallback}); + } catch (IOException | UnsupportedCallbackException e) { + throw new LoginException("Unable to get username/password due to exception: " + e); + } + + userName = nameCallback.getName(); + String modifiedUserName = userName; + + if (userName != null) { + int atStartsAt = userName.indexOf("@"); + if (atStartsAt > -1) { + modifiedUserName = userName.substring(0, atStartsAt); + } + } + + password = passwordCallback.getPassword(); + + log("userName:" + userName); + log("modified UserName:" + modifiedUserName); + + char[] modifiedPasschar; + if (password != null) { + modifiedPasschar = Arrays.copyOf(password, password.length); + } else { + modifiedPasschar = new char[0]; + } + + doLogin(modifiedUserName, modifiedPasschar); + + Arrays.fill(password, ' '); + Arrays.fill(modifiedPasschar, ' '); + loginSuccessful = true; + } + + return loginSuccessful; + } + + @Override + public boolean commit() throws LoginException { + log("RemoteUnixLoginModule::commit() has been called. -> isLoginSuccess [" + loginSuccessful + "]"); + if (loginSuccessful) { + if (subject != null) { + subject.getPrincipals().add(new UnixUserPrincipal(userName.trim())); + if (loginGroups != null) { + loginGroups = loginGroups.trim(); + for (String group : loginGroups.split(",")) { + subject.getPrincipals().add(new UnixGroupPrincipal(group.trim())); + } + } + } + } else { + if (subject != null) { + subject.getPrincipals().clear(); + } + } + return loginSuccessful; + } + + @Override + public boolean abort() throws LoginException { + log("RemoteUnixLoginModule::abort() has been called."); + loginSuccessful = false; + return true; + } + + @Override + public boolean logout() throws LoginException { + if (subject != null) { + subject.getPrincipals().clear(); + } + return true; + } + + public void initParams(Properties options) { + String val = (String) options.get(JAAS_ENABLED_PARAM); + + if (val != null) { + remoteLoginEnabled = val.trim().equalsIgnoreCase("true"); + if (!remoteLoginEnabled) { + log("Skipping RemoteLogin - [" + JAAS_ENABLED_PARAM + "] => [" + val + "]"); + return; + } + } else { + remoteLoginEnabled = true; + } + + val = (String) options.get(DEBUG_PARAM); + if (val != null && (!val.equalsIgnoreCase("false"))) { + debug = true; + } else { + debug = false; + } + + remoteHostName = (String) options.get(REMOTE_LOGIN_HOST_PARAM); + log("RemoteHostName:" + remoteHostName); + + val = (String) options.get(REMOTE_LOGIN_AUTH_SERVICE_PORT_PARAM); + if (val != null) { + remoteHostAuthServicePort = Integer.parseInt(val.trim()); + } + log("remoteHostAuthServicePort:" + remoteHostAuthServicePort); + + val = (String) options.get(SSL_ENABLED_PARAM); + ssEnabled = (val != null) && val.trim().equalsIgnoreCase("true"); + log("ssEnabled: " + ssEnabled); + + if (ssEnabled) { + trustStorePath = (String) options.get(SSL_TRUSTSTORE_PATH_PARAM); + log("trustStorePath: " + trustStorePath); + + if (trustStorePath != null) { + trustStorePathPassword = (String) options.get(SSL_TRUSTSTORE_PATH_PASSWORD_PARAM); + if (trustStorePathPassword == null) { + trustStorePathPassword = ""; + } + log("trustStorePathPassword:*****"); + } + + keyStorePath = (String) options.get(SSL_KEYSTORE_PATH_PARAM); + log("keyStorePath:" + keyStorePath); + if (keyStorePath != null) { + keyStorePathPassword = (String) options.get(SSL_KEYSTORE_PATH_PASSWORD_PARAM); + if (keyStorePathPassword == null) { + keyStorePathPassword = ""; + } + log("keyStorePathPassword:*****"); + } + + String certValidationFlag = (String) options.get(SERVER_CERT_VALIDATION_PARAM); + serverCertValidation = (!(certValidationFlag != null && ("false".equalsIgnoreCase(certValidationFlag.trim())))); + log("Server Cert Validation : " + serverCertValidation); + } + } + + public void doLogin(String aUserName, char[] modifiedPasschar) throws LoginException { + /* Possible values + * null + * OK: group1, group2, group3 + * FAILED: Invalid Password + */ + String ret = getLoginReplyFromAuthService(aUserName, modifiedPasschar); + + if (ret == null) { + throw new LoginException("FAILED: unable to authenticate to AuthenticationService: " + remoteHostName + ":" + remoteHostAuthServicePort); + } else if (ret.startsWith("OK:")) { + loginSuccessful = true; + if (ret.length() > 3) { + this.loginGroups = ret.substring(3); + } + } else if (ret.startsWith("FAILED:")) { + loginSuccessful = false; + throw new LoginException("FAILED: unable to authenticate to AuthenticationService: " + remoteHostName + ":" + remoteHostAuthServicePort); + } else { + throw new LoginException("FAILED: unable to authenticate to AuthenticationService: " + remoteHostName + ":" + remoteHostAuthServicePort + ", msg:" + ret); + } + } + + private String getLoginReplyFromAuthService(String aUserName, char[] modifiedPasschar) throws LoginException { + String ret = null; + Socket sslsocket = null; + + char[] prefix = ("LOGIN:" + aUserName + " ").toCharArray(); + char[] tail = "\n".toCharArray(); + char[] loginData = new char[prefix.length + modifiedPasschar.length + tail.length]; + System.arraycopy(prefix, 0, loginData, 0, prefix.length); + System.arraycopy(modifiedPasschar, 0, loginData, prefix.length, modifiedPasschar.length); + System.arraycopy(tail, 0, loginData, prefix.length + modifiedPasschar.length, tail.length); + try { + try { + if (ssEnabled) { + SSLContext context = SSLContext.getInstance(SSL_ALGORITHM); + + KeyManager[] km = null; + + if (keyStorePath != null) { + KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType()); + + InputStream in = null; + + in = getFileInputStream(keyStorePath); + + try { + ks.load(in, keyStorePathPassword.toCharArray()); + } finally { + if (in != null) { + in.close(); + } + } + + KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + kmf.init(ks, keyStorePathPassword.toCharArray()); + km = kmf.getKeyManagers(); + } + + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + + TrustManager[] tm = null; + + if (serverCertValidation) { + KeyStore trustStoreKeyStore = null; + + if (trustStorePath != null) { + trustStoreKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + + InputStream in = null; + + in = getFileInputStream(trustStorePath); + + try { + trustStoreKeyStore.load(in, trustStorePathPassword.toCharArray()); + + trustManagerFactory.init(trustStoreKeyStore); + + tm = trustManagerFactory.getTrustManagers(); + } finally { + if (in != null) { + in.close(); + } + } + } + } else { + TrustManager ignoreValidationTM = new X509TrustManager() { + public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { + // Ignore Server Certificate Validation + } + + public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { + // Ignore Server Certificate Validation + } + + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + }; + + tm = new TrustManager[] {ignoreValidationTM}; + } + + SecureRandom random = new SecureRandom(); + + context.init(km, tm, random); + + SSLSocketFactory sf = context.getSocketFactory(); + + sslsocket = sf.createSocket(remoteHostName, remoteHostAuthServicePort); + } else { + sslsocket = new Socket(remoteHostName, remoteHostAuthServicePort); + } + + OutputStreamWriter writer = new OutputStreamWriter(sslsocket.getOutputStream()); + + writer.write(loginData); + + writer.flush(); + + BufferedReader reader = new BufferedReader(new InputStreamReader(sslsocket.getInputStream())); + + ret = reader.readLine(); + + reader.close(); + + writer.close(); + } finally { + if (sslsocket != null) { + sslsocket.close(); + } + } + } catch (Throwable t) { + throw new LoginException("FAILED: unable to authenticate to AuthenticationService: " + remoteHostName + ":" + remoteHostAuthServicePort + ", Exception: [" + t + "]"); + } finally { + log("Login of user String: {" + aUserName + "}, return from AuthServer: {" + ret + "}"); + Arrays.fill(loginData, ' '); + Arrays.fill(modifiedPasschar, ' '); + } + + return ret; + } + + private InputStream getFileInputStream(String path) throws FileNotFoundException { + InputStream ret = null; + + File f = new File(path); + + if (f.exists()) { + ret = new FileInputStream(f); + } else { + ret = getClass().getResourceAsStream(path); + + if (ret == null) { + if (!path.startsWith("/")) { + ret = getClass().getResourceAsStream("/" + path); + } + } + + if (ret == null) { + ret = ClassLoader.getSystemClassLoader().getResourceAsStream(path); + if (ret == null) { + if (!path.startsWith("/")) { + ret = ClassLoader.getSystemResourceAsStream("/" + path); + } + } + } + } + + return ret; + } + + private void log(String msg) { + if (debug) { + System.err.println("RemoteUnixLoginModule: " + msg); + } + } } diff --git a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UnixGroupPrincipal.java b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UnixGroupPrincipal.java index ecc416b469..129330b11f 100644 --- a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UnixGroupPrincipal.java +++ b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UnixGroupPrincipal.java @@ -17,24 +17,21 @@ * under the License. */ - package org.apache.ranger.authentication.unix.jaas; +package org.apache.ranger.authentication.unix.jaas; import java.io.Serializable; import java.security.Principal; public class UnixGroupPrincipal implements Principal, Serializable { + private static final long serialVersionUID = 8137147441841439754L; + private String groupName; - private static final long serialVersionUID = 8137147441841439754L; - - private String groupName; - - public UnixGroupPrincipal(String groupName) { - this.groupName = groupName; - } - - @Override - public String getName() { - return groupName; - } + public UnixGroupPrincipal(String groupName) { + this.groupName = groupName; + } + @Override + public String getName() { + return groupName; + } } diff --git a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UnixUserPrincipal.java b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UnixUserPrincipal.java index 84cb6c3f91..49da784619 100644 --- a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UnixUserPrincipal.java +++ b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UnixUserPrincipal.java @@ -17,24 +17,21 @@ * under the License. */ - package org.apache.ranger.authentication.unix.jaas; +package org.apache.ranger.authentication.unix.jaas; import java.io.Serializable; import java.security.Principal; public class UnixUserPrincipal implements Principal, Serializable { + private static final long serialVersionUID = -3568658536591178268L; + private String userName; - private static final long serialVersionUID = -3568658536591178268L; - - private String userName; - - public UnixUserPrincipal(String userName) { - this.userName = userName; - } - - @Override - public String getName() { - return userName; - } + public UnixUserPrincipal(String userName) { + this.userName = userName; + } + @Override + public String getName() { + return userName; + } } diff --git a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UsernamePasswordCallbackHandler.java b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UsernamePasswordCallbackHandler.java index 58edfbc964..eda62d6237 100644 --- a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UsernamePasswordCallbackHandler.java +++ b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UsernamePasswordCallbackHandler.java @@ -16,17 +16,22 @@ */ package org.apache.ranger.authentication.unix.jaas; -import javax.security.auth.callback.*; +import javax.security.auth.callback.Callback; +import javax.security.auth.callback.CallbackHandler; +import javax.security.auth.callback.NameCallback; +import javax.security.auth.callback.PasswordCallback; +import javax.security.auth.callback.UnsupportedCallbackException; + import java.io.IOException; public class UsernamePasswordCallbackHandler implements CallbackHandler { - private String _user; - private String _password; + private String user; + private String password; public UsernamePasswordCallbackHandler(String user, String password) { super(); - _user = user; - _password = password; + user = user; + password = password; } @Override @@ -43,12 +48,11 @@ public void handle(Callback[] callbacks) throws IOException, UnsupportedCallback } private void handleName(NameCallback callback) { - callback.setName(_user); + callback.setName(user); } private void handlePassword(PasswordCallback callback) { - char[] passwordChars = _password.toCharArray(); + char[] passwordChars = password.toCharArray(); callback.setPassword(passwordChars); } } - diff --git a/unixauthclient/src/test/com/xasecure/test/authentication/UnixAuthenticationTester.java b/unixauthclient/src/test/com/xasecure/test/authentication/UnixAuthenticationTester.java index 41317f6a7d..d80dbc6ab9 100644 --- a/unixauthclient/src/test/com/xasecure/test/authentication/UnixAuthenticationTester.java +++ b/unixauthclient/src/test/com/xasecure/test/authentication/UnixAuthenticationTester.java @@ -17,22 +17,21 @@ * under the License. */ - package com.xasecure.test.authentication; +package com.xasecure.test.authentication; import javax.security.auth.login.LoginContext; public class UnixAuthenticationTester { - public static void main(String[] args) throws Throwable { - new UnixAuthenticationTester().run(); - } - - public void run() throws Throwable { - LoginContext loginContext = new LoginContext("PolicyManager"); - System.err.println("After login ..."); - loginContext.login(); - System.err.println("Subject:" + loginContext.getSubject() ); - loginContext.logout(); - } + public static void main(String[] args) throws Throwable { + new UnixAuthenticationTester().run(); + } + public void run() throws Throwable { + LoginContext loginContext = new LoginContext("PolicyManager"); + System.err.println("After login ..."); + loginContext.login(); + System.err.println("Subject:" + loginContext.getSubject()); + loginContext.logout(); + } } diff --git a/unixauthnative/pom.xml b/unixauthnative/pom.xml index f950630cdd..e6349aad77 100644 --- a/unixauthnative/pom.xml +++ b/unixauthnative/pom.xml @@ -27,6 +27,9 @@ uexe Unix Native Authenticator Unix authentication service + + true + diff --git a/unixauthnative/src/main/c/credValidator.c b/unixauthnative/src/main/c/credValidator.c index d79eb3b572..891b0b9e05 100644 --- a/unixauthnative/src/main/c/credValidator.c +++ b/unixauthnative/src/main/c/credValidator.c @@ -33,7 +33,7 @@ int main(int ac, char **av, char **ev) char line[512] ; char format[20]; struct passwd *pwp; - struct spwd *spwd ; + struct spwd *spwd ; fgets(line,512,stdin) ; sprintf(format, "LOGIN:%%%ds %%%ds", STRLEN-1, STRLEN-1); @@ -45,7 +45,7 @@ int main(int ac, char **av, char **ev) fprintf(stdout, "FAILED: [%s] does not exists.\n", username) ; exit(1) ; } - + spwd = getspnam(pwp->pw_name) ; if (spwd == (struct spwd *)NULL) { diff --git a/unixauthpam/pom.xml b/unixauthpam/pom.xml index f8bf697f59..0103e4f52c 100644 --- a/unixauthpam/pom.xml +++ b/unixauthpam/pom.xml @@ -27,6 +27,9 @@ uexe PAM Authenticator PAM authentication service + + true + diff --git a/unixauthservice/pom.xml b/unixauthservice/pom.xml index 07f7ab760f..be831c8773 100644 --- a/unixauthservice/pom.xml +++ b/unixauthservice/pom.xml @@ -27,6 +27,9 @@ jar Unix Authentication Service Unix authentication service + + true + ch.qos.logback diff --git a/unixauthservice/src/main/java/org/apache/ranger/authentication/PasswordValidator.java b/unixauthservice/src/main/java/org/apache/ranger/authentication/PasswordValidator.java index c924932aec..a6d6dd4d92 100644 --- a/unixauthservice/src/main/java/org/apache/ranger/authentication/PasswordValidator.java +++ b/unixauthservice/src/main/java/org/apache/ranger/authentication/PasswordValidator.java @@ -17,7 +17,10 @@ * under the License. */ - package org.apache.ranger.authentication; +package org.apache.ranger.authentication; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.IOException; @@ -27,135 +30,121 @@ import java.net.Socket; import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - public class PasswordValidator implements Runnable { - - private static final Logger LOG = LoggerFactory.getLogger(PasswordValidator.class); - - private static String validatorProgram = null; - - private static List adminUserList; - - private static String adminRoleNames; - - private Socket client; - - public PasswordValidator(Socket client) { - this.client = client; - } - - @Override - public void run() { - BufferedReader reader = null; - PrintWriter writer = null; - - String userName = null; - - try { - reader = new BufferedReader(new InputStreamReader(client.getInputStream())); - writer = new PrintWriter(new OutputStreamWriter(client.getOutputStream())); - String request = reader.readLine(); - - if (request.startsWith("LOGIN:")) { - String line = request.substring(6).trim(); - int passwordAt = line.indexOf(' '); - if (passwordAt != -1) { - userName = line.substring(0,passwordAt).trim(); - } - } - - if (validatorProgram == null) { - String res = "FAILED: Unable to validate credentials."; - writer.println(res); - writer.flush(); - LOG.error("Response [" + res + "] for user: " + userName + " as ValidatorProgram is not defined in configuration."); - - } - else { - - BufferedReader pReader = null; - PrintWriter pWriter = null; - Process p = null; - - try { - p = Runtime.getRuntime().exec(validatorProgram); - - pReader = new BufferedReader(new InputStreamReader(p.getInputStream())); - - pWriter = new PrintWriter(new OutputStreamWriter(p.getOutputStream())); - - pWriter.println(request); pWriter.flush(); - - String res = pReader.readLine(); - - - if (res != null && res.startsWith("OK")) { - if (adminRoleNames != null && adminUserList != null) { - if (adminUserList.contains(userName)) { - res = res + " " + adminRoleNames; - } - } - } - - LOG.info("Response [" + res + "] for user: " + userName); - - writer.println(res); writer.flush(); - } - finally { - if (p != null) { - p.destroy(); - } - } - } - - } - catch(Throwable t) { - if (userName != null && writer != null ) { - String res = "FAILED: unable to validate due to error " + t.getMessage(); - writer.println(res); - LOG.error("Response [" + res + "] for user: " + userName+", "+ t.getMessage()); - } - } - finally { - try { - if (client != null) { - client.close(); - } - } - catch(IOException ioe){ - LOG.debug("Close socket failure. Detail: \n", ioe); - } - finally { - client = null; - } - } - } - - - public static String getValidatorProgram() { - return validatorProgram; - } - - public static void setValidatorProgram(String validatorProgram) { - PasswordValidator.validatorProgram = validatorProgram; - } - - public static List getAdminUserList() { - return adminUserList; - } - - public static void setAdminUserList(List adminUserList) { - PasswordValidator.adminUserList = adminUserList; - } - - public static String getAdminRoleNames() { - return adminRoleNames; - } - - public static void setAdminRoleNames(String adminRoleNames) { - PasswordValidator.adminRoleNames = adminRoleNames; - } - + private static final Logger LOG = LoggerFactory.getLogger(PasswordValidator.class); + + private static String validatorProgram; + + private static List adminUserList; + + private static String adminRoleNames; + + private Socket client; + + public PasswordValidator(Socket client) { + this.client = client; + } + + public static String getValidatorProgram() { + return validatorProgram; + } + + public static void setValidatorProgram(String validatorProgram) { + PasswordValidator.validatorProgram = validatorProgram; + } + + public static List getAdminUserList() { + return adminUserList; + } + + public static void setAdminUserList(List adminUserList) { + PasswordValidator.adminUserList = adminUserList; + } + + public static String getAdminRoleNames() { + return adminRoleNames; + } + + public static void setAdminRoleNames(String adminRoleNames) { + PasswordValidator.adminRoleNames = adminRoleNames; + } + + @Override + public void run() { + BufferedReader reader = null; + PrintWriter writer = null; + + String userName = null; + + try { + reader = new BufferedReader(new InputStreamReader(client.getInputStream())); + writer = new PrintWriter(new OutputStreamWriter(client.getOutputStream())); + String request = reader.readLine(); + + if (request.startsWith("LOGIN:")) { + String line = request.substring(6).trim(); + int passwordAt = line.indexOf(' '); + if (passwordAt != -1) { + userName = line.substring(0, passwordAt).trim(); + } + } + + if (validatorProgram == null) { + String res = "FAILED: Unable to validate credentials."; + writer.println(res); + writer.flush(); + LOG.error("Response [{}] for user: {} as ValidatorProgram is not defined in configuration", res, userName); + } else { + BufferedReader pReader = null; + PrintWriter pWriter = null; + Process p = null; + + try { + p = Runtime.getRuntime().exec(validatorProgram); + + pReader = new BufferedReader(new InputStreamReader(p.getInputStream())); + + pWriter = new PrintWriter(new OutputStreamWriter(p.getOutputStream())); + + pWriter.println(request); + pWriter.flush(); + + String res = pReader.readLine(); + + if (res != null && res.startsWith("OK")) { + if (adminRoleNames != null && adminUserList != null) { + if (adminUserList.contains(userName)) { + res = res + " " + adminRoleNames; + } + } + } + + LOG.info("Response [{}] for user: {}", res, userName); + + writer.println(res); + writer.flush(); + } finally { + if (p != null) { + p.destroy(); + } + } + } + } catch (Throwable t) { + if (userName != null && writer != null) { + String res = "FAILED: unable to validate due to error " + t.getMessage(); + writer.println(res); + LOG.error("Response [{}] for user: {} message: {}", res, userName, t.getMessage()); + } + } finally { + try { + if (client != null) { + client.close(); + } + } catch (IOException ioe) { + LOG.debug("Close socket failure. Detail: ", ioe); + } finally { + client = null; + } + } + } } diff --git a/unixauthservice/src/main/java/org/apache/ranger/authentication/UnixAuthenticationService.java b/unixauthservice/src/main/java/org/apache/ranger/authentication/UnixAuthenticationService.java index d3d14a1751..97c07072cd 100644 --- a/unixauthservice/src/main/java/org/apache/ranger/authentication/UnixAuthenticationService.java +++ b/unixauthservice/src/main/java/org/apache/ranger/authentication/UnixAuthenticationService.java @@ -19,6 +19,23 @@ package org.apache.ranger.authentication; +import org.apache.ranger.credentialapi.CredentialReader; +import org.apache.ranger.plugin.util.XMLUtils; +import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; +import org.apache.ranger.unixusersync.ha.UserSyncHAInitializerImpl; +import org.apache.ranger.usergroupsync.UserGroupSync; +import org.apache.ranger.usergroupsync.UserSyncMetricsProducer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLServerSocket; +import javax.net.ssl.SSLServerSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; + import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -35,354 +52,292 @@ import java.util.Properties; import java.util.Set; -import javax.net.ssl.KeyManager; -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLServerSocket; -import javax.net.ssl.SSLServerSocketFactory; -import javax.net.ssl.TrustManager; -import javax.net.ssl.TrustManagerFactory; +public class UnixAuthenticationService { + private static final Logger LOG = LoggerFactory.getLogger(UnixAuthenticationService.class); + private static final String serviceName = "UnixAuthenticationService"; + private static final String SSL_ALGORITHM = "TLSv1.2"; + private static final String REMOTE_LOGIN_AUTH_SERVICE_PORT_PARAM = "ranger.usersync.port"; + private static final String SSL_KEYSTORE_PATH_PARAM = "ranger.usersync.keystore.file"; + private static final String SSL_TRUSTSTORE_PATH_PARAM = "ranger.usersync.truststore.file"; + private static final String SSL_KEYSTORE_FILE_TYPE_PARAM = "ranger.keystore.file.type"; + private static final String SSL_TRUSTSTORE_FILE_TYPE_PARAM = "ranger.truststore.file.type"; + private static final String SSL_KEYSTORE_PATH_PASSWORD_ALIAS = "usersync.ssl.key.password"; + private static final String SSL_TRUSTSTORE_PATH_PASSWORD_ALIAS = "usersync.ssl.truststore.password"; + private static final String CRED_VALIDATOR_PROG = "ranger.usersync.passwordvalidator.path"; + private static final String ADMIN_USER_LIST_PARAM = "admin.users"; + private static final String ADMIN_ROLE_LIST_PARAM = "admin.roleNames"; + private static final String SSL_ENABLED_PARAM = "ranger.usersync.ssl"; + private static final String CREDSTORE_FILENAME_PARAM = "ranger.usersync.credstore.filename"; + private static final String[] UGSYNC_CONFIG_XML_FILES = {"ranger-ugsync-default.xml", "ranger-ugsync-site.xml"}; + private static boolean enableUnixAuth; + private String keyStorePath; + private String keyStoreType; + private List enabledProtocolsList; + private List enabledCipherSuiteList; + private String keyStorePathPassword; + private String trustStorePath; + private String trustStorePathPassword; + private String trustStoreType; + private List adminUserList = new ArrayList(); + private String adminRoleNames; + private UserSyncHAInitializerImpl userSyncHAInitializerImpl; + private int portNum; + private boolean sslEnabled; -import org.apache.ranger.unixusersync.ha.UserSyncHAInitializerImpl; -import org.apache.ranger.credentialapi.CredentialReader; -import org.apache.ranger.plugin.util.XMLUtils; -import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; -import org.apache.ranger.usergroupsync.UserGroupSync; -import org.apache.ranger.usergroupsync.UserSyncMetricsProducer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + public UnixAuthenticationService() { + } -public class UnixAuthenticationService { + public static void main(String[] args) { + enableUnixAuth = Arrays.stream(args).anyMatch(arg -> "-enableUnixAuth".equalsIgnoreCase(arg)); + UnixAuthenticationService service = new UnixAuthenticationService(); + service.userSyncHAInitializerImpl = UserSyncHAInitializerImpl.getInstance(UserGroupSyncConfig.getInstance().getUserGroupConfig()); + service.run(); + } + + public void run() { + try { + LOG.info("Starting User Sync Service!"); + startUnixUserGroupSyncProcess(); + Thread.sleep(5000); + if (enableUnixAuth) { + LOG.info("Enabling Unix Auth Service!"); + init(); + startService(); + } else { + LOG.info("Unix Auth Service Disabled!"); + } + } catch (Throwable t) { + LOG.error("ERROR: Service: {}", serviceName, t); + } finally { + LOG.info("Service: {} - STOPPED", serviceName); + if (this.userSyncHAInitializerImpl != null) { + LOG.info("Stopping curator leader latch service as main thread is closing"); + this.userSyncHAInitializerImpl.stop(); + } + } + } + + public void startService() throws Throwable { + SSLContext context = SSLContext.getInstance(SSL_ALGORITHM); + + KeyManager[] km = null; + + if (keyStorePath != null && !keyStorePath.isEmpty()) { + KeyStore ks = KeyStore.getInstance(keyStoreType); + + InputStream in = null; - private static final Logger LOG = LoggerFactory.getLogger(UnixAuthenticationService.class); - - private static final String serviceName = "UnixAuthenticationService"; - - private static final String SSL_ALGORITHM = "TLSv1.2"; - private static final String REMOTE_LOGIN_AUTH_SERVICE_PORT_PARAM = "ranger.usersync.port"; - - private static final String SSL_KEYSTORE_PATH_PARAM = "ranger.usersync.keystore.file"; - private static final String SSL_TRUSTSTORE_PATH_PARAM = "ranger.usersync.truststore.file"; - - private static final String SSL_KEYSTORE_FILE_TYPE_PARAM = "ranger.keystore.file.type"; - private static final String SSL_TRUSTSTORE_FILE_TYPE_PARAM = "ranger.truststore.file.type"; - - private static final String SSL_KEYSTORE_PATH_PASSWORD_ALIAS = "usersync.ssl.key.password"; - private static final String SSL_TRUSTSTORE_PATH_PASSWORD_ALIAS = "usersync.ssl.truststore.password"; - - private static final String CRED_VALIDATOR_PROG = "ranger.usersync.passwordvalidator.path"; - private static final String ADMIN_USER_LIST_PARAM = "admin.users"; - private static final String ADMIN_ROLE_LIST_PARAM = "admin.roleNames"; - private static final String SSL_ENABLED_PARAM = "ranger.usersync.ssl"; - - private static final String CREDSTORE_FILENAME_PARAM = "ranger.usersync.credstore.filename"; - - private String keyStorePath; - private String keyStoreType; - private List enabledProtocolsList; - private List enabledCipherSuiteList; - private String keyStorePathPassword; - private String trustStorePath; - private String trustStorePathPassword; - private String trustStoreType; - private List adminUserList = new ArrayList(); - private String adminRoleNames; - private UserSyncHAInitializerImpl userSyncHAInitializerImpl = null; - - private int portNum; - - private boolean SSLEnabled = false; - - static private boolean enableUnixAuth = false; - - private static final String[] UGSYNC_CONFIG_XML_FILES = { "ranger-ugsync-default.xml", "ranger-ugsync-site.xml" }; - - public static void main(String[] args) { - if (args.length > 0) { - for (String s : args) { - if ("-enableUnixAuth".equalsIgnoreCase(s)) { - enableUnixAuth = true; - break; - } - } - } - UnixAuthenticationService service = new UnixAuthenticationService(); - service.userSyncHAInitializerImpl = UserSyncHAInitializerImpl.getInstance(UserGroupSyncConfig.getInstance().getUserGroupConfig()); - service.run(); - } - - public UnixAuthenticationService() { - } - - - public void run() { - try { - LOG.info("Starting User Sync Service!"); - startUnixUserGroupSyncProcess(); - Thread.sleep(5000); - if (enableUnixAuth) { - LOG.info("Enabling Unix Auth Service!"); - init(); - startService(); - } else { - LOG.info("Unix Auth Service Disabled!"); - } - } - catch(Throwable t) { - LOG.error("ERROR: Service: " + serviceName , t); - } - finally { - LOG.info("Service: " + serviceName + " - STOPPED."); - if(this.userSyncHAInitializerImpl != null) { - LOG.info("Stopping curator leader latch service as main thread is closing"); - this.userSyncHAInitializerImpl.stop(); - } - } - } - - private void startUnixUserGroupSyncProcess() { - // - // Start the synchronization service ... - // - LOG.info("Start : startUnixUserGroupSyncProcess " ); - UserGroupSync syncProc = new UserGroupSync(); - Thread newSyncProcThread = new Thread(syncProc); - newSyncProcThread.setName("UnixUserSyncThread"); - // If this thread is set as daemon, then the entire process will terminate if enableUnixAuth is false + in = getFileInputStream(keyStorePath); + + try { + if (keyStorePathPassword == null) { + keyStorePathPassword = ""; + } + ks.load(in, keyStorePathPassword.toCharArray()); + } finally { + if (in != null) { + in.close(); + } + } + + KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + kmf.init(ks, keyStorePathPassword.toCharArray()); + km = kmf.getKeyManagers(); + } + + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + + KeyStore trustStoreKeyStore = null; + + if (trustStorePath != null && !trustStorePath.isEmpty()) { + trustStoreKeyStore = KeyStore.getInstance(trustStoreType); + + InputStream in = null; + + in = getFileInputStream(trustStorePath); + + try { + if (trustStorePathPassword == null) { + trustStorePathPassword = ""; + } + trustStoreKeyStore.load(in, trustStorePathPassword.toCharArray()); + } finally { + if (in != null) { + in.close(); + } + } + } + + trustManagerFactory.init(trustStoreKeyStore); + + TrustManager[] tm = trustManagerFactory.getTrustManagers(); + + SecureRandom random = new SecureRandom(); + + context.init(km, tm, random); + + SSLServerSocketFactory sf = context.getServerSocketFactory(); + + try (ServerSocket socket = (sslEnabled ? sf.createServerSocket(portNum) : new ServerSocket(portNum))) { + if (sslEnabled) { + SSLServerSocket secureSocket = (SSLServerSocket) socket; + String[] protocols = secureSocket.getEnabledProtocols(); + Set allowedProtocols = new HashSet(); + for (String ep : protocols) { + if (enabledProtocolsList.contains(ep.toUpperCase())) { + LOG.info("Enabling Protocol: [{}]", ep); + allowedProtocols.add(ep); + } else { + LOG.info("Disabling Protocol: [{}]", ep); + } + } + + if (!allowedProtocols.isEmpty()) { + secureSocket.setEnabledProtocols(allowedProtocols.toArray(new String[0])); + } + String[] enabledCipherSuites = secureSocket.getEnabledCipherSuites(); + Set allowedCipherSuites = new HashSet(); + for (String enabledCipherSuite : enabledCipherSuites) { + if (enabledCipherSuiteList.contains(enabledCipherSuite)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Enabling CipherSuite : [{}]", enabledCipherSuite); + } + allowedCipherSuites.add(enabledCipherSuite); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Disabling CipherSuite : [{}]", enabledCipherSuite); + } + } + } + if (!allowedCipherSuites.isEmpty()) { + secureSocket.setEnabledCipherSuites(allowedCipherSuites.toArray(new String[0])); + } + } + + Socket client = null; + + try { + while ((client = socket.accept()) != null) { + Thread clientValidatorThread = new Thread(new PasswordValidator(client)); + clientValidatorThread.start(); + } + } catch (IOException e) { + socket.close(); + throw (e); + } + } + } + + private void startUnixUserGroupSyncProcess() { + // Start user group synchronization service + LOG.info("Start : startUnixUserGroupSyncProcess "); + UserGroupSync syncProc = new UserGroupSync(); + Thread newSyncProcThread = new Thread(syncProc); + newSyncProcThread.setName("UnixUserSyncThread"); + // If this thread is set as daemon, then the entire process will terminate if enableUnixAuth is false // Therefore this is marked as non-daemon thread. Don't change the following line - newSyncProcThread.setDaemon(false); - newSyncProcThread.start(); + newSyncProcThread.setDaemon(false); + newSyncProcThread.start(); LOG.info("UnixUserSyncThread started"); - LOG.info("creating UserSyncMetricsProducer thread with default metrics location : "+System.getProperty("logdir")); - //Start the user sync metrics - boolean isUserSyncMetricsEnabled = UserGroupSyncConfig.getInstance().isUserSyncMetricsEnabled(); - if (isUserSyncMetricsEnabled) { - UserSyncMetricsProducer userSyncMetricsProducer = new UserSyncMetricsProducer(); - Thread userSyncMetricsProducerThread = new Thread(userSyncMetricsProducer); - userSyncMetricsProducerThread.setName("UserSyncMetricsProducerThread"); - userSyncMetricsProducerThread.setDaemon(false); - userSyncMetricsProducerThread.start(); - LOG.info("UserSyncMetricsProducer started"); - } else { - LOG.info(" Ranger userSync metrics is not enabled"); - } - } - - - //TODO: add more validation code - private void init() throws Throwable { - Properties prop = new Properties(); - - for (String fn : UGSYNC_CONFIG_XML_FILES ) { + LOG.info("creating UserSyncMetricsProducer thread with default metrics location : {}", System.getProperty("logdir")); + //Start the user sync metrics + boolean isUserSyncMetricsEnabled = UserGroupSyncConfig.getInstance().isUserSyncMetricsEnabled(); + if (isUserSyncMetricsEnabled) { + UserSyncMetricsProducer userSyncMetricsProducer = new UserSyncMetricsProducer(); + Thread userSyncMetricsProducerThread = new Thread(userSyncMetricsProducer); + userSyncMetricsProducerThread.setName("UserSyncMetricsProducerThread"); + userSyncMetricsProducerThread.setDaemon(false); + userSyncMetricsProducerThread.start(); + LOG.info("UserSyncMetricsProducer started"); + } else { + LOG.info(" Ranger userSync metrics is not enabled"); + } + } + + //TODO: add more validation code + private void init() throws Throwable { + Properties prop = new Properties(); + + for (String fn : UGSYNC_CONFIG_XML_FILES) { XMLUtils.loadConfig(fn, prop); - } - - String credStoreFileName = prop.getProperty(CREDSTORE_FILENAME_PARAM); - - keyStorePath = prop.getProperty(SSL_KEYSTORE_PATH_PARAM); - - keyStoreType = prop.getProperty(SSL_KEYSTORE_FILE_TYPE_PARAM, KeyStore.getDefaultType()); - trustStoreType = prop.getProperty(SSL_TRUSTSTORE_FILE_TYPE_PARAM, KeyStore.getDefaultType()); - - if (credStoreFileName == null) { - throw new RuntimeException("Credential file is not defined. param = [" + CREDSTORE_FILENAME_PARAM + "]"); - } - - File credFile = new File(credStoreFileName); - - if (! credFile.exists()) { - throw new RuntimeException("Credential file [" + credStoreFileName + "]: does not exists." ); - } - - if ( ! credFile.canRead() ) { - throw new RuntimeException("Credential file [" + credStoreFileName + "]: can not be read." ); - } - - if ("bcfks".equalsIgnoreCase(keyStoreType)) { - String crendentialProviderPrefixBcfks= "bcfks" + "://file"; - credStoreFileName = crendentialProviderPrefixBcfks + credStoreFileName; - } - keyStorePathPassword = CredentialReader.getDecryptedString(credStoreFileName, SSL_KEYSTORE_PATH_PASSWORD_ALIAS, keyStoreType); - trustStorePathPassword = CredentialReader.getDecryptedString(credStoreFileName,SSL_TRUSTSTORE_PATH_PASSWORD_ALIAS, trustStoreType); - - trustStorePath = prop.getProperty(SSL_TRUSTSTORE_PATH_PARAM); - portNum = Integer.parseInt(prop.getProperty(REMOTE_LOGIN_AUTH_SERVICE_PORT_PARAM)); - String validatorProg = prop.getProperty(CRED_VALIDATOR_PROG); - if (validatorProg != null) { - PasswordValidator.setValidatorProgram(validatorProg); - } - - String adminUsers = prop.getProperty(ADMIN_USER_LIST_PARAM); - - if (adminUsers != null && adminUsers.trim().length() > 0) { - for(String u : adminUsers.split(",")) { - LOG.info("Adding Admin User:" + u.trim()); - adminUserList.add(u.trim()); - } - PasswordValidator.setAdminUserList(adminUserList); - } - - - adminRoleNames = prop.getProperty(ADMIN_ROLE_LIST_PARAM); - - if (adminRoleNames != null) { - LOG.info("Adding Admin Group:" + adminRoleNames); - PasswordValidator.setAdminRoleNames(adminRoleNames); - } - - String SSLEnabledProp = prop.getProperty(SSL_ENABLED_PARAM); - - SSLEnabled = (SSLEnabledProp != null && (SSLEnabledProp.equalsIgnoreCase("true"))); - String defaultEnabledProtocols = "TLSv1.2"; - String enabledProtocols = prop.getProperty("ranger.usersync.https.ssl.enabled.protocols", defaultEnabledProtocols); - String enabledCipherSuites = prop.getProperty("ranger.usersync.https.ssl.enabled.cipher.suites", ""); - enabledProtocolsList=new ArrayList(Arrays.asList(enabledProtocols.toUpperCase().trim().split("\\s*,\\s*"))); - enabledCipherSuiteList = new ArrayList(Arrays.asList(enabledCipherSuites.toUpperCase().trim().split("\\s*,\\s*"))); -// LOG.info("Key:" + keyStorePath); -// LOG.info("KeyPassword:" + keyStorePathPassword); -// LOG.info("TrustStore:" + trustStorePath); -// LOG.info("PortNum:" + portNum); -// LOG.info("ValidatorProg:" + validatorProg); - - } - - - public void startService() throws Throwable { - - SSLContext context = SSLContext.getInstance(SSL_ALGORITHM); - - KeyManager[] km = null; - - if (keyStorePath != null && ! keyStorePath.isEmpty()) { - KeyStore ks = KeyStore.getInstance(keyStoreType); - - InputStream in = null; - - in = getFileInputStream(keyStorePath); - - try { - if (keyStorePathPassword == null) { - keyStorePathPassword = ""; - } - ks.load(in, keyStorePathPassword.toCharArray()); - } - finally { - if (in != null) { - in.close(); - } - } - - KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - kmf.init(ks, keyStorePathPassword.toCharArray()); - km = kmf.getKeyManagers(); - } - - - TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - - KeyStore trustStoreKeyStore = null; - - if (trustStorePath != null && ! trustStorePath.isEmpty()) { - trustStoreKeyStore = KeyStore.getInstance(trustStoreType); - - InputStream in = null; - - in = getFileInputStream(trustStorePath); - - try { - if (trustStorePathPassword == null) { - trustStorePathPassword = ""; - } - trustStoreKeyStore.load(in, trustStorePathPassword.toCharArray()); - } - finally { - if (in != null) { - in.close(); - } - } - } - - trustManagerFactory.init(trustStoreKeyStore); - - TrustManager[] tm = trustManagerFactory.getTrustManagers(); - - SecureRandom random = new SecureRandom(); - - context.init(km, tm, random); - - SSLServerSocketFactory sf = context.getServerSocketFactory(); - - try(ServerSocket socket = (SSLEnabled ? sf.createServerSocket(portNum) : new ServerSocket(portNum) )) { - - if (SSLEnabled) { - SSLServerSocket secureSocket = (SSLServerSocket) socket; - String[] protocols = secureSocket.getEnabledProtocols(); - Set allowedProtocols = new HashSet(); - for (String ep : protocols) { - if (enabledProtocolsList.contains(ep.toUpperCase())) { - LOG.info("Enabling Protocol: [" + ep + "]"); - allowedProtocols.add(ep); - } else { - LOG.info("Disabling Protocol: [" + ep + "]"); - } - } - - if (!allowedProtocols.isEmpty()) { - secureSocket.setEnabledProtocols(allowedProtocols.toArray(new String[0])); - } - String[] enabledCipherSuites = secureSocket.getEnabledCipherSuites(); - Set allowedCipherSuites = new HashSet(); - for (String enabledCipherSuite : enabledCipherSuites) { - if (enabledCipherSuiteList.contains(enabledCipherSuite)) { - if (LOG.isDebugEnabled()) { - LOG.debug("Enabling CipherSuite : [" + enabledCipherSuite + "]"); - } - allowedCipherSuites.add(enabledCipherSuite); - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("Disabling CipherSuite : [" + enabledCipherSuite + "]"); - } - } - } - if (!allowedCipherSuites.isEmpty()) { - secureSocket.setEnabledCipherSuites(allowedCipherSuites.toArray(new String[0])); - } - } - - - Socket client = null; - - try { - - while ((client = socket.accept()) != null) { - Thread clientValidatorThread = new Thread(new PasswordValidator(client)); - clientValidatorThread.start(); - } - } catch (IOException e) { - socket.close(); - throw (e); - } - } - - } - - private InputStream getFileInputStream(String path) throws FileNotFoundException { - - InputStream ret = null; - - File f = new File(path); - - if (f.exists()) { - ret = new FileInputStream(f); - } - else { - ret = getClass().getResourceAsStream(path); - if (ret == null) { - ret = getClass().getResourceAsStream(File.separator + path); - } - } - - return ret; - } + } + + String credStoreFileName = prop.getProperty(CREDSTORE_FILENAME_PARAM); + + keyStorePath = prop.getProperty(SSL_KEYSTORE_PATH_PARAM); + + keyStoreType = prop.getProperty(SSL_KEYSTORE_FILE_TYPE_PARAM, KeyStore.getDefaultType()); + trustStoreType = prop.getProperty(SSL_TRUSTSTORE_FILE_TYPE_PARAM, KeyStore.getDefaultType()); + + if (credStoreFileName == null) { + throw new RuntimeException("Credential file is not defined. param = [" + CREDSTORE_FILENAME_PARAM + "]"); + } + + File credFile = new File(credStoreFileName); + + if (!credFile.exists()) { + throw new RuntimeException("Credential file [" + credStoreFileName + "]: does not exists."); + } + + if (!credFile.canRead()) { + throw new RuntimeException("Credential file [" + credStoreFileName + "]: can not be read."); + } + + if ("bcfks".equalsIgnoreCase(keyStoreType)) { + String crendentialProviderPrefixBcfks = "bcfks://file"; + credStoreFileName = crendentialProviderPrefixBcfks + credStoreFileName; + } + keyStorePathPassword = CredentialReader.getDecryptedString(credStoreFileName, SSL_KEYSTORE_PATH_PASSWORD_ALIAS, keyStoreType); + trustStorePathPassword = CredentialReader.getDecryptedString(credStoreFileName, SSL_TRUSTSTORE_PATH_PASSWORD_ALIAS, trustStoreType); + + trustStorePath = prop.getProperty(SSL_TRUSTSTORE_PATH_PARAM); + portNum = Integer.parseInt(prop.getProperty(REMOTE_LOGIN_AUTH_SERVICE_PORT_PARAM)); + String validatorProg = prop.getProperty(CRED_VALIDATOR_PROG); + if (validatorProg != null) { + PasswordValidator.setValidatorProgram(validatorProg); + } + + String adminUsers = prop.getProperty(ADMIN_USER_LIST_PARAM); + + if (adminUsers != null && adminUsers.trim().length() > 0) { + for (String u : adminUsers.split(",")) { + LOG.info("Adding Admin User: {}", u.trim()); + adminUserList.add(u.trim()); + } + PasswordValidator.setAdminUserList(adminUserList); + } + + adminRoleNames = prop.getProperty(ADMIN_ROLE_LIST_PARAM); + + if (adminRoleNames != null) { + LOG.info("Adding Admin Group: {}", adminRoleNames); + PasswordValidator.setAdminRoleNames(adminRoleNames); + } + + String sslEnabledProp = prop.getProperty(SSL_ENABLED_PARAM); + + sslEnabled = (sslEnabledProp != null && (sslEnabledProp.equalsIgnoreCase("true"))); + String defaultEnabledProtocols = "TLSv1.2"; + String enabledProtocols = prop.getProperty("ranger.usersync.https.ssl.enabled.protocols", defaultEnabledProtocols); + String enabledCipherSuites = prop.getProperty("ranger.usersync.https.ssl.enabled.cipher.suites", ""); + enabledProtocolsList = new ArrayList(Arrays.asList(enabledProtocols.toUpperCase().trim().split("\\s*,\\s*"))); + enabledCipherSuiteList = new ArrayList(Arrays.asList(enabledCipherSuites.toUpperCase().trim().split("\\s*,\\s*"))); + } + + private InputStream getFileInputStream(String path) throws FileNotFoundException { + InputStream ret = null; + + File f = new File(path); + + if (f.exists()) { + ret = new FileInputStream(f); + } else { + ret = getClass().getResourceAsStream(path); + if (ret == null) { + ret = getClass().getResourceAsStream(File.separator + path); + } + } + return ret; + } } From 5fdfd54cc5baf0172d01519bb9f320700c53ff5c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 11 Dec 2024 01:25:44 -0800 Subject: [PATCH 2/4] RANGER-5064: misc changes - to continue --- ugsync-util/pom.xml | 2 +- .../ldapconfigcheck/pom.xml | 1 + .../ldapconfigcheck/AuthenticationCheck.java | 24 +- .../ldapconfigcheck/CommandLineOptions.java | 46 ++-- .../ranger/ldapconfigcheck/LdapConfig.java | 43 ++- .../ldapconfigcheck/LdapConfigCheckMain.java | 9 +- .../ranger/ldapconfigcheck/UserSync.java | 107 ++++---- ugsync/pom.xml | 1 + .../process/CustomSSLSocketFactory.java | 42 ++- .../process/LdapUserGroupBuilder.java | 247 +++++++++--------- .../config/UserGroupSyncConfig.java | 187 ++++++------- .../ha/UserSyncHAInitializerImpl.java | 8 +- .../process/FileSourceUserGroupBuilder.java | 17 +- .../process/PolicyMgrUserGroupBuilder.java | 1 + .../process/RangerUgSyncRESTClient.java | 12 +- .../process/UnixUserGroupBuilder.java | 46 ++-- .../apache/ranger/usergroupsync/RegEx.java | 4 +- .../ranger/usergroupsync/UserGroupSync.java | 19 +- .../UserSyncMetricsProducer.java | 16 +- unixauthclient/pom.xml | 1 + unixauthnative/pom.xml | 1 + unixauthpam/pom.xml | 1 + unixauthservice/pom.xml | 1 + 23 files changed, 424 insertions(+), 412 deletions(-) diff --git a/ugsync-util/pom.xml b/ugsync-util/pom.xml index c6a68bb407..51bc6b02c2 100644 --- a/ugsync-util/pom.xml +++ b/ugsync-util/pom.xml @@ -28,6 +28,7 @@ User Group Synchronizer Util true + false @@ -77,5 +78,4 @@ - diff --git a/ugsync/ldapconfigchecktool/ldapconfigcheck/pom.xml b/ugsync/ldapconfigchecktool/ldapconfigcheck/pom.xml index fca9c9d708..7e79abe07c 100644 --- a/ugsync/ldapconfigchecktool/ldapconfigcheck/pom.xml +++ b/ugsync/ldapconfigchecktool/ldapconfigcheck/pom.xml @@ -29,6 +29,7 @@ Ldap configuration check tool true + false diff --git a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/AuthenticationCheck.java b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/AuthenticationCheck.java index bc15f685bd..772b1d6b6f 100644 --- a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/AuthenticationCheck.java +++ b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/AuthenticationCheck.java @@ -38,16 +38,16 @@ import java.util.List; public class AuthenticationCheck { - private String ldapUrl; - private String adDomain; - private String userDnPattern; - private String roleAttribute; - private String groupSearchBase; - private String groupSearchFilter; - private PrintStream logFile; - private PrintStream ambariProps; - private PrintStream installProps; - private String authMethod = "NONE"; + private final String ldapUrl; + private final String adDomain; + private final String userDnPattern; + private final String roleAttribute; + private final String groupSearchBase; + private final String groupSearchFilter; + private final PrintStream logFile; + private final PrintStream ambariProps; + private final PrintStream installProps; + private final String authMethod; public AuthenticationCheck(String ldapUrl, UserSync userSyncObj, PrintStream logFile, PrintStream ambariProps, PrintStream installProps) { this.logFile = logFile; @@ -59,6 +59,7 @@ public AuthenticationCheck(String ldapUrl, UserSync userSyncObj, PrintStream log } else { authMethod = "LDAP"; } + this.ldapUrl = ldapUrl; adDomain = userSyncObj.getSearchBase(); userDnPattern = userSyncObj.getUserNameAttribute() + "={0}," + userSyncObj.getUserSearchBase(); @@ -70,6 +71,7 @@ public AuthenticationCheck(String ldapUrl, UserSync userSyncObj, PrintStream log public void discoverAuthProperties() { ambariProps.println("\n# Possible values for authentication properties:"); installProps.println("\n# Possible values for authentication properties:"); + if (authMethod.equalsIgnoreCase("AD")) { installProps.println("xa_ldap_ad_url=" + ldapUrl); installProps.println("xa_ldap_ad_domain=" + adDomain); @@ -98,11 +100,13 @@ public boolean isAuthenticated(String ldapUrl, String bindDn, String bindPasswor boolean isAuthenticated = false; //Verify Authentication Authentication authentication; + if (authMethod.equalsIgnoreCase("AD")) { authentication = getADBindAuthentication(ldapUrl, bindDn, bindPassword, userName, userPassword); } else { authentication = getLdapBindAuthentication(ldapUrl, bindDn, bindPassword, userName, userPassword); } + if (authentication != null) { isAuthenticated = authentication.isAuthenticated(); } diff --git a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/CommandLineOptions.java b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/CommandLineOptions.java index b137aef0cc..a188c8b069 100644 --- a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/CommandLineOptions.java +++ b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/CommandLineOptions.java @@ -29,20 +29,21 @@ import java.io.Console; public class CommandLineOptions { - private String[] args; + private final Options options = new Options(); + private final String[] args; + private String input; private String output; private String discoverProperties; private String retrieveValues; - private String ldapUrl = ""; - private String bindDn = ""; - private String bindPassword = ""; - private String userSearchBase = ""; - private String userSearchFilter = ""; - private String authUser = ""; - private String authPass = ""; - private Options options = new Options(); - private boolean isAuthEnabled = true; + private String ldapUrl = ""; + private String bindDn = ""; + private String bindPassword = ""; + private String userSearchBase = ""; + private String userSearchFilter = ""; + private String authUser = ""; + private String authPass = ""; + private boolean isAuthEnabled = true; public CommandLineOptions(String[] args) { this.args = args; @@ -60,6 +61,7 @@ public void parse() { CommandLineParser parser = new BasicParser(); try { CommandLine cmd = parser.parse(options, args); + if (cmd.hasOption("p")) { bindPassword = cmd.getOptionValue("p"); if (bindPassword.trim().isEmpty()) { @@ -83,6 +85,7 @@ public void parse() { if (cmd.hasOption("d")) { discoverProperties = cmd.getOptionValue("d"); + if (discoverProperties == null || (!discoverProperties.equalsIgnoreCase("all") && !discoverProperties.equalsIgnoreCase("users") && !discoverProperties.equalsIgnoreCase("groups"))) { System.out.println("Unsupported value for option d"); @@ -189,8 +192,8 @@ private void readCLI() { System.out.println("Please enter valid ldap url."); repeat = true; } - } - while (repeat == true); + } while (repeat); + do { repeat = false; System.out.print("Bind DN [cn=admin,ou=users,dc=example,dc=com]: "); @@ -199,19 +202,19 @@ private void readCLI() { System.out.println("Please enter valid bindDn."); repeat = true; } - } - while (repeat == true); + } while (repeat); + do { repeat = false; System.out.print("Bind Password: "); char[] password = console.readPassword(); bindPassword = String.valueOf(password); - if (bindPassword == null || bindPassword.isEmpty()) { + if (bindPassword.isEmpty()) { System.out.println("Bind Password can't be empty."); repeat = true; } - } - while (repeat == true); + } while (repeat); + System.out.print("User Search Base [ou=users,dc=example,dc=com]: "); userSearchBase = console.readLine(); System.out.print("User Search Filter [cn=user1]: "); @@ -226,19 +229,18 @@ private void readCLI() { System.out.println("Sample Authentication user must not be empty!"); repeat = true; } - } - while (repeat == true); + } while (repeat); + do { repeat = false; System.out.print("Sample Authentication Password: "); char[] password = console.readPassword(); authPass = String.valueOf(password); - if (authPass == null || authPass.isEmpty()) { + if (authPass.isEmpty()) { System.out.println("Sample Authentication password must not be empty!"); repeat = true; } - } - while (repeat == true); + } while (repeat); } } } diff --git a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfig.java b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfig.java index 9165434c18..1ddd656d1c 100644 --- a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfig.java +++ b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfig.java @@ -37,6 +37,7 @@ public class LdapConfig { public static final String CONFIG_FILE = "input.properties"; public static final String UGSYNC_NONE_CASE_CONVERSION_VALUE = "none"; public static final String UGSYNC_LOWER_CASE_CONVERSION_VALUE = "lower"; + private static final String LGSYNC_LDAP_URL = "ranger.usersync.ldap.url"; private static final String LGSYNC_LDAP_BIND_DN = "ranger.usersync.ldap.binddn"; private static final String LGSYNC_LDAP_BIND_PASSWORD = "ranger.usersync.ldap.ldapbindpassword"; @@ -53,18 +54,18 @@ public class LdapConfig { private static final String UGSYNC_GROUPNAME_CASE_CONVERSION_PARAM = "ranger.usersync.ldap.groupname.caseconversion"; private static final String DEFAULT_UGSYNC_USERNAME_CASE_CONVERSION_VALUE = UGSYNC_NONE_CASE_CONVERSION_VALUE; private static final String DEFAULT_UGSYNC_GROUPNAME_CASE_CONVERSION_VALUE = UGSYNC_NONE_CASE_CONVERSION_VALUE; - private static final String LGSYNC_PAGED_RESULTS_ENABLED = "ranger.usersync.pagedresultsenabled"; - private static final String LGSYNC_PAGED_RESULTS_SIZE = "ranger.usersync.pagedresultssize"; - private static final String LGSYNC_GROUP_SEARCH_ENABLED = "ranger.usersync.group.searchenabled"; - private static final String LGSYNC_GROUP_SEARCH_BASE = "ranger.usersync.group.searchbase"; - private static final String LGSYNC_GROUP_SEARCH_SCOPE = "ranger.usersync.group.searchscope"; - private static final String LGSYNC_GROUP_OBJECT_CLASS = "ranger.usersync.group.objectclass"; - private static final String LGSYNC_GROUP_SEARCH_FILTER = "ranger.usersync.group.searchfilter"; - private static final String LGSYNC_GROUP_NAME_ATTRIBUTE = "ranger.usersync.group.nameattribute"; - private static final String LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = "ranger.usersync.group.memberattributename"; - private static final int DEFAULT_LGSYNC_PAGED_RESULTS_SIZE = 500; - private static final boolean DEFAULT_LGSYNC_PAGED_RESULTS_ENABLED = true; - private static final boolean DEFAULT_LGSYNC_GROUP_SEARCH_ENABLED = false; + private static final String LGSYNC_PAGED_RESULTS_ENABLED = "ranger.usersync.pagedresultsenabled"; + private static final String LGSYNC_PAGED_RESULTS_SIZE = "ranger.usersync.pagedresultssize"; + private static final String LGSYNC_GROUP_SEARCH_ENABLED = "ranger.usersync.group.searchenabled"; + private static final String LGSYNC_GROUP_SEARCH_BASE = "ranger.usersync.group.searchbase"; + private static final String LGSYNC_GROUP_SEARCH_SCOPE = "ranger.usersync.group.searchscope"; + private static final String LGSYNC_GROUP_OBJECT_CLASS = "ranger.usersync.group.objectclass"; + private static final String LGSYNC_GROUP_SEARCH_FILTER = "ranger.usersync.group.searchfilter"; + private static final String LGSYNC_GROUP_NAME_ATTRIBUTE = "ranger.usersync.group.nameattribute"; + private static final String LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = "ranger.usersync.group.memberattributename"; + private static final int DEFAULT_LGSYNC_PAGED_RESULTS_SIZE = 500; + private static final boolean DEFAULT_LGSYNC_PAGED_RESULTS_ENABLED = true; + private static final boolean DEFAULT_LGSYNC_GROUP_SEARCH_ENABLED = false; //Authentication related properties private static final String AUTHENTICATION_METHOD = "ranger.authentication.method"; @@ -75,13 +76,14 @@ public class LdapConfig { private static final String GROUP_SEARCH_FILTER = "ranger.ldap.group.searchfilter"; private static final String AUTH_USERNAME = "ranger.admin.auth.sampleuser"; private static final String AUTH_PASSWORD = "ranger.admin.auth.samplepassword"; - private Properties prop = new Properties(); + + private final Properties prop = new Properties(); public LdapConfig(String configFile, String bindPasswd) { init(configFile, bindPasswd); } - public String getLdapUrl() throws Throwable { + public String getLdapUrl() { String val = prop.getProperty(LGSYNC_LDAP_URL); if (val == null || val.trim().isEmpty()) { throw new NullArgumentException(LGSYNC_LDAP_URL); @@ -89,7 +91,7 @@ public String getLdapUrl() throws Throwable { return val; } - public String getLdapBindDn() throws Throwable { + public String getLdapBindDn() { String val = prop.getProperty(LGSYNC_LDAP_BIND_DN); if (val == null || val.trim().isEmpty()) { throw new NullArgumentException(LGSYNC_LDAP_BIND_DN); @@ -99,9 +101,6 @@ public String getLdapBindDn() throws Throwable { public String getLdapBindPassword() { //update credential from keystore - if (prop == null) { - return null; - } return prop.getProperty(LGSYNC_LDAP_BIND_PASSWORD); } @@ -173,7 +172,7 @@ public boolean isPagedResultsEnabled() { if (val == null || val.trim().isEmpty()) { pagedResultsEnabled = DEFAULT_LGSYNC_PAGED_RESULTS_ENABLED; } else { - pagedResultsEnabled = Boolean.valueOf(val); + pagedResultsEnabled = Boolean.parseBoolean(val); } return pagedResultsEnabled; } @@ -198,7 +197,7 @@ public boolean isGroupSearchEnabled() { if (val == null || val.trim().isEmpty()) { groupSearchEnabled = DEFAULT_LGSYNC_GROUP_SEARCH_ENABLED; } else { - groupSearchEnabled = Boolean.valueOf(val); + groupSearchEnabled = Boolean.parseBoolean(val); } return groupSearchEnabled; } @@ -311,7 +310,7 @@ private void readConfigFile(String fileName) { try { in.close(); } catch (IOException ioe) { - System.out.println(ioe); + System.out.println(ioe.getMessage()); } } } @@ -321,7 +320,7 @@ private void readConfigFile(String fileName) { } private InputStream getFileInputStream(String path) throws FileNotFoundException { - InputStream ret = null; + InputStream ret; File f = new File(path); if (f.exists()) { diff --git a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfigCheckMain.java b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfigCheckMain.java index 9cf69e9fa6..8043ea62e0 100644 --- a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfigCheckMain.java +++ b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfigCheckMain.java @@ -28,7 +28,6 @@ import javax.naming.ldap.LdapContext; import javax.naming.ldap.PagedResultsControl; -import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; @@ -63,9 +62,9 @@ public static void main(String[] args) { LdapContext ldapContext = null; try { - logFile = new PrintStream(new File(outputDir + LOG_FILE)); - ambariProps = new PrintStream(new File(outputDir + AMBARI_PROPERTIES)); - installProps = new PrintStream(new File(outputDir + INSTALL_PROPERTIES)); + logFile = new PrintStream(outputDir + LOG_FILE); + ambariProps = new PrintStream(outputDir + AMBARI_PROPERTIES); + installProps = new PrintStream(outputDir + INSTALL_PROPERTIES); UserSync userSyncObj = new UserSync(config, logFile, ambariProps, installProps); @@ -148,7 +147,7 @@ private static void findAllUserSyncProperties(LdapContext ldapContext, UserSync userSyncObj.findGroupProperties(ldapContext); } - private static void authenticate(UserSync userSyncObj, LdapConfig config, PrintStream logFile, PrintStream ambariProps, PrintStream installProps) throws Throwable { + private static void authenticate(UserSync userSyncObj, LdapConfig config, PrintStream logFile, PrintStream ambariProps, PrintStream installProps) { AuthenticationCheck auth = new AuthenticationCheck(config.getLdapUrl(), userSyncObj, logFile, ambariProps, installProps); auth.discoverAuthProperties(); diff --git a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/UserSync.java b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/UserSync.java index 01b2f008f5..d0ede31330 100644 --- a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/UserSync.java +++ b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/UserSync.java @@ -36,25 +36,25 @@ import java.util.Set; public class UserSync { - private static String[] userNameAttrValues = { + private static final String[] userNameAttrValues = { "sAMAccountName", "uid", "cn" }; /* Not needed as this is read from the second occurrence of objectClass attribute from user entry */ - private static String[] userObjClassValues = { + private static final String[] userObjClassValues = { "person", "posixAccount" }; - private static String[] userGroupMemAttrValues = { + private static final String[] userGroupMemAttrValues = { "memberOf", "ismemberOf" }; - private static String[] groupObjectClassValues = { + private static final String[] groupObjectClassValues = { "group", "groupOfNames", "posixGroup" }; - private static String[] groupNameAttrValues = { + private static final String[] groupNameAttrValues = { "distinguishedName", "cn" }; - private static String[] groupMemAttrValues = { + private static final String[] groupMemAttrValues = { "member", "memberUid" }; @@ -69,11 +69,11 @@ public class UserSync { private String userSearchBase; private String userSearchFilter; private String searchBase; - private String groupName; - private PrintStream logFile; - private PrintStream ambariProps; - private PrintStream installProps; - private LdapConfig config; + private String groupName; + private final PrintStream logFile; + private final PrintStream ambariProps; + private final PrintStream installProps; + private final LdapConfig config; public UserSync(LdapConfig config, PrintStream logFile, PrintStream ambariProps, PrintStream installProps) { this.config = config; @@ -131,7 +131,7 @@ public void getAllUsers(LdapContext ldapContext) throws Throwable { int noOfUsers = 0; Attribute userNameAttr = null; //String groupName = null; - Attribute groupMemberAttr = null; + Attribute groupMemberAttr; NamingEnumeration userSearchResultEnum = null; SearchControls userSearchControls = new SearchControls(); userSearchControls.setSearchScope(config.getUserSearchScope()); @@ -143,9 +143,8 @@ public void getAllUsers(LdapContext ldapContext) throws Throwable { userSearchAttributes.add(userGroupMemberName); } - if (userSearchAttributes.size() > 0) { - userSearchControls.setReturningAttributes(userSearchAttributes.toArray( - new String[userSearchAttributes.size()])); + if (!userSearchAttributes.isEmpty()) { + userSearchControls.setReturningAttributes(userSearchAttributes.toArray(new String[userSearchAttributes.size()])); } else { userSearchControls.setReturningAttributes(new java.lang.String[] {"*", "+"}); } @@ -182,17 +181,17 @@ public void getAllUsers(LdapContext ldapContext) throws Throwable { } if (userNameAttribute == null || userNameAttribute.isEmpty()) { - for (int i = 0; i < userNameAttrValues.length; i++) { - userNameAttr = attributes.get(userNameAttrValues[i]); + for (String userNameAttrValue : userNameAttrValues) { + userNameAttr = attributes.get(userNameAttrValue); if (userNameAttr != null) { - userNameAttribute = userNameAttrValues[i]; + userNameAttribute = userNameAttrValue; break; } } if (userNameAttr == null) { logFile.print("WARN: Failed to find any of ( "); - for (int i = 0; i < userNameAttrValues.length; i++) { - logFile.print(userNameAttrValues[i] + " "); + for (String userNameAttrValue : userNameAttrValues) { + logFile.print(userNameAttrValue + " "); } logFile.println(") for entry " + userEntry.getNameInNamespace()); continue; @@ -316,8 +315,7 @@ public void getAllGroups(LdapContext ldapContext) throws Throwable { } try { - groupSearchResultEnum = ldapContext.search(groupSearchBase, extendedGroupSearchFilter, - groupSearchControls); + groupSearchResultEnum = ldapContext.search(groupSearchBase, extendedGroupSearchFilter, groupSearchControls); logFile.println("\nINFO: First 20 Groups and associated Users are:"); @@ -326,7 +324,9 @@ public void getAllGroups(LdapContext ldapContext) throws Throwable { if (groupEntry == null) { continue; } + Attributes groupAttributes = groupEntry.getAttributes(); + if (groupAttributes == null) { logFile.println("WARN: Attributes missing for entry " + groupEntry.getNameInNamespace()); continue; @@ -442,12 +442,13 @@ private void findBasicUserProperties(LdapContext ldapContext, boolean isOutputNe } try { - userSearchResultEnum = ldapContext.search(userSBase, - userSFilter, userSearchControls); + userSearchResultEnum = ldapContext.search(userSBase, userSFilter, userSearchControls); while (userSearchResultEnum.hasMore()) { + if (noOfUsers >= 5) { break; } + final SearchResult userEntry = userSearchResultEnum.next(); if (userEntry == null) { @@ -462,17 +463,18 @@ private void findBasicUserProperties(LdapContext ldapContext, boolean isOutputNe } if (userNameAttribute == null || userNameAttribute.isEmpty()) { - for (int i = 0; i < userNameAttrValues.length; i++) { - userNameAttr = attributes.get(userNameAttrValues[i]); + for (String nameAttrValue : userNameAttrValues) { + userNameAttr = attributes.get(nameAttrValue); if (userNameAttr != null) { - userNameAttribute = userNameAttrValues[i]; + userNameAttribute = nameAttrValue; break; } } + if (userNameAttr == null) { logFile.print("WARN: Failed to find any of ( "); - for (int i = 0; i < userNameAttrValues.length; i++) { - logFile.print(userNameAttrValues[i] + " "); + for (String userNameAttrValue : userNameAttrValues) { + logFile.print(userNameAttrValue + " "); } logFile.println(") for entry " + userEntry.getNameInNamespace()); continue; @@ -499,15 +501,14 @@ private void findBasicUserProperties(LdapContext ldapContext, boolean isOutputNe userObjClass = userObjClassEnum.next().toString(); if (userObjClassName == null || userObjClassName.isEmpty()) { if (userObjClass != null) { - for (int i = 0; i < userObjClassValues.length; i++) { - if (userObjClass.equalsIgnoreCase(userObjClassValues[i])) { + for (String userObjClassValue : userObjClassValues) { + if (userObjClass.equalsIgnoreCase(userObjClassValue)) { userObjClassName = userObjClass; break; } } } else { logFile.println("WARN: Failed to find objectClass attribute for " + userName); - //continue; } } } @@ -516,10 +517,10 @@ private void findBasicUserProperties(LdapContext ldapContext, boolean isOutputNe userObjClassName = userObjClass; } - for (int i = 0; i < userGroupMemAttrValues.length; i++) { - groupMemberAttr = attributes.get(userGroupMemAttrValues[i]); + for (String userGroupMemAttrValue : userGroupMemAttrValues) { + groupMemberAttr = attributes.get(userGroupMemAttrValue); if (groupMemberAttr != null) { - userGroupMemberName = userGroupMemAttrValues[i]; + userGroupMemberName = userGroupMemAttrValue; groupName = groupMemberAttr.get(0).toString(); break; } @@ -528,7 +529,7 @@ private void findBasicUserProperties(LdapContext ldapContext, boolean isOutputNe noOfUsers++; } } catch (NamingException ne) { - String msg = "Exception occured while discovering basic user properties:\n" + + String msg = "Exception occurred while discovering basic user properties:\n" + "ranger.usersync.ldap.user.nameattribute\n" + "ranger.usersync.ldap.user.objectclass\n" + "ranger.usersync.ldap.user.groupnameattribute\n"; @@ -557,7 +558,7 @@ private void findBasicUserProperties(LdapContext ldapContext, boolean isOutputNe userSearchResultEnum.close(); } } catch (NamingException ne) { - throw new Exception("Exception occured while closing user search result: " + ne); + logFile.println("ERROR: Exception occurred while closing user search result: " + ne.getMessage()); } } } @@ -625,10 +626,10 @@ private void findAdvUserProperties(LdapContext ldapContext, boolean isOutputNeed } Integer ouOccrs = ouOccurences.get(dnValue); if (ouOccrs == null) { - ouOccrs = Integer.valueOf(0); + ouOccrs = 0; } - int val = ouOccrs.intValue(); - ouOccrs = Integer.valueOf(++val); + int val = ouOccrs; + ouOccrs = ++val; ouOccurences.put(dnValue, ouOccrs); noOfUsers++; } @@ -637,7 +638,7 @@ private void findAdvUserProperties(LdapContext ldapContext, boolean isOutputNeed Set keys = ouOccurences.keySet(); int maxOUOccr = 0; for (String key : keys) { - int ouOccurVal = ouOccurences.get(key).intValue(); + int ouOccurVal = ouOccurences.get(key); logFile.println("INFO: No. of users from " + key + " = " + ouOccurVal); if (ouOccurVal > maxOUOccr) { maxOUOccr = ouOccurVal; @@ -718,8 +719,8 @@ private void findBasicGroupProperties(LdapContext ldapContext) throws Throwable NamingEnumeration groupObjClassEnum = groupObjClassAttr.getAll(); while (groupObjClassEnum.hasMore()) { String groupObjClassStr = groupObjClassEnum.next().toString(); - for (int i = 0; i < groupObjectClassValues.length; i++) { - if (groupObjClassStr.equalsIgnoreCase(groupObjectClassValues[i])) { + for (String groupObjectClassValue : groupObjectClassValues) { + if (groupObjClassStr.equalsIgnoreCase(groupObjectClassValue)) { groupObjClassName = groupObjClassStr; break; } @@ -731,19 +732,19 @@ private void findBasicGroupProperties(LdapContext ldapContext) throws Throwable } if (groupNameAttrName == null || groupNameAttrName.isEmpty()) { - for (int i = 0; i < groupNameAttrValues.length; i++) { - groupNameAttr = groupAttributes.get(groupNameAttrValues[i]); + for (String groupNameAttrValue : groupNameAttrValues) { + groupNameAttr = groupAttributes.get(groupNameAttrValue); if (groupNameAttr != null) { - groupNameAttrName = groupNameAttrValues[i]; + groupNameAttrName = groupNameAttrValue; break; } } } - for (int i = 0; i < groupMemAttrValues.length; i++) { - groupMemberAttr = groupAttributes.get(groupMemAttrValues[i]); + for (String groupMemAttrValue : groupMemAttrValues) { + groupMemberAttr = groupAttributes.get(groupMemAttrValue); if (groupMemberAttr != null) { - groupMemberName = groupMemAttrValues[i]; + groupMemberName = groupMemAttrValue; break; } } @@ -821,10 +822,10 @@ private void findAdvGroupProperties(LdapContext ldapContext) throws Throwable { Integer ouOccrs = ouOccurences.get(dnValue); if (ouOccrs == null) { - ouOccrs = Integer.valueOf(0); + ouOccrs = 0; } - int val = ouOccrs.intValue(); - ouOccrs = Integer.valueOf(++val); + int val = ouOccrs; + ouOccrs = ++val; ouOccurences.put(dnValue, ouOccrs); noOfGroups++; @@ -834,7 +835,7 @@ private void findAdvGroupProperties(LdapContext ldapContext) throws Throwable { Set keys = ouOccurences.keySet(); int maxOUOccr = 0; for (String key : keys) { - int ouOccurVal = ouOccurences.get(key).intValue(); + int ouOccurVal = ouOccurences.get(key); logFile.println("INFO: No. of groups from " + key + " = " + ouOccurVal); if (ouOccurVal > maxOUOccr) { maxOUOccr = ouOccurVal; diff --git a/ugsync/pom.xml b/ugsync/pom.xml index eaef820d5f..161b36947f 100644 --- a/ugsync/pom.xml +++ b/ugsync/pom.xml @@ -29,6 +29,7 @@ User, Group sync from sources such as Unix and LDAP true + false diff --git a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/CustomSSLSocketFactory.java b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/CustomSSLSocketFactory.java index 89035a3435..d7f71cb494 100644 --- a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/CustomSSLSocketFactory.java +++ b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/CustomSSLSocketFactory.java @@ -38,32 +38,30 @@ import java.io.InputStream; import java.net.InetAddress; import java.net.Socket; -import java.net.UnknownHostException; import java.security.KeyStore; import java.security.SecureRandom; public class CustomSSLSocketFactory extends SSLSocketFactory { private static final Logger LOG = LoggerFactory.getLogger(CustomSSLSocketFactory.class); + private SSLSocketFactory sockFactory; - private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); public CustomSSLSocketFactory() { - SSLContext sslContext = null; - String keyStoreFile = config.getSSLKeyStorePath(); - String keyStoreFilepwd = config.getSSLKeyStorePathPassword(); - String trustStoreFile = config.getSSLTrustStorePath(); - String trustStoreFilepwd = config.getSSLTrustStorePathPassword(); - String keyStoreType = config.getSSLKeyStoreType(); - String trustStoreType = config.getSSLTrustStoreType(); + SSLContext sslContext; + UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); + String keyStoreFile = config.getSSLKeyStorePath(); + String keyStoreFilepwd = config.getSSLKeyStorePathPassword(); + String trustStoreFile = config.getSSLTrustStorePath(); + String trustStoreFilepwd = config.getSSLTrustStorePathPassword(); + String keyStoreType = config.getSSLKeyStoreType(); + String trustStoreType = config.getSSLTrustStoreType(); try { KeyManager[] kmList = null; TrustManager[] tmList = null; if (keyStoreFile != null && keyStoreFilepwd != null) { KeyStore keyStore = KeyStore.getInstance(keyStoreType); - InputStream in = null; - try { - in = getFileInputStream(keyStoreFile); + try (InputStream in = getFileInputStream(keyStoreFile)) { if (in == null) { LOG.error("Unable to obtain keystore from file [{}]", keyStoreFile); return; @@ -72,30 +70,20 @@ public CustomSSLSocketFactory() { KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray()); kmList = keyManagerFactory.getKeyManagers(); - } finally { - if (in != null) { - in.close(); - } } } if (trustStoreFile != null && trustStoreFilepwd != null) { KeyStore trustStore = KeyStore.getInstance(trustStoreType); - InputStream in = null; - try { - in = getFileInputStream(trustStoreFile); + try (InputStream in = getFileInputStream(trustStoreFile)) { if (in == null) { - LOG.error("Unable to obtain keystore from file [{}]", trustStoreFile); + LOG.error("Unable to obtain truststore from file [{}]", trustStoreFile); return; } trustStore.load(in, trustStoreFilepwd.toCharArray()); TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); trustManagerFactory.init(trustStore); tmList = trustManagerFactory.getTrustManagers(); - } finally { - if (in != null) { - in.close(); - } } } @@ -128,12 +116,12 @@ public Socket createSocket(Socket socket, String host, int port, boolean bln) th } @Override - public Socket createSocket(String host, int port) throws IOException, UnknownHostException { + public Socket createSocket(String host, int port) throws IOException { return sockFactory.createSocket(host, port); } @Override - public Socket createSocket(String host, int port, InetAddress localHost, int localPort) throws IOException, UnknownHostException { + public Socket createSocket(String host, int port, InetAddress localHost, int localPort) throws IOException { return sockFactory.createSocket(host, port, localHost, localPort); } @@ -148,7 +136,7 @@ public Socket createSocket(InetAddress address, int port, InetAddress localHost, } private InputStream getFileInputStream(String path) throws FileNotFoundException { - InputStream ret = null; + InputStream ret; File f = new File(path); diff --git a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java index 17992d728b..a517ff9d0d 100644 --- a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java @@ -56,7 +56,6 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; @@ -66,6 +65,7 @@ public class LdapUserGroupBuilder implements UserGroupSource { private static final Logger LOG = LoggerFactory.getLogger(LdapUserGroupBuilder.class); + private static final String DATA_TYPE_BYTEARRAY = "byte[]"; private static final String DATE_FORMAT = "yyyyMMddHHmmss"; private static final String MEMBER_OF_ATTR = "memberof="; @@ -85,9 +85,9 @@ public class LdapUserGroupBuilder implements UserGroupSource { private int userSearchScope; private int groupSearchScope; - private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); + private final UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); /* for OpenLdap modifyTimestamp */ - private String deltaSyncUserTimeStamp; + private String deltaSyncUserTimeStamp; private String deltaSyncGroupTimeStamp; /* ******************************** */ private String ldapUrl; @@ -178,12 +178,15 @@ public void updateSink(UserGroupSink sink) throws Throwable { computeDeletes = true; LOG.debug("Compute deleted users/groups is enabled for this sync cycle"); } + if (config.isUserSyncDeletesEnabled()) { deleteCycles++; } + if (groupSearchEnabled) { highestdeltaSyncGroupTime = getGroups(computeDeletes); } + if (userSearchEnabled) { LOG.info("Performing user search to retrieve users from AD/LDAP"); highestdeltaSyncUserTime = getUsers(computeDeletes); @@ -199,11 +202,10 @@ public void updateSink(UserGroupSink sink) throws Throwable { LOG.info("Completed group hierarchy computation"); } - Iterator groupUserTableIterator = groupUserTable.rowKeySet().iterator(); - while (groupUserTableIterator.hasNext()) { - String groupName = groupUserTableIterator.next(); + for (String groupName : groupUserTable.rowKeySet()) { Map groupUsersMap = groupUserTable.row(groupName); - Set userSet = new HashSet(); + Set userSet = new HashSet<>(); + for (Map.Entry entry : groupUsersMap.entrySet()) { if (sourceUsers.containsKey(entry.getValue())) { userSet.add(entry.getValue()); @@ -245,7 +247,7 @@ public void updateSink(UserGroupSink sink) throws Throwable { try { sink.postUserGroupAuditInfo(ugsyncAuditInfo); } catch (Throwable t) { - LOG.error("sink.postUserGroupAuditInfo failed with exception: " + t.getMessage()); + LOG.error("sink.postUserGroupAuditInfo failed with exception: {}", t.getMessage()); } } @@ -282,6 +284,7 @@ private void createLdapContext() throws Throwable { env.put("java.naming.ldap.attributes.binary", otherGroupAttribute); } } + ldapContext = new InitialLdapContext(env, null); if (!ldapUrl.startsWith("ldaps")) { if (config.isStartTlsEnabled()) { @@ -303,6 +306,7 @@ private void createLdapContext() throws Throwable { private void setConfig() throws Throwable { LOG.info("LdapUserGroupBuilder initialization started"); + groupSearchFirstEnabled = true; currentSyncSource = config.getCurrentSyncSource(); userSearchEnabled = config.isUserSearchEnabled(); @@ -319,27 +323,25 @@ private void setConfig() throws Throwable { userSearchFilter = config.getUserSearchFilter(); userNameAttribute = config.getUserNameAttribute(); userCloudIdAttribute = config.getUserCloudIdAttribute(); + userGroupNameAttributeSet = config.getUserGroupNameAttributeSet(); + otherUserAttributes = config.getOtherUserAttributes(); + + Set userSearchAttributes = new HashSet<>(); - Set userSearchAttributes = new HashSet(); userSearchAttributes.add(userNameAttribute); - userGroupNameAttributeSet = config.getUserGroupNameAttributeSet(); - for (String useGroupNameAttribute : userGroupNameAttributeSet) { - userSearchAttributes.add(useGroupNameAttribute); - } + userSearchAttributes.addAll(userGroupNameAttributeSet); userSearchAttributes.add(userCloudIdAttribute); - otherUserAttributes = config.getOtherUserAttributes(); - for (String otherUserAttribute : otherUserAttributes) { - userSearchAttributes.add(otherUserAttribute); - } + userSearchAttributes.addAll(otherUserAttributes); userSearchAttributes.add("uSNChanged"); userSearchAttributes.add("modifytimestamp"); + userSearchControls = new SearchControls(); + userSearchControls.setSearchScope(userSearchScope); userSearchControls.setReturningAttributes(userSearchAttributes.toArray(new String[userSearchAttributes.size()])); pagedResultsEnabled = config.isPagedResultsEnabled(); pagedResultsSize = config.getPagedResultsSize(); - groupSearchBase = config.getGroupSearchBase().split(";"); groupSearchScope = config.getGroupSearchScope(); groupObjectClass = config.getGroupObjectClass(); @@ -348,39 +350,38 @@ private void setConfig() throws Throwable { groupNameAttribute = config.getGroupNameAttribute(); groupCloudIdAttribute = config.getGroupCloudIdAttribute(); groupHierarchyLevels = config.getGroupHierarchyLevels(); - extendedGroupSearchFilter = "(&" + extendedGroupSearchFilter + "(|(" + groupMemberAttributeName + "={0})(" + groupMemberAttributeName + "={1})))"; groupSearchControls = new SearchControls(); groupSearchControls.setSearchScope(groupSearchScope); - Set groupSearchAttributes = new HashSet(); + Set groupSearchAttributes = new HashSet<>(); groupSearchAttributes.add(groupNameAttribute); groupSearchAttributes.add(groupCloudIdAttribute); groupSearchAttributes.add(groupMemberAttributeName); groupSearchAttributes.add("uSNChanged"); groupSearchAttributes.add("modifytimestamp"); otherGroupAttributes = config.getOtherGroupAttributes(); - for (String otherGroupAttribute : otherGroupAttributes) { - groupSearchAttributes.add(otherGroupAttribute); - } + groupSearchAttributes.addAll(otherGroupAttributes); groupSearchControls.setReturningAttributes(groupSearchAttributes.toArray(new String[groupSearchAttributes.size()])); if (StringUtils.isEmpty(userSearchFilter)) { groupNameSet = config.getGroupNameSet(); String computedSearchFilter = ""; for (String groupName : groupNameSet) { - if (LOG.isDebugEnabled()) { - LOG.debug("groupName = {}", groupName); - } + LOG.debug("groupName = {}", groupName); + if (!groupName.startsWith(MEMBER_OF_ATTR) && !groupName.startsWith(GROUP_NAME_ATTRIBUTE)) { LOG.info("Ignoring unsupported format for {}", groupName); continue; } + String searchFilter = groupName; + if (groupName.startsWith(MEMBER_OF_ATTR)) { searchFilter = groupName.substring(MEMBER_OF_ATTR.length()); } + searchFilter = getFirstRDN(searchFilter); computedSearchFilter += getDNForMemberOf(searchFilter); } @@ -444,7 +445,7 @@ private long getUsers(boolean computeDeletes) throws Throwable { ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, Control.NONCRITICAL)}); } DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); - if (groupUserTable.rowKeySet().size() != 0 || !config.isDeltaSyncEnabled() || (computeDeletes)) { + if (!groupUserTable.rowKeySet().isEmpty() || !config.isDeltaSyncEnabled() || (computeDeletes)) { // Fix RANGER-1957: Perform full sync when there are updates to the groups or when incremental sync is not enabled deltaSyncUserTime = 0; deltaSyncUserTimeStamp = dateFormat.format(new Date(0)); @@ -479,30 +480,13 @@ private long getUsers(boolean computeDeletes) throws Throwable { // searchResults contains all the user entries final SearchResult userEntry = userSearchResultEnum.next(); - if (userEntry == null) { - LOG.info("userEntry null, skipping sync for the entry"); + if (!isSearchResultValid(userEntry, false)) { continue; } Attributes attributes = userEntry.getAttributes(); - if (attributes == null) { - LOG.info("attributes missing for entry {}, skipping sync", userEntry.getNameInNamespace()); - continue; - } - - Attribute userNameAttr = attributes.get(userNameAttribute); - if (userNameAttr == null) { - LOG.info("{} missing for entry {}, skipping sync", userNameAttribute, userEntry.getNameInNamespace()); - continue; - } - - String userFullName = userEntry.getNameInNamespace(); - String userName = (String) userNameAttr.get(); - - if (userName == null || userName.trim().isEmpty()) { - LOG.info("{} empty for entry {}, skipping sync", userNameAttribute, userEntry.getNameInNamespace()); - continue; - } + String userName = (String) attributes.get(userNameAttribute).get(); + String userFullName = userEntry.getNameInNamespace(); Attribute timeStampAttr = attributes.get("uSNChanged"); if (timeStampAttr != null) { @@ -534,9 +518,7 @@ private long getUsers(boolean computeDeletes) throws Throwable { NamingEnumeration groupEnum = userGroupfAttribute.getAll(); while (groupEnum.hasMore()) { String groupDN = (String) groupEnum.next(); - if (LOG.isDebugEnabled()) { - LOG.debug("Adding {} to {}", groupDN, userName); - } + LOG.debug("Adding {} to {}", groupDN, userName); Map groupAttrMap = new HashMap<>(); String groupName = getShortName(groupDN); groupAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, groupName); @@ -544,9 +526,7 @@ private long getUsers(boolean computeDeletes) throws Throwable { groupAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); groupAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl()); sourceGroups.put(groupDN, groupAttrMap); - if (LOG.isDebugEnabled()) { - LOG.debug("As groupsearch is disabled, adding group {} from user memberof attribute for user {}", groupName, userName); - } + LOG.debug("As groupsearch is disabled, adding group {} from user memberof attribute for user {}", groupName, userName); groupUserTable.put(groupDN, userFullName, userFullName); } } @@ -562,6 +542,7 @@ private long getUsers(boolean computeDeletes) throws Throwable { if (userCloudIdAttr != null) { addToAttrMap(userAttrMap, "cloud_id", userCloudIdAttr, config.getUserCloudIdAttributeDataType()); } + for (String otherUserAttribute : otherUserAttributes) { if (attributes.get(otherUserAttribute) != null) { String attrType = config.getOtherUserAttributeDataType(otherUserAttribute); @@ -583,7 +564,7 @@ private long getUsers(boolean computeDeletes) throws Throwable { counter++; if (counter <= 2000) { - LOG.info("Updating user count: " + counter + ", userName: " + userName); + LOG.info("Updating user count: {}, userName: {}", counter, userName); if (counter == 2000) { LOG.info("===> 2000 user records have been synchronized so far. From now on, only a summary progress log will be written for every 100 users. To continue to see detailed log for every user, please enable Trace level logging. <==="); } @@ -599,13 +580,12 @@ private long getUsers(boolean computeDeletes) throws Throwable { // Examine the paged results control response Control[] controls = ldapContext.getResponseControls(); if (controls != null) { - for (int i = 0; i < controls.length; i++) { - if (controls[i] instanceof PagedResultsResponseControl) { - PagedResultsResponseControl prrc = - (PagedResultsResponseControl) controls[i]; + for (Control control : controls) { + if (control instanceof PagedResultsResponseControl) { + PagedResultsResponseControl prrc = (PagedResultsResponseControl) control; total = prrc.getResultSize(); if (total != 0) { - LOG.debug("END-OF-PAGE total : " + total); + LOG.debug("END-OF-PAGE total : {}", total); } else { LOG.debug("END-OF-PAGE total : unknown"); } @@ -617,7 +597,7 @@ private long getUsers(boolean computeDeletes) throws Throwable { } // Re-activate paged results if (pagedResultsEnabled) { - LOG.debug(String.format("Fetched paged results round: %s", ++paged)); + LOG.debug("Fetched paged results round: {}", ++paged); ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, cookie, Control.CRITICAL)}); } } @@ -733,7 +713,11 @@ private long getGroups(boolean computeDeletes) throws Throwable { Attribute groupMemberAttr = attributes.get(groupMemberAttributeName); int userCount = 0; if (groupMemberAttr == null || groupMemberAttr.size() <= 0) { - LOG.info("No members available for {}", gName); + try { + LOG.info("No members available for {}", gName); + } catch (Exception e) { + throw new RuntimeException(e); + } sourceGroupUsers.put(groupFullName, new HashSet<>()); continue; } @@ -755,9 +739,7 @@ private long getGroups(boolean computeDeletes) throws Throwable { userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl()); sourceUsers.put(originalUserFullName, userAttrMap); - if (LOG.isDebugEnabled()) { - LOG.debug("As usersearch is disabled, adding user {} from group member attribute for group {}", userName, gName); - } + LOG.debug("As usersearch is disabled, adding user {} from group member attribute for group {}", userName, gName); } groupUserTable.put(groupFullName, originalUserFullName, originalUserFullName); @@ -773,25 +755,19 @@ private long getGroups(boolean computeDeletes) throws Throwable { PagedResultsResponseControl prrc = (PagedResultsResponseControl) control; total = prrc.getResultSize(); if (total != 0) { - if (LOG.isDebugEnabled()) { - LOG.debug("END-OF-PAGE total: {}", total); - } + LOG.debug("END-OF-PAGE total: {}", total); } else { - if (LOG.isDebugEnabled()) { - LOG.debug("END-OF-PAGE total: unknown"); - } + LOG.debug("END-OF-PAGE total: unknown"); } cookie = prrc.getCookie(); } } } else { - if (LOG.isDebugEnabled()) { - LOG.debug("No controls were sent from the server"); - } + LOG.debug("No controls were sent from the server"); } // Re-activate paged results if (pagedResultsEnabled) { - LOG.debug(String.format("Fetched paged results round: %s", ++paged)); + LOG.debug("Fetched paged results round: {}", ++paged); ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, cookie, Control.CRITICAL)}); } } @@ -821,7 +797,7 @@ private long getGroups(boolean computeDeletes) throws Throwable { return highestdeltaSyncGroupTime; } - private void goUpGroupHierarchy(Set groups, int groupHierarchyLevels, String groupSName) throws InvalidNameException { + private void goUpGroupHierarchy(Set groups, int groupHierarchyLevels, String groupSName) { if (groupHierarchyLevels <= 0 || groups.isEmpty()) { return; } @@ -835,8 +811,11 @@ private void goUpGroupHierarchy(Set groups, int groupHierarchyLevels, St for (String member : allMembers) { if (!groupUserTable.containsRow(member)) { //Check if the member of a group is in turn a group LOG.info("Adding {} to {}", member, group); + String userSName = groupUserTable.get(groupSName, member); + LOG.info("Short name of {} = {}", member, userSName); + if (userSName != null) { groupUserTable.put(group, member, userSName); //Add users from the nested group to parent group } @@ -851,7 +830,7 @@ private void goUpGroupHierarchyLdap(Set groupDNs, int groupHierarchyLeve if (groupHierarchyLevels <= 0 || groupDNs.isEmpty()) { return; } - Set nextLevelGroups = new HashSet(); + Set nextLevelGroups = new HashSet<>(); NamingEnumeration groupSearchResultEnum = null; try { @@ -948,21 +927,15 @@ private void goUpGroupHierarchyLdap(Set groupDNs, int groupHierarchyLeve PagedResultsResponseControl prrc = (PagedResultsResponseControl) control; total = prrc.getResultSize(); if (total != 0) { - if (LOG.isDebugEnabled()) { - LOG.debug("END-OF-PAGE total : {}", total); - } + LOG.debug("END-OF-PAGE total : {}", total); } else { - if (LOG.isDebugEnabled()) { - LOG.debug("END-OF-PAGE total : unknown"); - } + LOG.debug("END-OF-PAGE total : unknown"); } cookie = prrc.getCookie(); } } } else { - if (LOG.isDebugEnabled()) { - LOG.debug("No controls were sent from the server"); - } + LOG.debug("No controls were sent from the server"); } // Re-activate paged results if (pagedResultsEnabled) { @@ -998,7 +971,7 @@ private void addToAttrMap(Map userAttrMap, String attrName, Attr String attrVal = UUID.nameUUIDFromBytes(otherUserAttrBytes).toString(); userAttrMap.put(attrName, attrVal); } catch (ClassCastException e) { - LOG.error("{} type is not set properly ", attrName, e.getMessage()); + LOG.error("{} type is not set properly {}", attrName, e.getMessage()); } } else if (attrType.equals("String")) { userAttrMap.put(attrName, (String) attr.get()); @@ -1037,16 +1010,15 @@ private static String getShortName(String longName) { } } } - } catch (NoSuchElementException ignore) { - shortName = longName; - } catch (NamingException ignore) { + } catch (NoSuchElementException | NamingException ignore) { shortName = longName; } } } catch (InvalidNameException ex) { shortName = longName; } - LOG.info("longName: {}, userName: ", longName, shortName); + LOG.info("longName: {}, userName: {}", longName, shortName); + return shortName; } @@ -1058,11 +1030,8 @@ private String getFirstRDN(String name) { try { LdapName subjectDN = new LdapName(name); List rdns = subjectDN.getRdns(); - for (int i = rdns.size() - 1; i >= 0; i--) { - if (StringUtils.isNotEmpty(shortName)) { - break; - } - Rdn rdn = rdns.get(i); + + for (Rdn rdn : rdns) { Attributes attributes = rdn.toAttributes(); try { Attribute cn = attributes.get("cn"); @@ -1070,6 +1039,7 @@ private String getFirstRDN(String name) { Object value = cn.get(); if (value != null) { shortName = GROUP_NAME_ATTRIBUTE + value; + break; } } } catch (NoSuchElementException ignore) { @@ -1079,11 +1049,11 @@ private String getFirstRDN(String name) { } } } catch (InvalidNameException ex) { - LOG.warn("InvalidNameException while retrieving first RDN for " + name); - } - if (LOG.isDebugEnabled()) { - LOG.debug("Input group name: {}, first RDN: {}", name, shortName); + LOG.warn("InvalidNameException while retrieving first RDN for {}", name); } + + LOG.debug("Input group name: {}, first RDN: {}", name, shortName); + return shortName; } @@ -1095,15 +1065,16 @@ private String getDNForMemberOf(String searchFilter) throws Throwable { String computedSearchFilter = ""; try { createLdapContext(); + SearchControls searchControls = new SearchControls(); + Set searchAttributes = new HashSet<>(); int total; - // Activate paged results - if (pagedResultsEnabled) { + + if (pagedResultsEnabled) { // Activate paged results ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, Control.NONCRITICAL)}); } - SearchControls searchControls = new SearchControls(); + searchControls.setSearchScope(groupSearchScope); - Set searchAttributes = new HashSet(); searchAttributes.add(groupNameAttribute); searchControls.setReturningAttributes(searchAttributes.toArray(new String[searchAttributes.size()])); @@ -1139,7 +1110,9 @@ private String getDNForMemberOf(String searchFilter) throws Throwable { } String groupFullName = userEntry.getNameInNamespace(); + LOG.info("groupFullName = {}", groupFullName); + computedSearchFilter += "(" + MEMBER_OF_ATTR + groupFullName + ")"; counter++; @@ -1165,27 +1138,19 @@ private String getDNForMemberOf(String searchFilter) throws Throwable { PagedResultsResponseControl prrc = (PagedResultsResponseControl) control; total = prrc.getResultSize(); if (total != 0) { - if (LOG.isDebugEnabled()) { - LOG.debug("END-OF-PAGE total : {}", total); - } + LOG.debug("END-OF-PAGE total : {}", total); } else { - if (LOG.isDebugEnabled()) { - LOG.debug("END-OF-PAGE total : unknown"); - } + LOG.debug("END-OF-PAGE total : unknown"); } cookie = prrc.getCookie(); } } } else { - if (LOG.isDebugEnabled()) { - LOG.debug("No controls were sent from the server"); - } + LOG.debug("No controls were sent from the server"); } // Re-activate paged results if (pagedResultsEnabled) { - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("Fetched paged results round: %s", ++paged)); - } + LOG.debug("Fetched paged results round: {}", ++paged); ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, cookie, Control.CRITICAL)}); } } @@ -1202,9 +1167,55 @@ private String getDNForMemberOf(String searchFilter) throws Throwable { } closeLdapContext(); } - if (LOG.isDebugEnabled()) { - LOG.debug("computedSearchFilter = {}", computedSearchFilter); - } + + LOG.debug("computedSearchFilter = {}", computedSearchFilter); + return computedSearchFilter; } + + private boolean isSearchResultValid(SearchResult searchResult, boolean isGroup) throws NamingException { + boolean result = true; + if (searchResult == null) { + LOG.debug("searchResult is null!"); + result = false; + } else { + Attributes attributes = searchResult.getAttributes(); + if (attributes == null) { + LOG.debug("Attributes missing for entry {}", searchResult.getNameInNamespace()); + result = false; + } else { + if (!isGroup) { + Attribute userNameAttr = attributes.get(userNameAttribute); + if (userNameAttr == null) { + LOG.debug("{} missing for entry {}", userNameAttribute, searchResult.getNameInNamespace()); + result = false; + } else { + String userName = (String) userNameAttr.get(); + + if (userName == null || userName.trim().isEmpty()) { + LOG.debug("{} empty for entry {}", userNameAttribute, searchResult.getNameInNamespace()); + result = false; + } + } + } else { + Attribute groupNameAttr = attributes.get(groupNameAttribute); + if (groupNameAttr == null) { + LOG.debug("{} missing for entry {}", groupNameAttribute, searchResult.getNameInNamespace()); + result = false; + } else { + String groupName = (String) groupNameAttr.get(); + + if (groupName == null || groupName.trim().isEmpty()) { + LOG.debug("{} empty for entry {}", groupNameAttr, searchResult.getNameInNamespace()); + result = false; + } + } + } + } + } + if (!result) { + LOG.debug("Skipping sync!"); + } + return result; + } } diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java index d0b9004cd6..d411a85409 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java @@ -44,64 +44,69 @@ public class UserGroupSyncConfig { private static final Logger LOG = LoggerFactory.getLogger(UserGroupSyncConfig.class); - public static final String CONFIG_FILE = "ranger-ugsync-site.xml"; - public static final String DEFAULT_CONFIG_FILE = "ranger-ugsync-default.xml"; + private static final String CORE_SITE_CONFIG_FILE = "core-site.xml"; - public static final String UGSYNC_ENABLED_PROP = "ranger.usersync.enabled"; - public static final String UGSYNC_PM_URL_PROP = "ranger.usersync.policymanager.baseURL"; - public static final String UGSYNC_UNIX_PASSWORD_FILE = "ranger.usersync.unix.password.file"; - public static final String DEFAULT_UGSYNC_UNIX_PASSWORD_FILE = "/etc/passwd"; - public static final String UGSYNC_UNIX_GROUP_FILE = "ranger.usersync.unix.group.file"; - public static final String DEFAULT_UGSYNC_UNIX_GROUP_FILE = "/etc/group"; - public static final String UGSYNC_MIN_USERID_PROP = "ranger.usersync.unix.minUserId"; - public static final String UGSYNC_MIN_GROUPID_PROP = "ranger.usersync.unix.minGroupId"; - public static final String DEFAULT_UGSYNC_MIN_GROUPID = "0"; - public static final String UGSYNC_MAX_RECORDS_PER_API_CALL_PROP = "ranger.usersync.policymanager.maxrecordsperapicall"; - public static final String UGSYNC_MOCK_RUN_PROP = "ranger.usersync.policymanager.mockrun"; - public static final String UGSYNC_TEST_RUN_PROP = "ranger.usersync.policymanager.testrun"; - public static final String UGSYNC_SOURCE_FILE_PROC = "ranger.usersync.filesource.file"; - public static final String UGSYNC_SOURCE_FILE_DELIMITER = "ranger.usersync.filesource.text.delimiter"; - public static final String UGSYNC_SOURCE_FILE_DELIMITERER = "ranger.usersync.filesource.text.delimiterer"; - public static final String UGSYNC_NONE_CASE_CONVERSION_VALUE = "none"; - public static final String UGSYNC_LOWER_CASE_CONVERSION_VALUE = "lower"; - public static final String UGSYNC_UPPER_CASE_CONVERSION_VALUE = "upper"; - public static final String SYNC_MAPPING_USERNAME = "ranger.usersync.mapping.username.regex"; - public static final String SYNC_MAPPING_GROUPNAME = "ranger.usersync.mapping.groupname.regex"; - public static final String UGSYNC_METRICS_ENABLED_PROP = "ranger.usersync.metrics.enabled"; - public static final String UGSYNC_NAME_VALIDATION_ENABLED = "ranger.usersync.name.validation.enabled"; - public static final String UGSYNC_SERVER_HA_ENABLED_PARAM = "ranger-ugsync.server.ha.enabled"; - public static final String UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED = "ranger.usersync.syncsource.validation.enabled"; - - private static final String SSL_KEYSTORE_FILE_TYPE_PARAM = "ranger.keystore.file.type"; - private static final String SSL_TRUSTSTORE_FILE_TYPE_PARAM = "ranger.truststore.file.type"; - private static final String SSL_KEYSTORE_PATH_PARAM = "ranger.usersync.keystore.file"; - private static final String SSL_KEYSTORE_PATH_PASSWORD_PARAM = "ranger.usersync.keystore.password"; - private static final String SSL_KEYSTORE_PATH_PASSWORD_ALIAS = "usersync.ssl.key.password"; - private static final String SSL_TRUSTSTORE_PATH_PARAM = "ranger.usersync.truststore.file"; - private static final String SSL_TRUSTSTORE_PATH_PASSWORD_PARAM = "ranger.usersync.truststore.password"; - private static final String SSL_TRUSTSTORE_PATH_PASSWORD_ALIAS = "usersync.ssl.truststore.password"; - private static final String UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM = "ranger.usersync.sleeptimeinmillisbetweensynccycle"; - private static final String UGSYNC_SLEEP_LDAP_FORCE_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM_ENABLED = "ranger.usersync.ldap.force.sleeptimeinmillisbetweensynccycle.enabled"; - private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE = 60000L; - private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_UNIX_DEFAULT_VALUE = 60000L; - private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_LDAP_DEFAULT_VALUE = 3600000L; - private static final String UGSYNC_SOURCE_CLASS_PARAM = "ranger.usersync.source.impl.class"; - private static final String UGSYNC_SINK_CLASS_PARAM = "ranger.usersync.sink.impl.class"; - private static final String UGSYNC_SOURCE_CLASS = "org.apache.ranger.unixusersync.process.UnixUserGroupBuilder"; - private static final String UGSYNC_SINK_CLASS = "org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder"; - private static final String LGSYNC_SOURCE_CLASS = "org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder"; - private static final String LGSYNC_LDAP_URL = "ranger.usersync.ldap.url"; - private static final String LGSYNC_LDAP_DELTASYNC_ENABLED = "ranger.usersync.ldap.deltasync"; - private static final boolean DEFAULT_LGSYNC_LDAP_DELTASYNC_ENABLED = false; - private static final String LGSYNC_LDAP_STARTTLS_ENABLED = "ranger.usersync.ldap.starttls"; - private static final boolean DEFAULT_LGSYNC_LDAP_STARTTLS_ENABLED = false; - private static final String LGSYNC_LDAP_BIND_DN = "ranger.usersync.ldap.binddn"; - private static final String LGSYNC_LDAP_BIND_KEYSTORE = "ranger.usersync.credstore.filename"; - private static final String LGSYNC_LDAP_BIND_ALIAS = "ranger.usersync.ldap.bindalias"; - private static final String LGSYNC_LDAP_BIND_PASSWORD = "ranger.usersync.ldap.ldapbindpassword"; - private static final String LGSYNC_LDAP_AUTHENTICATION_MECHANISM = "ranger.usersync.ldap.authentication.mechanism"; - private static final String DEFAULT_AUTHENTICATION_MECHANISM = "simple"; - private static final String LGSYNC_SEARCH_BASE = "ranger.usersync.ldap.searchBase"; + public static final String CONFIG_FILE = "ranger-ugsync-site.xml"; + public static final String DEFAULT_CONFIG_FILE = "ranger-ugsync-default.xml"; + + private static final String UGSYNC_SOURCE_CLASS_PARAM = "ranger.usersync.source.impl.class"; + private static final String UGSYNC_SINK_CLASS_PARAM = "ranger.usersync.sink.impl.class"; + private static final String UGSYNC_SOURCE_CLASS = "org.apache.ranger.unixusersync.process.UnixUserGroupBuilder"; + private static final String UGSYNC_SINK_CLASS = "org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder"; + private static final String LGSYNC_SOURCE_CLASS = "org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder"; + + public static final String UGSYNC_ENABLED_PROP = "ranger.usersync.enabled"; + public static final String UGSYNC_PM_URL_PROP = "ranger.usersync.policymanager.baseURL"; + public static final String UGSYNC_UNIX_PASSWORD_FILE = "ranger.usersync.unix.password.file"; + public static final String DEFAULT_UGSYNC_UNIX_PASSWORD_FILE = "/etc/passwd"; + public static final String UGSYNC_UNIX_GROUP_FILE = "ranger.usersync.unix.group.file"; + public static final String DEFAULT_UGSYNC_UNIX_GROUP_FILE = "/etc/group"; + public static final String UGSYNC_MIN_USERID_PROP = "ranger.usersync.unix.minUserId"; + public static final String UGSYNC_MIN_GROUPID_PROP = "ranger.usersync.unix.minGroupId"; + public static final String DEFAULT_UGSYNC_MIN_GROUPID = "0"; + public static final String UGSYNC_MAX_RECORDS_PER_API_CALL_PROP = "ranger.usersync.policymanager.maxrecordsperapicall"; + public static final String UGSYNC_MOCK_RUN_PROP = "ranger.usersync.policymanager.mockrun"; + public static final String UGSYNC_TEST_RUN_PROP = "ranger.usersync.policymanager.testrun"; + public static final String UGSYNC_SOURCE_FILE_PROC = "ranger.usersync.filesource.file"; + public static final String UGSYNC_SOURCE_FILE_DELIMITER = "ranger.usersync.filesource.text.delimiter"; + public static final String UGSYNC_SOURCE_FILE_DELIMITERER = "ranger.usersync.filesource.text.delimiterer"; + public static final String UGSYNC_NONE_CASE_CONVERSION_VALUE = "none"; + public static final String UGSYNC_LOWER_CASE_CONVERSION_VALUE = "lower"; + public static final String UGSYNC_UPPER_CASE_CONVERSION_VALUE = "upper"; + public static final String SYNC_MAPPING_USERNAME = "ranger.usersync.mapping.username.regex"; + public static final String SYNC_MAPPING_GROUPNAME = "ranger.usersync.mapping.groupname.regex"; + public static final String UGSYNC_METRICS_ENABLED_PROP = "ranger.usersync.metrics.enabled"; + public static final String UGSYNC_NAME_VALIDATION_ENABLED = "ranger.usersync.name.validation.enabled"; + public static final String UGSYNC_SERVER_HA_ENABLED_PARAM = "ranger-ugsync.server.ha.enabled"; + public static final String UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED = "ranger.usersync.syncsource.validation.enabled"; + + private static final String SSL_KEYSTORE_FILE_TYPE_PARAM = "ranger.keystore.file.type"; + private static final String SSL_TRUSTSTORE_FILE_TYPE_PARAM = "ranger.truststore.file.type"; + private static final String SSL_KEYSTORE_PATH_PARAM = "ranger.usersync.keystore.file"; + private static final String SSL_KEYSTORE_PATH_PASSWORD_PARAM = "ranger.usersync.keystore.password"; + private static final String SSL_KEYSTORE_PATH_PASSWORD_ALIAS = "usersync.ssl.key.password"; + private static final String SSL_TRUSTSTORE_PATH_PARAM = "ranger.usersync.truststore.file"; + private static final String SSL_TRUSTSTORE_PATH_PASSWORD_PARAM = "ranger.usersync.truststore.password"; + private static final String SSL_TRUSTSTORE_PATH_PASSWORD_ALIAS = "usersync.ssl.truststore.password"; + + private static final String UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM = "ranger.usersync.sleeptimeinmillisbetweensynccycle"; + private static final String UGSYNC_SLEEP_LDAP_FORCE_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM_ENABLED = "ranger.usersync.ldap.force.sleeptimeinmillisbetweensynccycle.enabled"; + private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE = 60000L; + private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_UNIX_DEFAULT_VALUE = 60000L; + private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_LDAP_DEFAULT_VALUE = 3600000L; + + private static final String LGSYNC_LDAP_URL = "ranger.usersync.ldap.url"; + private static final String LGSYNC_LDAP_DELTASYNC_ENABLED = "ranger.usersync.ldap.deltasync"; + private static final boolean DEFAULT_LGSYNC_LDAP_DELTASYNC_ENABLED = false; + private static final String LGSYNC_LDAP_STARTTLS_ENABLED = "ranger.usersync.ldap.starttls"; + private static final boolean DEFAULT_LGSYNC_LDAP_STARTTLS_ENABLED = false; + private static final String LGSYNC_LDAP_BIND_DN = "ranger.usersync.ldap.binddn"; + private static final String LGSYNC_LDAP_BIND_KEYSTORE = "ranger.usersync.credstore.filename"; + private static final String LGSYNC_LDAP_BIND_ALIAS = "ranger.usersync.ldap.bindalias"; + private static final String LGSYNC_LDAP_BIND_PASSWORD = "ranger.usersync.ldap.ldapbindpassword"; + private static final String LGSYNC_LDAP_AUTHENTICATION_MECHANISM = "ranger.usersync.ldap.authentication.mechanism"; + private static final String DEFAULT_AUTHENTICATION_MECHANISM = "simple"; + private static final String LGSYNC_SEARCH_BASE = "ranger.usersync.ldap.searchBase"; private static final String LGSYNC_USER_SEARCH_BASE = "ranger.usersync.ldap.user.searchbase"; private static final String LGSYNC_USER_SEARCH_SCOPE = "ranger.usersync.ldap.user.searchscope"; private static final String LGSYNC_USER_OBJECT_CLASS = "ranger.usersync.ldap.user.objectclass"; @@ -198,7 +203,7 @@ public class UserGroupSyncConfig { private static final long UGSYNC_INIT_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE_FOR_HA = 5000L; private static final boolean DEFAULT_UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED = true; private static volatile UserGroupSyncConfig me; - private Properties prop = new Properties(); + private final Properties prop = new Properties(); private Configuration userGroupConfig; private UserGroupSyncConfig() { @@ -343,7 +348,7 @@ public String getSSLKeyStorePathPassword() { if (prop.containsKey(LGSYNC_LDAP_BIND_KEYSTORE)) { String path = prop.getProperty(LGSYNC_LDAP_BIND_KEYSTORE); String alias = SSL_KEYSTORE_PATH_PASSWORD_ALIAS; - if (path != null && alias != null) { + if (path != null) { if (!path.trim().isEmpty() && !alias.trim().isEmpty()) { if ("bcfks".equalsIgnoreCase(getSSLKeyStoreType())) { String crendentialProviderPrefixBcfks = "bcfks" + "://file"; @@ -396,7 +401,7 @@ public long getUpdateMillisMin() { } public long getInitSleepTimeInMillisBetweenCycle() throws Throwable { - long initSleepValue = 0; + long initSleepValue; Configuration config = getUserGroupConfig(); if (config.getBoolean(UGSYNC_SERVER_HA_ENABLED_PARAM, false)) { initSleepValue = UGSYNC_INIT_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE_FOR_HA; @@ -444,9 +449,7 @@ public UserGroupSource getUserGroupSource() throws Throwable { Class ugSourceClass = (Class) Class.forName(className); - UserGroupSource ret = ugSourceClass.newInstance(); - - return ret; + return ugSourceClass.newInstance(); } public UserGroupSink getUserGroupSink() throws Throwable { @@ -458,9 +461,7 @@ public UserGroupSink getUserGroupSink() throws Throwable { Class ugSinkClass = (Class) Class.forName(val); - UserGroupSink ret = ugSinkClass.newInstance(); - - return ret; + return ugSinkClass.newInstance(); } public String getLdapUrl() throws Throwable { @@ -522,7 +523,7 @@ public String getUserSearchBase() throws Throwable { } /* Used only for unit testing */ - public void setUserSearchBase(String userSearchBase) throws Throwable { + public void setUserSearchBase(String userSearchBase) { prop.setProperty(LGSYNC_USER_SEARCH_BASE, userSearchBase); } @@ -591,7 +592,7 @@ public String getGroupNames() { public Set getGroupNameSet() { String groupNames = getGroupNames(); - Set groupNamegSet = new HashSet(); + Set groupNamegSet = new HashSet<>(); if (StringUtils.isNotEmpty(groupNames)) { StringTokenizer st = new StringTokenizer(groupNames, ";"); while (st.hasMoreTokens()) { @@ -604,7 +605,7 @@ public Set getGroupNameSet() { public Set getUserGroupNameAttributeSet() { String uga = getUserGroupNameAttribute(); StringTokenizer st = new StringTokenizer(uga, ","); - Set userGroupNameAttributeSet = new HashSet(); + Set userGroupNameAttributeSet = new HashSet<>(); while (st.hasMoreTokens()) { userGroupNameAttributeSet.add(st.nextToken().trim()); } @@ -617,7 +618,7 @@ public Set getOtherUserAttributes() { otherAttributes = DEFAULT_OTHER_USER_ATTRIBUTES; } StringTokenizer st = new StringTokenizer(otherAttributes, ","); - Set otherUserAttributes = new HashSet(); + Set otherUserAttributes = new HashSet<>(); while (st.hasMoreTokens()) { otherUserAttributes.add(st.nextToken().trim()); } @@ -668,7 +669,7 @@ public boolean isPagedResultsEnabled() { if (val == null || val.trim().isEmpty()) { pagedResultsEnabled = DEFAULT_LGSYNC_PAGED_RESULTS_ENABLED; } else { - pagedResultsEnabled = Boolean.valueOf(val); + pagedResultsEnabled = Boolean.parseBoolean(val); } return pagedResultsEnabled; } @@ -679,7 +680,7 @@ public void setPagedResultsEnabled(boolean pagedResultsEnabled) { } public int getPagedResultsSize() { - int pagedResultsSize = DEFAULT_LGSYNC_PAGED_RESULTS_SIZE; + int pagedResultsSize; String val = prop.getProperty(LGSYNC_PAGED_RESULTS_SIZE); if (val == null || val.trim().isEmpty()) { pagedResultsSize = DEFAULT_LGSYNC_PAGED_RESULTS_SIZE; @@ -698,7 +699,7 @@ public boolean isGroupSearchEnabled() { if (val == null || val.trim().isEmpty()) { groupSearchEnabled = DEFAULT_LGSYNC_GROUP_SEARCH_ENABLED; } else { - groupSearchEnabled = Boolean.valueOf(val); + groupSearchEnabled = Boolean.parseBoolean(val); } return groupSearchEnabled; } @@ -714,7 +715,7 @@ public boolean isGroupSearchFirstEnabled() { if (val == null || val.trim().isEmpty()) { groupSearchFirstEnabled = DEFAULT_LGSYNC_GROUP_SEARCH_FIRST_ENABLED; } else { - groupSearchFirstEnabled = Boolean.valueOf(val); + groupSearchFirstEnabled = Boolean.parseBoolean(val); } if (!isGroupSearchEnabled()) { groupSearchFirstEnabled = false; @@ -733,7 +734,7 @@ public boolean isUserSearchEnabled() { if (val == null || val.trim().isEmpty()) { userSearchEnabled = DEFAULT_LGSYNC_USER_SEARCH_ENABLED; } else { - userSearchEnabled = Boolean.valueOf(val); + userSearchEnabled = Boolean.parseBoolean(val); } if (!isGroupSearchFirstEnabled()) { userSearchEnabled = true; @@ -758,7 +759,7 @@ public String getGroupSearchBase() throws Throwable { } /* Used only for unit testing */ - public void setGroupSearchBase(String groupSearchBase) throws Throwable { + public void setGroupSearchBase(String groupSearchBase) { prop.setProperty(LGSYNC_GROUP_SEARCH_BASE, groupSearchBase); } @@ -843,7 +844,7 @@ public Set getOtherGroupAttributes() { otherAttributes = DEFAULT_OTHER_GROUP_ATTRIBUTES; } StringTokenizer st = new StringTokenizer(otherAttributes, ","); - Set otherGroupAttributes = new HashSet(); + Set otherGroupAttributes = new HashSet<>(); while (st.hasMoreTokens()) { otherGroupAttributes.add(st.nextToken().trim()); } @@ -882,7 +883,7 @@ public String getProperty(String aPropertyName, String aDefaultValue) { public String getPolicyMgrPassword() { //update credential from keystore - String password = null; + String password; if (prop != null && prop.containsKey(SYNC_POLICY_MGR_KEYSTORE)) { password = prop.getProperty(SYNC_POLICY_MGR_PASSWORD); if (password != null && !password.isEmpty()) { @@ -958,8 +959,8 @@ public String getContextReferral() { return referral; } - public List getAllRegexPatterns(String baseProperty) throws Throwable { - List regexPatterns = new ArrayList(); + public List getAllRegexPatterns(String baseProperty) { + List regexPatterns = new ArrayList<>(); if (prop != null) { String baseRegex = prop.getProperty(baseProperty); if (baseRegex == null) { @@ -1037,7 +1038,7 @@ public String getUserGroupNameDelimiter() { public boolean isUserSyncRangerCookieEnabled() { String val = prop.getProperty(USERSYNC_RANGER_COOKIE_ENABLED_PROP); - return val == null || Boolean.valueOf(val.trim()); + return val == null || Boolean.parseBoolean(val.trim()); } public String getRangerAdminCookieName() { @@ -1066,7 +1067,7 @@ public boolean isStartTlsEnabled() { if (val == null || val.trim().isEmpty()) { starttlsEnabled = DEFAULT_LGSYNC_LDAP_STARTTLS_ENABLED; } else { - starttlsEnabled = Boolean.valueOf(val); + starttlsEnabled = Boolean.parseBoolean(val); } return starttlsEnabled; } @@ -1077,7 +1078,7 @@ public boolean isDeltaSyncEnabled() { if (val == null || val.trim().isEmpty()) { deltaSyncEnabled = DEFAULT_LGSYNC_LDAP_DELTASYNC_ENABLED; } else { - deltaSyncEnabled = Boolean.valueOf(val); + deltaSyncEnabled = Boolean.parseBoolean(val); } return deltaSyncEnabled; } @@ -1142,7 +1143,7 @@ public long getUserSyncMetricsFrequency() { String val = prop.getProperty(UGSYNC_METRICS_FREQUENCY_TIME_IN_MILLIS_PARAM); if (StringUtils.isNotBlank(val)) { try { - ret = Long.valueOf(val); + ret = Long.parseLong(val); } catch (NumberFormatException exception) { // Ignore } @@ -1161,7 +1162,7 @@ public boolean isUserSyncDeletesEnabled() { if (StringUtils.isEmpty(val)) { isUserSyncDeletesEnabled = DEFAULT_UGSYNC_DELETES_ENABLED; } else { - isUserSyncDeletesEnabled = Boolean.valueOf(val); + isUserSyncDeletesEnabled = Boolean.parseBoolean(val); } return isUserSyncDeletesEnabled; } @@ -1172,12 +1173,12 @@ public boolean isUserSyncDeletesEnabled() { * If the delete frequency interval value is less than sync interval and greater than 8hrs, * then deleted objects are computed at every sync cycle. */ - public long getUserSyncDeletesFrequency() throws Throwable { + public long getUserSyncDeletesFrequency() { long ret = 1; String val = prop.getProperty(UGSYNC_DELETES_FREQUENCY); if (StringUtils.isNotBlank(val)) { - ret = Long.valueOf(val); + ret = Long.parseLong(val); if (!isTestRunEnabled() && ret < DEFAULT_UGSYNC_DELETES_FREQUENCY) { LOG.info("Frequency of computing deletes cannot be set below " + DEFAULT_UGSYNC_DELETES_FREQUENCY); ret = DEFAULT_UGSYNC_DELETES_FREQUENCY; @@ -1205,7 +1206,7 @@ public boolean isUserSyncNameValidationEnabled() { if (StringUtils.isEmpty(val)) { isUserSyncNameValidationEnabled = DEFAULT_UGSYNC_NAME_VALIDATION_ENABLED; } else { - isUserSyncNameValidationEnabled = Boolean.valueOf(val); + isUserSyncNameValidationEnabled = Boolean.parseBoolean(val); } return isUserSyncNameValidationEnabled; } @@ -1220,7 +1221,7 @@ public String getRegexSeparator() { LOG.warn("More than one character found in RegEx Separator, using default RegEx Separator /"); } } - LOG.info(String.format("Using %s as the RegEx Separator", ret)); + LOG.info("Using {} as the RegEx Separator", ret); return ret; } @@ -1242,9 +1243,9 @@ private void init() { private String getUserGroupSourceClassName() { String val = prop.getProperty(UGSYNC_SOURCE_CLASS_PARAM); - String className = UGSYNC_SOURCE_CLASS; + String className; - String syncSource = null; + String syncSource; if (val == null || val.trim().isEmpty()) { syncSource = getSyncSource(); @@ -1273,8 +1274,8 @@ private int getIntProperty(Properties prop, String key, int defaultValue) { if (StringUtils.isNotBlank(val)) { try { ret = Integer.parseInt(val); - } catch (NumberFormatException excp) { - LOG.warn("Invalid value for property: " + key + "=" + val + ". Will use default value: " + defaultValue, excp); + } catch (NumberFormatException exp) { + LOG.warn("Invalid value for property: {} = {} , will use default value: {}", key, val, defaultValue, exp); } } diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/ha/UserSyncHAInitializerImpl.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/ha/UserSyncHAInitializerImpl.java index fb56cc7ec1..e999d9c70c 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/ha/UserSyncHAInitializerImpl.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/ha/UserSyncHAInitializerImpl.java @@ -37,7 +37,9 @@ public class UserSyncHAInitializerImpl extends RangerHAInitializer { private static final Logger LOG = LoggerFactory.getLogger(UserSyncHAInitializerImpl.class); + private static UserSyncHAInitializerImpl theInstance; + ActiveInstanceElectorService activeInstanceElectorService; ActiveStateChangeHandler activeStateChangeHandler; List haRangerService; @@ -48,7 +50,7 @@ private UserSyncHAInitializerImpl(Configuration configuration) { LOG.debug("==> UserSyncHAInitializerImpl.UserSyncHAInitializerImpl()"); } try { - LOG.info("Ranger UserSync server is HA enabled : " + configuration.getBoolean(UserGroupSyncConfig.UGSYNC_SERVER_HA_ENABLED_PARAM, false)); + LOG.info("Ranger UserSync server is HA enabled : {}", configuration.getBoolean(UserGroupSyncConfig.UGSYNC_SERVER_HA_ENABLED_PARAM, false)); init(configuration); } catch (Exception e) { LOG.error("UserSyncHAInitializerImpl initialization failed {}", e.getMessage()); @@ -73,10 +75,10 @@ public void init(Configuration configuration) throws Exception { super.init(configuration); LOG.info("==> UserSyncHAInitializerImpl.init() initialization started "); - Set activeStateChangeHandlerProviders = new HashSet(); + Set activeStateChangeHandlerProviders = new HashSet<>(); activeInstanceElectorService = new ActiveInstanceElectorService(activeStateChangeHandlerProviders, curatorFactory, activeInstanceState, serviceState, configuration); - haRangerService = new ArrayList(); + haRangerService = new ArrayList<>(); haRangerService.add(activeInstanceElectorService); serviceManager = new ServiceManager(haRangerService); LOG.info("<== UserSyncHAInitializerImpl.init() initialization completed."); diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java index 2d7b3432d9..233c9bf197 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java @@ -50,12 +50,13 @@ public class FileSourceUserGroupBuilder extends AbstractUserGroupSource implements UserGroupSource { private static final Logger LOG = LoggerFactory.getLogger(FileSourceUserGroupBuilder.class); + private String userGroupFilename; private String currentSyncSource; private Map> sourceUsers; // Stores username and attr name & value pairs private Map> sourceGroups; // Stores groupname and attr name & value pairs private Map> sourceGroupUsers; - private Map> user2GroupListMap = new HashMap>(); + private Map> user2GroupListMap = new HashMap<>(); private UgsyncAuditInfo ugsyncAuditInfo; private FileSyncSourceInfo fileSyncSourceInfo; private int deleteCycles; @@ -169,9 +170,9 @@ public void updateSink(UserGroupSink sink) throws Throwable { } } - LOG.debug("Users = " + sourceUsers.keySet()); - LOG.debug("Groups = " + sourceGroups.keySet()); - LOG.debug("GroupUsers = " + sourceGroupUsers.keySet()); + LOG.debug("Users = {}", sourceUsers.keySet()); + LOG.debug("Groups = {}", sourceGroups.keySet()); + LOG.debug("GroupUsers = {}", sourceGroupUsers.keySet()); try { sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers, sourceGroupUsers, computeDeletes); @@ -183,7 +184,7 @@ public void updateSink(UserGroupSink sink) throws Throwable { try { sink.postUserGroupAuditInfo(ugsyncAuditInfo); } catch (Throwable t) { - LOG.error("sink.postUserGroupAuditInfo failed with exception: " + t.getMessage()); + LOG.error("sink.postUserGroupAuditInfo failed with exception: {}", t.getMessage()); } isStartupFlag = false; } @@ -206,7 +207,7 @@ public void buildUserGroupList() throws Throwable { File f = new File(userGroupFilename); if (f.exists() && f.canRead()) { - Map> tmpUser2GroupListMap = null; + Map> tmpUser2GroupListMap; if (isJsonFile(userGroupFilename)) { tmpUser2GroupListMap = readJSONfile(f); @@ -235,7 +236,7 @@ public Map> readJSONfile(File jsonFile) throws Exception { } public Map> readTextFile(File textFile) throws Exception { - Map> ret = new HashMap>(); + Map> ret = new HashMap<>(); String delimiter = config.getUserSyncFileSourceDelimiter(); @@ -246,7 +247,7 @@ public Map> readTextFile(File textFile) throws Exception { if (csvRecordList != null) { for (CSVRecord csvRecord : csvRecordList) { - List groups = new ArrayList(); + List groups = new ArrayList<>(); String user = csvRecord.get(0); user = user.replaceAll("^\"|\"$", ""); diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java index ca842f9c39..78bb0883b2 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java @@ -60,6 +60,7 @@ public class PolicyMgrUserGroupBuilder extends AbstractUserGroupSource implements UserGroupSink { private static final Logger LOG = LoggerFactory.getLogger(PolicyMgrUserGroupBuilder.class); + /* ***** POST APIs **** */ private static final String PM_ADD_USERS_URI = "/service/xusers/ugsync/users"; private static final String PM_ADD_GROUPS_URI = "/service/xusers/ugsync/groups/"; diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/RangerUgSyncRESTClient.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/RangerUgSyncRESTClient.java index a44b95e53b..67a4179c3e 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/RangerUgSyncRESTClient.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/RangerUgSyncRESTClient.java @@ -33,15 +33,13 @@ import javax.net.ssl.HostnameVerifier; import javax.net.ssl.KeyManager; import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSession; import javax.net.ssl.TrustManager; public class RangerUgSyncRESTClient extends RangerRESTClient { - private String authKerberos = "kerberos"; - public RangerUgSyncRESTClient(String policyMgrBaseUrls, String ugKeyStoreFile, String ugKeyStoreFilepwd, String ugKeyStoreType, String ugTrustStoreFile, String ugTrustStoreFilepwd, String ugTrustStoreType, String authenticationType, String principal, String keytab, String polMgrUsername, String polMgrPassword) { super(policyMgrBaseUrls, "", UserGroupSyncConfig.getInstance().getConfig()); - if (!(authenticationType != null && authKerberos.equalsIgnoreCase(authenticationType) && SecureClientLogin.isKerberosCredentialExists(principal, keytab))) { + String authKerberos = "kerberos"; + if (!(authKerberos.equalsIgnoreCase(authenticationType) && SecureClientLogin.isKerberosCredentialExists(principal, keytab))) { setBasicAuthInfo(polMgrUsername, polMgrPassword); } @@ -54,11 +52,7 @@ public RangerUgSyncRESTClient(String policyMgrBaseUrls, String ugKeyStoreFile, S ClientConfig config = new DefaultClientConfig(); config.getClasses().add(JacksonJsonProvider.class); // to handle List<> unmarshalling - HostnameVerifier hv = new HostnameVerifier() { - public boolean verify(String urlHostName, SSLSession session) { - return session.getPeerHost().equals(urlHostName); - } - }; + HostnameVerifier hv = (urlHostName, session) -> session.getPeerHost().equals(urlHostName); config.getProperties().put(HTTPSProperties.PROPERTY_HTTPS_PROPERTIES, new HTTPSProperties(hv, sslContext)); setClient(Client.create(config)); diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java index 6a256812e6..a29bb5730c 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java @@ -42,12 +42,12 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.Map; import java.util.Set; public class UnixUserGroupBuilder implements UserGroupSource { private static final Logger LOG = LoggerFactory.getLogger(UnixUserGroupBuilder.class); + private static final String OS = System.getProperty("os.name"); /** * Shell commands to get users and groups @@ -70,22 +70,22 @@ public class UnixUserGroupBuilder implements UserGroupSource { static final String BACKEND_PASSWD = "passwd"; Set allGroups = new HashSet<>(); private boolean useNss; - private boolean enumerateGroupMembers; + private final boolean enumerateGroupMembers; private boolean isUpdateSinkSucc = true; - private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); + private final UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); private Map groupId2groupNameMap; private Map> sourceUsers; // Stores username and attr name & value pairs private Map> sourceGroups; // Stores groupname and attr name & value pairs private Map> sourceGroupUsers; private Table groupUserTable; // groupname, username, group id - private String unixPasswordFile; - private String unixGroupFile; + private final String unixPasswordFile; + private final String unixGroupFile; private String currentSyncSource; private int deleteCycles; - private int minimumUserId; - private int minimumGroupId; + private final int minimumUserId; + private final int minimumGroupId; private long lastUpdateTime; - private long timeout; + private final long timeout; private long passwordFileModifiedAt; private long groupFileModifiedAt; private UgsyncAuditInfo ugsyncAuditInfo; @@ -184,9 +184,9 @@ public void updateSink(UserGroupSink sink) throws Throwable { unixSyncSourceInfo.setSyncTime(formatter.format(syncTime)); if (isChanged() || isStartupFlag) { buildUserGroupInfo(); - LOG.debug("Users = " + sourceUsers.keySet()); - LOG.debug("Groups = " + sourceGroups.keySet()); - LOG.debug("GroupUsers = " + sourceGroupUsers.keySet()); + LOG.debug("Users = {}", sourceUsers.keySet()); + LOG.debug("Groups = {}", sourceGroups.keySet()); + LOG.debug("GroupUsers = {}", sourceGroupUsers.keySet()); try { sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers, sourceGroupUsers, computeDeletes); @@ -237,9 +237,7 @@ private void buildUserGroupInfo() throws Throwable { buildUnixUserList(LINUX_GET_ALL_USERS_CMD); } - Iterator groupUserTableIterator = groupUserTable.rowKeySet().iterator(); - while (groupUserTableIterator.hasNext()) { - String groupName = groupUserTableIterator.next(); + for (String groupName : groupUserTable.rowKeySet()) { Map groupUsersMap = groupUserTable.row(groupName); Set userSet = new HashSet<>(); @@ -260,10 +258,10 @@ private void buildUserGroupInfo() throws Throwable { private void print() { for (String user : sourceUsers.keySet()) { - LOG.debug("USER:" + user); + LOG.debug("USER: {}", user); Set groups = groupUserTable.column(user).keySet(); for (String group : groups) { - LOG.debug("\tGROUP: " + group); + LOG.debug("\tGROUP: {}", group); } } } @@ -294,7 +292,7 @@ private void buildUnixUserList(String command) throws Throwable { int len = tokens.length; if (len < 4) { - LOG.warn("Unable to parse: " + line); + LOG.warn("Unable to parse: {}", line); continue; } @@ -307,7 +305,7 @@ private void buildUnixUserList(String command) throws Throwable { userId = tokens[2]; groupId = tokens[3]; } catch (ArrayIndexOutOfBoundsException aiobe) { - LOG.warn("Ignoring line - [" + line + "]: Unable to parse line for getting user information", aiobe); + LOG.warn("Ignoring line - [{}]: Unable to parse line for getting user information", line, aiobe); continue; } @@ -315,7 +313,7 @@ private void buildUnixUserList(String command) throws Throwable { try { numUserId = Integer.parseInt(userId); } catch (NumberFormatException nfe) { - LOG.warn("Unix UserId: [" + userId + "]: can not be parsed as valid int. considering as -1.", nfe); + LOG.warn("Unix UserId: [{}]: can not be parsed as valid int. considering as -1.", userId, nfe); numUserId = -1; } @@ -374,11 +372,11 @@ private void buildUnixUserList(String command) throws Throwable { } if (LOG.isDebugEnabled()) { - LOG.debug("id -G returned " + line); + LOG.debug("id -G returned {}", line); } if (line == null || line.trim().isEmpty()) { - LOG.warn("User " + userName + " could not be resolved"); + LOG.warn("User {} could not be resolved", userName); continue; } @@ -444,7 +442,7 @@ private void parseMembers(String line) { groupUserTable.put(groupName, user, groupId); } } else { - sourceGroupUsers.put(groupName, new HashSet()); + sourceGroupUsers.put(groupName, new HashSet<>()); } } @@ -484,8 +482,8 @@ private void buildUnixGroupList(String allGroupsCmd, String groupCmd, boolean us if (enumerateGroupMembers) { LOG.debug("Start enumerating group members"); - String line = null; - Map copy = new HashMap(groupId2groupNameMap); + String line; + Map copy = new HashMap<>(groupId2groupNameMap); for (Map.Entry group : copy.entrySet()) { LOG.debug("Enumerating group: {} GID({})", group.getValue(), group.getKey()); diff --git a/ugsync/src/main/java/org/apache/ranger/usergroupsync/RegEx.java b/ugsync/src/main/java/org/apache/ranger/usergroupsync/RegEx.java index 4e97c7e097..d991f856e5 100644 --- a/ugsync/src/main/java/org/apache/ranger/usergroupsync/RegEx.java +++ b/ugsync/src/main/java/org/apache/ranger/usergroupsync/RegEx.java @@ -64,8 +64,8 @@ public String transform(String attrValue) { return result; } - protected void populateReplacementPatterns(String baseProperty, List regexPatterns, String regexSeparator) throws Throwable { - replacementPattern = new LinkedHashMap(); + protected void populateReplacementPatterns(String baseProperty, List regexPatterns, String regexSeparator) { + replacementPattern = new LinkedHashMap<>(); String regex = String.format("s%s([^%s]*)%s([^%s]*)%s(g)?", regexSeparator, regexSeparator, regexSeparator, regexSeparator, regexSeparator); Pattern p = Pattern.compile(regex); for (String regexPattern : regexPatterns) { diff --git a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java index 87bf6566a0..523582069b 100644 --- a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java +++ b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java @@ -25,6 +25,7 @@ public class UserGroupSync implements Runnable { private static final Logger LOG = LoggerFactory.getLogger(UserGroupSync.class); + private UserGroupSink ugSink; private UserGroupSource ugSource; @@ -44,11 +45,11 @@ public void run() { try { if (UserGroupSyncConfig.isUgsyncServiceActive()) { ugSink = UserGroupSyncConfig.getInstance().getUserGroupSink(); - LOG.info("initializing sink: " + ugSink.getClass().getName()); + LOG.info("initializing sink: {}", ugSink.getClass().getName()); ugSink.init(); ugSource = UserGroupSyncConfig.getInstance().getUserGroupSource(); - LOG.info("initializing source: " + ugSource.getClass().getName()); + LOG.info("initializing source: {}", ugSource.getClass().getName()); ugSource.init(); LOG.info("Begin: initial load of user/group from source==>sink"); @@ -59,19 +60,19 @@ public void run() { LOG.info("Done initializing user/group source and sink"); } else { if (LOG.isDebugEnabled()) { - LOG.debug("Sleeping for [" + initSleepTimeBetweenCycleInMillis + "] milliSeconds as this server is running in passive mode"); + LOG.debug("Sleeping for [{}] milliSeconds as this server is running in passive mode", initSleepTimeBetweenCycleInMillis); } Thread.sleep(initSleepTimeBetweenCycleInMillis); } } catch (Throwable t) { - LOG.error("Failed to initialize UserGroup source/sink. Will retry after " + sleepTimeBetweenCycleInMillis + " milliseconds. Error details: ", t); + LOG.error("Failed to initialize UserGroup source/sink. Will retry after {} milliseconds. Error details: ", sleepTimeBetweenCycleInMillis, t); try { if (LOG.isDebugEnabled()) { - LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); + LOG.debug("Sleeping for [{}] milliSeconds", sleepTimeBetweenCycleInMillis); } Thread.sleep(sleepTimeBetweenCycleInMillis); } catch (Exception e) { - LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to initialize UserGroup source/sink", e); + LOG.error("Failed to wait for [{}] milliseconds before attempting to initialize UserGroup source/sink", sleepTimeBetweenCycleInMillis, e); } } } @@ -79,11 +80,11 @@ public void run() { while (true) { try { if (LOG.isDebugEnabled()) { - LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); + LOG.debug("Sleeping for {} milliSeconds", sleepTimeBetweenCycleInMillis); } Thread.sleep(sleepTimeBetweenCycleInMillis); } catch (InterruptedException e) { - LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to synchronize UserGroup information", e); + LOG.error("Failed to wait for [{}] milliseconds before attempting to synchronize UserGroup information", sleepTimeBetweenCycleInMillis, e); } try { @@ -92,7 +93,7 @@ public void run() { syncUserGroup(); LOG.info("End: update user/group from source==>sink"); } else { - LOG.info("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds as this server is running in passive mode"); + LOG.info("Sleeping for [{}] milliSeconds as this server is running in passive mode", sleepTimeBetweenCycleInMillis); } } catch (Throwable t) { LOG.error("Failed to synchronize UserGroup information. Error details: ", t); diff --git a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserSyncMetricsProducer.java b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserSyncMetricsProducer.java index e3e028a7ac..e3af416c6c 100644 --- a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserSyncMetricsProducer.java +++ b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserSyncMetricsProducer.java @@ -28,7 +28,7 @@ import java.io.File; public class UserSyncMetricsProducer implements Runnable { - private static final Logger LOG = LoggerFactory.getLogger(UserSyncMetricsProducer.class); + private static final Logger LOG = LoggerFactory.getLogger(UserSyncMetricsProducer.class); private boolean shutdownFlag; public static void main(String[] args) { @@ -46,7 +46,9 @@ public void run() { UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); long sleepTimeBetweenCycleInMillis = config.getUserSyncMetricsFrequency(); String logFileNameWithPath = config.getUserSyncMetricsFileName(); + LOG.info("User Sync metrics frequency : {} and metrics file : {}", sleepTimeBetweenCycleInMillis, logFileNameWithPath); + if (logFileNameWithPath != null) { while (!shutdownFlag) { try { @@ -73,19 +75,21 @@ public void run() { private void writeJVMMetrics(String logFileNameWithPath) throws Throwable { try { - File userMetricFile = null; + File userMetricFile = new File(logFileNameWithPath); UserGroupSyncConfig userConfig = UserGroupSyncConfig.getInstance(); Configuration config = userConfig.getUserGroupConfig(); - userMetricFile = new File(logFileNameWithPath); + if (!userMetricFile.exists()) { userMetricFile.createNewFile(); } + RangerMetricsUtil rangerMetricsUtil = new RangerMetricsUtil(); + if (config.getBoolean(UserGroupSyncConfig.UGSYNC_SERVER_HA_ENABLED_PARAM, false)) { - if (userConfig.isUgsyncServiceActive()) { - rangerMetricsUtil.setIsRoleActive(1); + if (UserGroupSyncConfig.isUgsyncServiceActive()) { + RangerMetricsUtil.setIsRoleActive(1); } else { - rangerMetricsUtil.setIsRoleActive(0); + RangerMetricsUtil.setIsRoleActive(0); } } rangerMetricsUtil.writeMetricsToFile(userMetricFile); diff --git a/unixauthclient/pom.xml b/unixauthclient/pom.xml index e037a87501..7abe59e959 100644 --- a/unixauthclient/pom.xml +++ b/unixauthclient/pom.xml @@ -29,6 +29,7 @@ Unix authentication client true + false diff --git a/unixauthnative/pom.xml b/unixauthnative/pom.xml index e6349aad77..63de0af5dc 100644 --- a/unixauthnative/pom.xml +++ b/unixauthnative/pom.xml @@ -29,6 +29,7 @@ Unix authentication service true + false diff --git a/unixauthpam/pom.xml b/unixauthpam/pom.xml index 0103e4f52c..35937446e6 100644 --- a/unixauthpam/pom.xml +++ b/unixauthpam/pom.xml @@ -29,6 +29,7 @@ PAM authentication service true + false diff --git a/unixauthservice/pom.xml b/unixauthservice/pom.xml index be831c8773..52384a70c8 100644 --- a/unixauthservice/pom.xml +++ b/unixauthservice/pom.xml @@ -29,6 +29,7 @@ Unix authentication service true + false From de3e8810129fc105ba5f4dc32d9c9a449242ee83 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 16 Dec 2024 20:32:45 -0800 Subject: [PATCH 3/4] RANGER-5064: Incorporating IntelliJ suggestions and refactoring --- .../ugsyncutil/model/UnixSyncSourceInfo.java | 8 +- .../model/UsersGroupRoleAssignments.java | 4 - .../ranger/ugsyncutil/model/XUserInfo.java | 6 +- .../util/UgsyncCommonConstants.java | 1 + .../ldapconfigcheck/AuthenticationCheck.java | 2 +- .../ldapconfigcheck/CommandLineOptions.java | 24 +- .../ranger/ldapconfigcheck/LdapConfig.java | 35 +- .../ldapconfigcheck/LdapConfigCheckMain.java | 3 +- .../ranger/ldapconfigcheck/UserSync.java | 69 +-- .../process/CustomSSLSocketFactory.java | 2 +- .../process/LdapUserGroupBuilder.java | 114 ++--- .../config/UserGroupSyncConfig.java | 435 +++++++++--------- .../process/FileSourceUserGroupBuilder.java | 4 +- .../process/PolicyMgrUserGroupBuilder.java | 72 +-- .../process/UnixUserGroupBuilder.java | 46 +- .../apache/ranger/usergroupsync/RegEx.java | 4 +- .../UserSyncMetricsProducer.java | 1 + .../TestFileSourceUserGroupBuilder.java | 2 +- .../PolicyMgrUserGroupBuilderTest.java | 12 +- .../ranger/usergroupsync/TestRegEx.java | 22 +- .../unix/jaas/PamLoginModule.java | 1 + .../unix/jaas/PamPrincipal.java | 14 +- .../unix/jaas/RemoteUnixLoginModule.java | 38 +- .../unix/jaas/UnixGroupPrincipal.java | 5 +- .../unix/jaas/UnixUserPrincipal.java | 5 +- .../authentication/PasswordValidator.java | 11 +- .../UnixAuthenticationService.java | 88 ++-- 27 files changed, 486 insertions(+), 542 deletions(-) diff --git a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UnixSyncSourceInfo.java b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UnixSyncSourceInfo.java index e6cb6a630a..dbc2e0500a 100644 --- a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UnixSyncSourceInfo.java +++ b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UnixSyncSourceInfo.java @@ -26,10 +26,10 @@ public class UnixSyncSourceInfo { private String lastModified; private String minUserId; private String minGroupId; - private long totalUsersSynced; - private long totalGroupsSynced; - private long totalUsersDeleted; - private long totalGroupsDeleted; + private long totalUsersSynced; + private long totalGroupsSynced; + private long totalUsersDeleted; + private long totalGroupsDeleted; public String getMinGroupId() { return minGroupId; diff --git a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UsersGroupRoleAssignments.java b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UsersGroupRoleAssignments.java index 589b5d2a5c..9ee741cc87 100644 --- a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UsersGroupRoleAssignments.java +++ b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/UsersGroupRoleAssignments.java @@ -31,13 +31,9 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class UsersGroupRoleAssignments { List users; - Map groupRoleAssignments; - Map userRoleAssignments; - Map whiteListGroupRoleAssignments; - Map whiteListUserRoleAssignments; boolean isReset; diff --git a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/XUserInfo.java b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/XUserInfo.java index fd61ca5465..ff8b85defd 100644 --- a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/XUserInfo.java +++ b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/model/XUserInfo.java @@ -31,12 +31,12 @@ public class XUserInfo { private String description; private String otherAttributes; private String syncSource; - private Map otherAttrsMap = new HashMap<>(); private String userSource; private String status; private String isVisible; - private List groupNameList = new ArrayList(); - private List userRoleList = new ArrayList(); + private List groupNameList = new ArrayList<>(); + private List userRoleList = new ArrayList<>(); + private Map otherAttrsMap = new HashMap<>(); public String getId() { return id; diff --git a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java index 704e8c4be2..ca1b82df3d 100644 --- a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java +++ b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java @@ -26,5 +26,6 @@ public class UgsyncCommonConstants { public static final String LDAP_URL = "ldap_url"; private UgsyncCommonConstants() { + // to block instantiation } } diff --git a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/AuthenticationCheck.java b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/AuthenticationCheck.java index 772b1d6b6f..7f6b6fb81c 100644 --- a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/AuthenticationCheck.java +++ b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/AuthenticationCheck.java @@ -190,7 +190,7 @@ private Authentication getLdapBindAuthentication(String ldapUrl, String bindDn, result = ldapAuthenticationProvider.authenticate(finalAuthentication); } } catch (BadCredentialsException bce) { - logFile.println("ERROR: LDAP Authentication Failed. Please verify values for ranger.admin.auth.sampleuser and ranger.admin.auth.samplepassword\n"); + logFile.println("ERROR: LDAP Authentication Failed. Please verify values for ranger.admin.auth.sampleuser and ranger.admin.auth.samplepassword"); } catch (Exception e) { logFile.println("ERROR: LDAP Authentication Failed: " + e); } diff --git a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/CommandLineOptions.java b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/CommandLineOptions.java index a188c8b069..c92e58bea8 100644 --- a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/CommandLineOptions.java +++ b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/CommandLineOptions.java @@ -32,18 +32,18 @@ public class CommandLineOptions { private final Options options = new Options(); private final String[] args; - private String input; - private String output; - private String discoverProperties; - private String retrieveValues; - private String ldapUrl = ""; - private String bindDn = ""; - private String bindPassword = ""; - private String userSearchBase = ""; - private String userSearchFilter = ""; - private String authUser = ""; - private String authPass = ""; - private boolean isAuthEnabled = true; + private String input; + private String output; + private String discoverProperties; + private String retrieveValues; + private String ldapUrl = ""; + private String bindDn = ""; + private String bindPassword = ""; + private String userSearchBase = ""; + private String userSearchFilter = ""; + private String authUser = ""; + private String authPass = ""; + private boolean isAuthEnabled = true; public CommandLineOptions(String[] args) { this.args = args; diff --git a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfig.java b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfig.java index 1ddd656d1c..5d28914ca0 100644 --- a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfig.java +++ b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfig.java @@ -54,28 +54,29 @@ public class LdapConfig { private static final String UGSYNC_GROUPNAME_CASE_CONVERSION_PARAM = "ranger.usersync.ldap.groupname.caseconversion"; private static final String DEFAULT_UGSYNC_USERNAME_CASE_CONVERSION_VALUE = UGSYNC_NONE_CASE_CONVERSION_VALUE; private static final String DEFAULT_UGSYNC_GROUPNAME_CASE_CONVERSION_VALUE = UGSYNC_NONE_CASE_CONVERSION_VALUE; - private static final String LGSYNC_PAGED_RESULTS_ENABLED = "ranger.usersync.pagedresultsenabled"; - private static final String LGSYNC_PAGED_RESULTS_SIZE = "ranger.usersync.pagedresultssize"; - private static final String LGSYNC_GROUP_SEARCH_ENABLED = "ranger.usersync.group.searchenabled"; - private static final String LGSYNC_GROUP_SEARCH_BASE = "ranger.usersync.group.searchbase"; - private static final String LGSYNC_GROUP_SEARCH_SCOPE = "ranger.usersync.group.searchscope"; - private static final String LGSYNC_GROUP_OBJECT_CLASS = "ranger.usersync.group.objectclass"; - private static final String LGSYNC_GROUP_SEARCH_FILTER = "ranger.usersync.group.searchfilter"; - private static final String LGSYNC_GROUP_NAME_ATTRIBUTE = "ranger.usersync.group.nameattribute"; - private static final String LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = "ranger.usersync.group.memberattributename"; + private static final String LGSYNC_PAGED_RESULTS_ENABLED = "ranger.usersync.pagedresultsenabled"; + private static final String LGSYNC_PAGED_RESULTS_SIZE = "ranger.usersync.pagedresultssize"; + private static final String LGSYNC_GROUP_SEARCH_ENABLED = "ranger.usersync.group.searchenabled"; + private static final String LGSYNC_GROUP_SEARCH_BASE = "ranger.usersync.group.searchbase"; + private static final String LGSYNC_GROUP_SEARCH_SCOPE = "ranger.usersync.group.searchscope"; + private static final String LGSYNC_GROUP_OBJECT_CLASS = "ranger.usersync.group.objectclass"; + private static final String LGSYNC_GROUP_SEARCH_FILTER = "ranger.usersync.group.searchfilter"; + private static final String LGSYNC_GROUP_NAME_ATTRIBUTE = "ranger.usersync.group.nameattribute"; + private static final String LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = "ranger.usersync.group.memberattributename"; + private static final int DEFAULT_LGSYNC_PAGED_RESULTS_SIZE = 500; private static final boolean DEFAULT_LGSYNC_PAGED_RESULTS_ENABLED = true; private static final boolean DEFAULT_LGSYNC_GROUP_SEARCH_ENABLED = false; //Authentication related properties - private static final String AUTHENTICATION_METHOD = "ranger.authentication.method"; - private static final String AD_DOMAIN = "ranger.ldap.ad.domain"; - private static final String USER_DN_PATTERN = "ranger.ldap.user.dnpattern"; - private static final String GROUP_ROLE_ATTRIBUTE = "ranger.ldap.group.roleattribute"; - private static final String GROUP_SEARCH_BASE = "ranger.ldap.group.searchbase"; - private static final String GROUP_SEARCH_FILTER = "ranger.ldap.group.searchfilter"; - private static final String AUTH_USERNAME = "ranger.admin.auth.sampleuser"; - private static final String AUTH_PASSWORD = "ranger.admin.auth.samplepassword"; + private static final String AUTHENTICATION_METHOD = "ranger.authentication.method"; + private static final String AD_DOMAIN = "ranger.ldap.ad.domain"; + private static final String USER_DN_PATTERN = "ranger.ldap.user.dnpattern"; + private static final String GROUP_ROLE_ATTRIBUTE = "ranger.ldap.group.roleattribute"; + private static final String GROUP_SEARCH_BASE = "ranger.ldap.group.searchbase"; + private static final String GROUP_SEARCH_FILTER = "ranger.ldap.group.searchfilter"; + private static final String AUTH_USERNAME = "ranger.admin.auth.sampleuser"; + private static final String AUTH_PASSWORD = "ranger.admin.auth.samplepassword"; private final Properties prop = new Properties(); diff --git a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfigCheckMain.java b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfigCheckMain.java index 8043ea62e0..9a2a105a07 100644 --- a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfigCheckMain.java +++ b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/LdapConfigCheckMain.java @@ -52,8 +52,7 @@ public static void main(String[] args) { LdapConfig config = new LdapConfig(inFileName, cli.getBindPassword()); if (cli.getLdapUrl() != null && !cli.getLdapUrl().isEmpty()) { - config.updateInputPropFile(cli.getLdapUrl(), cli.getBindDn(), cli.getBindPassword(), - cli.getUserSearchBase(), cli.getUserSearchFilter(), cli.getAuthUser(), cli.getAuthPass()); + config.updateInputPropFile(cli.getLdapUrl(), cli.getBindDn(), cli.getBindPassword(), cli.getUserSearchBase(), cli.getUserSearchFilter(), cli.getAuthUser(), cli.getAuthPass()); } PrintStream logFile = null; diff --git a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/UserSync.java b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/UserSync.java index d0ede31330..0ac03950f6 100644 --- a/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/UserSync.java +++ b/ugsync/ldapconfigchecktool/ldapconfigcheck/src/main/java/org/apache/ranger/ldapconfigcheck/UserSync.java @@ -58,22 +58,22 @@ public class UserSync { "member", "memberUid" }; - private String userNameAttribute; - private String userObjClassName; - private String userGroupMemberName; - private String groupMemberName; - private String groupNameAttrName; - private String groupObjClassName; - private String groupSearchBase; - private String groupSearchFilter; - private String userSearchBase; - private String userSearchFilter; - private String searchBase; - private String groupName; + private String userNameAttribute; + private String userObjClassName; + private String userGroupMemberName; + private String groupMemberName; + private String groupNameAttrName; + private String groupObjClassName; + private String groupSearchBase; + private String groupSearchFilter; + private String userSearchBase; + private String userSearchFilter; + private String searchBase; + private String groupName; private final PrintStream logFile; private final PrintStream ambariProps; private final PrintStream installProps; - private final LdapConfig config; + private final LdapConfig config; public UserSync(LdapConfig config, PrintStream logFile, PrintStream ambariProps, PrintStream installProps) { this.config = config; @@ -123,14 +123,12 @@ public void findUserProperties(LdapContext ldapContext) throws Throwable { // 1. find basic user properties // 2. find user search base and user search filter by passing basic attributes findBasicUserProperties(ldapContext, true); - findAdvUserProperties(ldapContext, true); } public void getAllUsers(LdapContext ldapContext) throws Throwable { int noOfUsers = 0; Attribute userNameAttr = null; - //String groupName = null; Attribute groupMemberAttr; NamingEnumeration userSearchResultEnum = null; SearchControls userSearchControls = new SearchControls(); @@ -257,11 +255,7 @@ public void getAllUsers(LdapContext ldapContext) throws Throwable { (config.getGroupNameAttribute() != null && !config.getGroupNameAttribute().isEmpty()) || (config.getUserSearchBase() != null && !config.getUserSearchBase().isEmpty()) || (config.getUserSearchFilter() != null && !config.getUserSearchFilter().isEmpty())) { - throw new Exception("Please verify values for:\n ranger.usersync.ldap.user.nameattribute\n " + - "ranger.usersync.ldap.user.objectclass\n" + - "ranger.usersync.ldap.user.groupnameattribute\n" + - "ranger.usersync.ldap.user.searchbase\n" + - "ranger.usersync.ldap.user.searchfilter\n"); + throw new Exception("Please verify values for:\n ranger.usersync.ldap.user.nameattribute\n " + "ranger.usersync.ldap.user.objectclass\n" + "ranger.usersync.ldap.user.groupnameattribute\n" + "ranger.usersync.ldap.user.searchbase\n" + "ranger.usersync.ldap.user.searchfilter\n"); } else { throw new Exception(msg + ne); } @@ -302,8 +296,7 @@ public void getAllGroups(LdapContext ldapContext) throws Throwable { groupSearchAttributes.add(groupNameAttrName); groupSearchAttributes.add(groupMemberName); groupSearchAttributes.add("distinguishedName"); - groupSearchControls.setReturningAttributes(groupSearchAttributes.toArray( - new String[groupSearchAttributes.size()])); + groupSearchControls.setReturningAttributes(groupSearchAttributes.toArray(new String[groupSearchAttributes.size()])); String extendedGroupSearchFilter = "(objectclass=" + groupObjClassName + ")"; if (groupSearchFilter != null && !groupSearchFilter.trim().isEmpty()) { @@ -358,11 +351,7 @@ public void getAllGroups(LdapContext ldapContext) throws Throwable { (config.getUserGroupMemberAttributeName() != null && !config.getUserGroupMemberAttributeName().isEmpty()) || (config.getGroupSearchBase() != null && !config.getGroupSearchBase().isEmpty()) || (config.getGroupSearchFilter() != null && !config.getGroupSearchFilter().isEmpty())) { - throw new Exception("Please verify values for:\n ranger.usersync.group.memberattributename\n " + - "ranger.usersync.group.nameattribute\n" + - "ranger.usersync.group.objectclass\n" + - "ranger.usersync.group.searchbase\n" + - "ranger.usersync.group.searchfilter\n"); + throw new Exception("Please verify values for:\n ranger.usersync.group.memberattributename\n " + "ranger.usersync.group.nameattribute\n" + "ranger.usersync.group.objectclass\n" + "ranger.usersync.group.searchbase\n" + "ranger.usersync.group.searchfilter\n"); } else { throw new Exception(msg + ne); } @@ -403,8 +392,7 @@ private void initUserSync() { } } - /* Use the provided bind dn or the user search base and user search filter for sample user and determine the basic user attribute. - */ + // Use the provided bind dn or the user search base and user search filter for sample user and determine the basic user attribute. private void findBasicUserProperties(LdapContext ldapContext, boolean isOutputNeeded) throws Throwable { String bindDn = config.getLdapBindDn(); String userSFilter = config.getUserSearchFilter(); @@ -428,7 +416,6 @@ private void findBasicUserProperties(LdapContext ldapContext, boolean isOutputNe //int dcIndex = bindDn.toLowerCase().indexOf("dc="); userSBase = bindDn.substring(bindDn.indexOf(",") + 1); } - //System.out.println("Derived user search base = " + userSearchBase); } if (userSFilter == null || userSFilter.isEmpty()) { @@ -438,13 +425,11 @@ private void findBasicUserProperties(LdapContext ldapContext, boolean isOutputNe int cnEndIndex = bindDn.indexOf(","); userSFilter = bindDn.substring(0, cnEndIndex); } - //System.out.println("Derived user search filter = " + userSearchFilter); } try { userSearchResultEnum = ldapContext.search(userSBase, userSFilter, userSearchControls); while (userSearchResultEnum.hasMore()) { - if (noOfUsers >= 5) { break; } @@ -529,12 +514,8 @@ private void findBasicUserProperties(LdapContext ldapContext, boolean isOutputNe noOfUsers++; } } catch (NamingException ne) { - String msg = "Exception occurred while discovering basic user properties:\n" + - "ranger.usersync.ldap.user.nameattribute\n" + - "ranger.usersync.ldap.user.objectclass\n" + - "ranger.usersync.ldap.user.groupnameattribute\n"; - if ((config.getUserSearchBase() != null && !config.getUserSearchBase().isEmpty()) || - (config.getUserSearchFilter() != null && !config.getUserSearchFilter().isEmpty())) { + String msg = "Exception occurred while discovering basic user properties:\n" + "ranger.usersync.ldap.user.nameattribute\n" + "ranger.usersync.ldap.user.objectclass\n" + "ranger.usersync.ldap.user.groupnameattribute\n"; + if ((config.getUserSearchBase() != null && !config.getUserSearchBase().isEmpty()) || (config.getUserSearchFilter() != null && !config.getUserSearchFilter().isEmpty())) { throw new Exception(msg + "Please verify values for ranger.usersync.ldap.user.searchbase and ranger.usersync.ldap.user.searchfilter"); } else { throw new Exception(msg + ne); @@ -620,7 +601,7 @@ private void findAdvUserProperties(LdapContext ldapContext, boolean isOutputNeed dnValue = dnValue.substring(dnValue.indexOf(",") + 1); } } else { - // If distinguishedName is not found, strip off the userName from the long name for OU or sub domain + // If distinguishedName is not found, strip off the userName from the long name for OU or subdomain dnValue = userEntry.getNameInNamespace(); dnValue = dnValue.substring(dnValue.indexOf(",") + 1); } @@ -659,15 +640,11 @@ private void findAdvUserProperties(LdapContext ldapContext, boolean isOutputNeed ambariProps.println("ranger.usersync.ldap.user.searchfilter=" + userSearchFilter); } } catch (NamingException ne) { - String msg = "Exception occured while discovering user properties:\n" + - "ranger.usersync.ldap.user.searchbase\n" + - "ranger.usersync.ldap.user.searchfilter\n"; + String msg = "Exception occured while discovering user properties:\n" + "ranger.usersync.ldap.user.searchbase\n" + "ranger.usersync.ldap.user.searchfilter\n"; if ((config.getUserNameAttribute() != null && !config.getUserNameAttribute().isEmpty()) || (config.getUserObjectClass() != null && !config.getUserObjectClass().isEmpty()) || (config.getGroupNameAttribute() != null && !config.getGroupNameAttribute().isEmpty())) { - throw new Exception("Please verify values for ranger.usersync.ldap.user.nameattribute, " + - "ranger.usersync.ldap.user.objectclass, and" + - "ranger.usersync.ldap.user.groupnameattribute"); + throw new Exception("Please verify values for ranger.usersync.ldap.user.nameattribute, ranger.usersync.ldap.user.objectclass, and ranger.usersync.ldap.user.groupnameattribute"); } else { throw new Exception(msg + ne); } @@ -815,7 +792,7 @@ private void findAdvGroupProperties(LdapContext ldapContext) throws Throwable { dnValue = dnValue.substring(dnValue.indexOf(",") + 1); } } else { - // If distinguishedName is not found, strip off the userName from the long name for OU or sub domain + // If distinguishedName is not found, strip off the userName from the long name for OU or subdomain dnValue = groupEntry.getNameInNamespace(); dnValue = dnValue.substring(dnValue.indexOf(",") + 1); } diff --git a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/CustomSSLSocketFactory.java b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/CustomSSLSocketFactory.java index d7f71cb494..cb32756478 100644 --- a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/CustomSSLSocketFactory.java +++ b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/CustomSSLSocketFactory.java @@ -44,7 +44,7 @@ public class CustomSSLSocketFactory extends SSLSocketFactory { private static final Logger LOG = LoggerFactory.getLogger(CustomSSLSocketFactory.class); - private SSLSocketFactory sockFactory; + private SSLSocketFactory sockFactory; public CustomSSLSocketFactory() { SSLContext sslContext; diff --git a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java index a517ff9d0d..056d289f6e 100644 --- a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java @@ -70,26 +70,30 @@ public class LdapUserGroupBuilder implements UserGroupSource { private static final String DATE_FORMAT = "yyyyMMddHHmmss"; private static final String MEMBER_OF_ATTR = "memberof="; private static final String GROUP_NAME_ATTRIBUTE = "cn="; - private static final int PAGE_SIZE = 500; + private static final int PAGE_SIZE = 500; + /* for AD uSNChanged */ - private static long deltaSyncUserTime; - private static long deltaSyncGroupTime; + private static long deltaSyncUserTime; + private static long deltaSyncGroupTime; /* ***************** */ - private boolean pagedResultsEnabled = true; - private boolean groupSearchFirstEnabled = true; - private boolean userSearchEnabled = true; - private boolean groupSearchEnabled = true; - private int pagedResultsSize = PAGE_SIZE; - private int groupHierarchyLevels; - private int deleteCycles; - private int userSearchScope; - private int groupSearchScope; private final UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); + + private boolean pagedResultsEnabled = true; + private boolean groupSearchFirstEnabled = true; + private boolean userSearchEnabled = true; + private boolean groupSearchEnabled = true; + private int pagedResultsSize = PAGE_SIZE; + private int groupHierarchyLevels; + private int deleteCycles; + private int userSearchScope; + private int groupSearchScope; + /* for OpenLdap modifyTimestamp */ - private String deltaSyncUserTimeStamp; + private String deltaSyncUserTimeStamp; private String deltaSyncGroupTimeStamp; /* ******************************** */ + private String ldapUrl; private String ldapBindDn; private String ldapBindPassword; @@ -118,13 +122,16 @@ public class LdapUserGroupBuilder implements UserGroupSource { private LdapContext ldapContext; private SearchControls userSearchControls; private SearchControls groupSearchControls; - private Table groupUserTable; + private Table groupUserTable; + /* { key = user DN, value = map of user attributes {original name, DN, etc.}} */ private Map> sourceUsers; + /* { key = group DN, value = map of group attributes {original name, DN, etc.}} */ private Map> sourceGroups; + /* { key = group DN, value = set of user DNs (members) } */ - private Map> sourceGroupUsers; + private Map> sourceGroupUsers; StartTlsResponse tls; UgsyncAuditInfo ugsyncAuditInfo; @@ -221,7 +228,7 @@ public void updateSink(UserGroupSink sink) throws Throwable { try { sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers, sourceGroupUsers, computeDeletes); DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); - LOG.info("deltaSyncUserTime = {} and highestdeltaSyncUserTime = {}", deltaSyncUserTime, highestdeltaSyncUserTime); + LOG.info("deltaSyncUserTime = {} and highestDeltaSyncUserTime = {}", deltaSyncUserTime, highestdeltaSyncUserTime); if (deltaSyncUserTime < highestdeltaSyncUserTime) { // Incrementing highestdeltaSyncUserTime (for AD) in order to avoid search record repetition for next sync cycle. deltaSyncUserTime = highestdeltaSyncUserTime + 1; @@ -229,7 +236,7 @@ public void updateSink(UserGroupSink sink) throws Throwable { deltaSyncUserTimeStamp = dateFormat.format(new Date(highestdeltaSyncUserTime + 60L)); } - LOG.info("deltaSyncGroupTime = {} and highestdeltaSyncGroupTime = {} ", deltaSyncGroupTime, highestdeltaSyncGroupTime); + LOG.info("deltaSyncGroupTime = {} and highestDeltaSyncGroupTime = {} ", deltaSyncGroupTime, highestdeltaSyncGroupTime); // Update deltaSyncUserTime/deltaSyncUserTimeStamp here so that in case of failures, we get updates in next cycle if (deltaSyncGroupTime < highestdeltaSyncGroupTime) { // Incrementing highestdeltaSyncGroupTime (for AD) in order to avoid search record repetition for next sync cycle. @@ -480,13 +487,30 @@ private long getUsers(boolean computeDeletes) throws Throwable { // searchResults contains all the user entries final SearchResult userEntry = userSearchResultEnum.next(); - if (!isSearchResultValid(userEntry, false)) { + if (userEntry == null) { + LOG.info("userEntry null, skipping sync for the entry"); continue; } - Attributes attributes = userEntry.getAttributes(); - String userName = (String) attributes.get(userNameAttribute).get(); - String userFullName = userEntry.getNameInNamespace(); + Attributes attributes = userEntry.getAttributes(); + if (attributes == null) { + LOG.info("attributes missing for entry {}, skipping sync", userEntry.getNameInNamespace()); + continue; + } + + Attribute userNameAttr = attributes.get(userNameAttribute); + if (userNameAttr == null) { + LOG.info("{} missing for entry {}, skipping sync", userNameAttribute, userEntry.getNameInNamespace()); + continue; + } + + String userFullName = (userEntry.getNameInNamespace()); + String userName = (String) userNameAttr.get(); + + if (userName == null || userName.trim().isEmpty()) { + LOG.info("{} empty for entry {}, skipping sync", userNameAttribute, userEntry.getNameInNamespace()); + continue; + } Attribute timeStampAttr = attributes.get("uSNChanged"); if (timeStampAttr != null) { @@ -618,7 +642,7 @@ private long getUsers(boolean computeDeletes) throws Throwable { closeLdapContext(); } - LOG.debug("highestdeltaSyncUserTime = {}", highestdeltaSyncUserTime); + LOG.debug("highestDeltaSyncUserTime = {}", highestdeltaSyncUserTime); return highestdeltaSyncUserTime; } @@ -1172,50 +1196,4 @@ private String getDNForMemberOf(String searchFilter) throws Throwable { return computedSearchFilter; } - - private boolean isSearchResultValid(SearchResult searchResult, boolean isGroup) throws NamingException { - boolean result = true; - if (searchResult == null) { - LOG.debug("searchResult is null!"); - result = false; - } else { - Attributes attributes = searchResult.getAttributes(); - if (attributes == null) { - LOG.debug("Attributes missing for entry {}", searchResult.getNameInNamespace()); - result = false; - } else { - if (!isGroup) { - Attribute userNameAttr = attributes.get(userNameAttribute); - if (userNameAttr == null) { - LOG.debug("{} missing for entry {}", userNameAttribute, searchResult.getNameInNamespace()); - result = false; - } else { - String userName = (String) userNameAttr.get(); - - if (userName == null || userName.trim().isEmpty()) { - LOG.debug("{} empty for entry {}", userNameAttribute, searchResult.getNameInNamespace()); - result = false; - } - } - } else { - Attribute groupNameAttr = attributes.get(groupNameAttribute); - if (groupNameAttr == null) { - LOG.debug("{} missing for entry {}", groupNameAttribute, searchResult.getNameInNamespace()); - result = false; - } else { - String groupName = (String) groupNameAttr.get(); - - if (groupName == null || groupName.trim().isEmpty()) { - LOG.debug("{} empty for entry {}", groupNameAttr, searchResult.getNameInNamespace()); - result = false; - } - } - } - } - } - if (!result) { - LOG.debug("Skipping sync!"); - } - return result; - } } diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java index d411a85409..ae39859909 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java @@ -43,165 +43,202 @@ import java.util.StringTokenizer; public class UserGroupSyncConfig { - private static final Logger LOG = LoggerFactory.getLogger(UserGroupSyncConfig.class); - - private static final String CORE_SITE_CONFIG_FILE = "core-site.xml"; - public static final String CONFIG_FILE = "ranger-ugsync-site.xml"; - public static final String DEFAULT_CONFIG_FILE = "ranger-ugsync-default.xml"; - - private static final String UGSYNC_SOURCE_CLASS_PARAM = "ranger.usersync.source.impl.class"; - private static final String UGSYNC_SINK_CLASS_PARAM = "ranger.usersync.sink.impl.class"; - private static final String UGSYNC_SOURCE_CLASS = "org.apache.ranger.unixusersync.process.UnixUserGroupBuilder"; - private static final String UGSYNC_SINK_CLASS = "org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder"; - private static final String LGSYNC_SOURCE_CLASS = "org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder"; - - public static final String UGSYNC_ENABLED_PROP = "ranger.usersync.enabled"; - public static final String UGSYNC_PM_URL_PROP = "ranger.usersync.policymanager.baseURL"; - public static final String UGSYNC_UNIX_PASSWORD_FILE = "ranger.usersync.unix.password.file"; - public static final String DEFAULT_UGSYNC_UNIX_PASSWORD_FILE = "/etc/passwd"; - public static final String UGSYNC_UNIX_GROUP_FILE = "ranger.usersync.unix.group.file"; - public static final String DEFAULT_UGSYNC_UNIX_GROUP_FILE = "/etc/group"; - public static final String UGSYNC_MIN_USERID_PROP = "ranger.usersync.unix.minUserId"; - public static final String UGSYNC_MIN_GROUPID_PROP = "ranger.usersync.unix.minGroupId"; - public static final String DEFAULT_UGSYNC_MIN_GROUPID = "0"; - public static final String UGSYNC_MAX_RECORDS_PER_API_CALL_PROP = "ranger.usersync.policymanager.maxrecordsperapicall"; - public static final String UGSYNC_MOCK_RUN_PROP = "ranger.usersync.policymanager.mockrun"; - public static final String UGSYNC_TEST_RUN_PROP = "ranger.usersync.policymanager.testrun"; - public static final String UGSYNC_SOURCE_FILE_PROC = "ranger.usersync.filesource.file"; - public static final String UGSYNC_SOURCE_FILE_DELIMITER = "ranger.usersync.filesource.text.delimiter"; - public static final String UGSYNC_SOURCE_FILE_DELIMITERER = "ranger.usersync.filesource.text.delimiterer"; - public static final String UGSYNC_NONE_CASE_CONVERSION_VALUE = "none"; - public static final String UGSYNC_LOWER_CASE_CONVERSION_VALUE = "lower"; - public static final String UGSYNC_UPPER_CASE_CONVERSION_VALUE = "upper"; - public static final String SYNC_MAPPING_USERNAME = "ranger.usersync.mapping.username.regex"; - public static final String SYNC_MAPPING_GROUPNAME = "ranger.usersync.mapping.groupname.regex"; - public static final String UGSYNC_METRICS_ENABLED_PROP = "ranger.usersync.metrics.enabled"; - public static final String UGSYNC_NAME_VALIDATION_ENABLED = "ranger.usersync.name.validation.enabled"; - public static final String UGSYNC_SERVER_HA_ENABLED_PARAM = "ranger-ugsync.server.ha.enabled"; - public static final String UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED = "ranger.usersync.syncsource.validation.enabled"; - - private static final String SSL_KEYSTORE_FILE_TYPE_PARAM = "ranger.keystore.file.type"; - private static final String SSL_TRUSTSTORE_FILE_TYPE_PARAM = "ranger.truststore.file.type"; - private static final String SSL_KEYSTORE_PATH_PARAM = "ranger.usersync.keystore.file"; - private static final String SSL_KEYSTORE_PATH_PASSWORD_PARAM = "ranger.usersync.keystore.password"; - private static final String SSL_KEYSTORE_PATH_PASSWORD_ALIAS = "usersync.ssl.key.password"; - private static final String SSL_TRUSTSTORE_PATH_PARAM = "ranger.usersync.truststore.file"; - private static final String SSL_TRUSTSTORE_PATH_PASSWORD_PARAM = "ranger.usersync.truststore.password"; - private static final String SSL_TRUSTSTORE_PATH_PASSWORD_ALIAS = "usersync.ssl.truststore.password"; - - private static final String UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM = "ranger.usersync.sleeptimeinmillisbetweensynccycle"; - private static final String UGSYNC_SLEEP_LDAP_FORCE_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM_ENABLED = "ranger.usersync.ldap.force.sleeptimeinmillisbetweensynccycle.enabled"; - private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE = 60000L; - private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_UNIX_DEFAULT_VALUE = 60000L; - private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_LDAP_DEFAULT_VALUE = 3600000L; - - private static final String LGSYNC_LDAP_URL = "ranger.usersync.ldap.url"; - private static final String LGSYNC_LDAP_DELTASYNC_ENABLED = "ranger.usersync.ldap.deltasync"; - private static final boolean DEFAULT_LGSYNC_LDAP_DELTASYNC_ENABLED = false; - private static final String LGSYNC_LDAP_STARTTLS_ENABLED = "ranger.usersync.ldap.starttls"; - private static final boolean DEFAULT_LGSYNC_LDAP_STARTTLS_ENABLED = false; - private static final String LGSYNC_LDAP_BIND_DN = "ranger.usersync.ldap.binddn"; - private static final String LGSYNC_LDAP_BIND_KEYSTORE = "ranger.usersync.credstore.filename"; - private static final String LGSYNC_LDAP_BIND_ALIAS = "ranger.usersync.ldap.bindalias"; - private static final String LGSYNC_LDAP_BIND_PASSWORD = "ranger.usersync.ldap.ldapbindpassword"; - private static final String LGSYNC_LDAP_AUTHENTICATION_MECHANISM = "ranger.usersync.ldap.authentication.mechanism"; - private static final String DEFAULT_AUTHENTICATION_MECHANISM = "simple"; - private static final String LGSYNC_SEARCH_BASE = "ranger.usersync.ldap.searchBase"; - private static final String LGSYNC_USER_SEARCH_BASE = "ranger.usersync.ldap.user.searchbase"; - private static final String LGSYNC_USER_SEARCH_SCOPE = "ranger.usersync.ldap.user.searchscope"; - private static final String LGSYNC_USER_OBJECT_CLASS = "ranger.usersync.ldap.user.objectclass"; - private static final String DEFAULT_USER_OBJECT_CLASS = "person"; - private static final String LGSYNC_GROUPNAMES = "ranger.usersync.ldap.groupnames"; - private static final String LGSYNC_USER_SEARCH_FILTER = "ranger.usersync.ldap.user.searchfilter"; - private static final String LGSYNC_USER_NAME_ATTRIBUTE = "ranger.usersync.ldap.user.nameattribute"; - private static final String DEFAULT_USER_NAME_ATTRIBUTE = "cn"; - private static final String LGSYNC_USER_GROUP_NAME_ATTRIBUTE = "ranger.usersync.ldap.user.groupnameattribute"; - private static final String DEFAULT_USER_GROUP_NAME_ATTRIBUTE = "memberof,ismemberof"; - private static final String LGSYNC_USER_CLOUDID_ATTRIBUTE = "ranger.usersync.ldap.user.cloudid.attribute"; - private static final String DEFAULT_USER_CLOUDID_ATTRIBUTE = "objectid"; - private static final String LGSYNC_USER_CLOUDID_ATTRIBUTE_DATATYPE = "ranger.usersync.ldap.user.cloudid.attribute.datatype"; - private static final String DEFAULT_USER_CLOUDID_ATTRIBUTE_DATATYPE = "byte[]"; - private static final String LGSYNC_OTHER_USER_ATTRIBUTES = "ranger.usersync.ldap.user.otherattributes"; - private static final String DEFAULT_OTHER_USER_ATTRIBUTES = "userurincipaluame,"; - private static final String UGSYNC_USERNAME_CASE_CONVERSION_PARAM = "ranger.usersync.ldap.username.caseconversion"; - private static final String DEFAULT_UGSYNC_USERNAME_CASE_CONVERSION_VALUE = UGSYNC_NONE_CASE_CONVERSION_VALUE; - private static final String UGSYNC_GROUPNAME_CASE_CONVERSION_PARAM = "ranger.usersync.ldap.groupname.caseconversion"; - private static final String DEFAULT_UGSYNC_GROUPNAME_CASE_CONVERSION_VALUE = UGSYNC_NONE_CASE_CONVERSION_VALUE; - private static final String DEFAULT_USER_GROUP_TEXTFILE_DELIMITER = ","; - private static final String LGSYNC_PAGED_RESULTS_ENABLED = "ranger.usersync.pagedresultsenabled"; - private static final boolean DEFAULT_LGSYNC_PAGED_RESULTS_ENABLED = true; - private static final String LGSYNC_PAGED_RESULTS_SIZE = "ranger.usersync.pagedresultssize"; - private static final int DEFAULT_LGSYNC_PAGED_RESULTS_SIZE = 500; - private static final String LGSYNC_GROUP_SEARCH_ENABLED = "ranger.usersync.group.searchenabled"; - private static final boolean DEFAULT_LGSYNC_GROUP_SEARCH_ENABLED = true; - private static final String LGSYNC_GROUP_SEARCH_FIRST_ENABLED = "ranger.usersync.group.search.first.enabled"; - private static final boolean DEFAULT_LGSYNC_GROUP_SEARCH_FIRST_ENABLED = true; - /** This flag (ranger.usersync.user.searchenabled) is used only when group search first is enabled to get username either - - * from the group member attribute of the group or - * from the additional user search based on the user attribute configuration + private static final Logger LOG = LoggerFactory.getLogger(UserGroupSyncConfig.class); + + private static final String CORE_SITE_CONFIG_FILE = "core-site.xml"; + + public static final String CONFIG_FILE = "ranger-ugsync-site.xml"; + public static final String DEFAULT_CONFIG_FILE = "ranger-ugsync-default.xml"; + public static final String UGSYNC_ENABLED_PROP = "ranger.usersync.enabled"; + + private static final String UGSYNC_SOURCE_CLASS_PARAM = "ranger.usersync.source.impl.class"; + private static final String UGSYNC_SINK_CLASS_PARAM = "ranger.usersync.sink.impl.class"; + private static final String UGSYNC_SOURCE_CLASS = "org.apache.ranger.unixusersync.process.UnixUserGroupBuilder"; + private static final String UGSYNC_SINK_CLASS = "org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder"; + private static final String LGSYNC_SOURCE_CLASS = "org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder"; + + /* LDAP Configs */ + private static final String LGSYNC_LDAP_URL = "ranger.usersync.ldap.url"; + private static final String LGSYNC_LDAP_AUTHENTICATION_MECHANISM = "ranger.usersync.ldap.authentication.mechanism"; + private static final String LGSYNC_REFERRAL = "ranger.usersync.ldap.referral"; + private static final String LGSYNC_LDAP_BIND_DN = "ranger.usersync.ldap.binddn"; + private static final String LGSYNC_LDAP_BIND_ALIAS = "ranger.usersync.ldap.bindalias"; + private static final String LGSYNC_LDAP_BIND_PASSWORD = "ranger.usersync.ldap.ldapbindpassword"; + private static final String LGSYNC_SEARCH_BASE = "ranger.usersync.ldap.searchBase"; + private static final String LGSYNC_USER_SEARCH_BASE = "ranger.usersync.ldap.user.searchbase"; + private static final String LGSYNC_USER_SEARCH_SCOPE = "ranger.usersync.ldap.user.searchscope"; + private static final String LGSYNC_USER_OBJECT_CLASS = "ranger.usersync.ldap.user.objectclass"; + private static final String LGSYNC_USER_SEARCH_FILTER = "ranger.usersync.ldap.user.searchfilter"; + private static final String LGSYNC_USER_NAME_ATTRIBUTE = "ranger.usersync.ldap.user.nameattribute"; + private static final String LGSYNC_USER_GROUP_NAME_ATTRIBUTE = "ranger.usersync.ldap.user.groupnameattribute"; + private static final String LGSYNC_OTHER_USER_ATTRIBUTES = "ranger.usersync.ldap.user.otherattributes"; + private static final String LGSYNC_USER_CLOUDID_ATTRIBUTE = "ranger.usersync.ldap.user.cloudid.attribute"; + private static final String LGSYNC_USER_CLOUDID_ATTRIBUTE_DATATYPE = "ranger.usersync.ldap.user.cloudid.attribute.datatype"; + private static final String UGSYNC_USERNAME_CASE_CONVERSION_PARAM = "ranger.usersync.ldap.username.caseconversion"; + /** + * ranger.usersync.user.searchenabled is used only when group search first is enabled to get username from - + * 1. the group member attribute of the group or + * 2. the additional user search based on the user attribute configuration */ - private static final String LGSYNC_USER_SEARCH_ENABLED = "ranger.usersync.user.searchenabled"; - private static final boolean DEFAULT_LGSYNC_USER_SEARCH_ENABLED = true; - private static final String LGSYNC_GROUP_SEARCH_BASE = "ranger.usersync.group.searchbase"; - private static final String LGSYNC_GROUP_SEARCH_SCOPE = "ranger.usersync.group.searchscope"; - private static final String LGSYNC_GROUP_OBJECT_CLASS = "ranger.usersync.group.objectclass"; - private static final String DEFAULT_LGSYNC_GROUP_OBJECT_CLASS = "groupofnames"; - private static final String LGSYNC_GROUP_SEARCH_FILTER = "ranger.usersync.group.searchfilter"; - private static final String LGSYNC_GROUP_NAME_ATTRIBUTE = "ranger.usersync.group.nameattribute"; - private static final String DEFAULT_LGSYNC_GROUP_NAME_ATTRIBUTE = "cn"; - private static final String LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = "ranger.usersync.group.memberattributename"; - private static final String DEFAULT_LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = "member"; - private static final String LGSYNC_GROUP_CLOUDID_ATTRIBUTE = "ranger.usersync.ldap.group.cloudid.attribute"; - private static final String DEFAULT_GROUP_CLOUDID_ATTRIBUTE = "objectid"; - private static final String LGSYNC_GROUP_CLOUDID_ATTRIBUTE_DATATYPE = "ranger.usersync.ldap.group.cloudid.attribute.datatype"; - private static final String DEFAULT_GROUP_CLOUDID_ATTRIBUTE_DATATYPE = "byte[]"; - private static final String LGSYNC_OTHER_GROUP_ATTRIBUTES = "ranger.usersync.ldap.group.otherattributes"; - private static final String DEFAULT_OTHER_GROUP_ATTRIBUTES = "displayname,"; - private static final String LGSYNC_GROUP_HIERARCHY_LEVELS = "ranger.usersync.ldap.grouphierarchylevels"; - private static final int DEFAULT_LGSYNC_GROUP_HIERARCHY_LEVELS = 0; - private static final String UGSYNC_UPDATE_MILLIS_MIN = "ranger.usersync.unix.updatemillismin"; - private static final long DEFAULT_UGSYNC_UPDATE_MILLIS_MIN = 60_000; // ms - private static final String UGSYNC_UNIX_BACKEND = "ranger.usersync.unix.backend"; - private static final String DEFAULT_UGSYNC_UNIX_BACKEND = "passwd"; - private static final String UGSYNC_GROUP_ENUMERATE_ENABLED = "ranger.usersync.group.enumerate"; - private static final String UGSYNC_GROUP_ENUMERATE_GROUPS = "ranger.usersync.group.enumerategroup"; - private static final String SYNC_POLICY_MGR_KEYSTORE = "ranger.usersync.policymgr.keystore"; - private static final String SYNC_POLICY_MGR_ALIAS = "ranger.usersync.policymgr.alias"; - private static final String SYNC_POLICY_MGR_PASSWORD = "ranger.usersync.policymgr.password"; - private static final String SYNC_POLICY_MGR_USERNAME = "ranger.usersync.policymgr.username"; - private static final String SYNC_POLICY_MGR_MAX_RETRY_ATTEMPTS = "ranger.usersync.policymgr.max.retry.attempts"; - private static final String SYNC_POLICY_MGR_RETRY_INTERVAL_MS = "ranger.usersync.policymgr.retry.interval.ms"; - private static final String DEFAULT_POLICYMGR_USERNAME = "rangerusersync"; - private static final String SYNC_SOURCE = "ranger.usersync.sync.source"; - private static final String LGSYNC_REFERRAL = "ranger.usersync.ldap.referral"; - private static final String DEFAULT_LGSYNC_REFERRAL = "ignore"; - private static final String SYNC_MAPPING_USERNAME_HANDLER = "ranger.usersync.mapping.username.handler"; - private static final String DEFAULT_SYNC_MAPPING_USERNAME_HANDLER = "org.apache.ranger.usergroupsync.RegEx"; - private static final String SYNC_MAPPING_GROUPNAME_HANDLER = "ranger.usersync.mapping.groupname.handler"; - private static final String DEFAULT_SYNC_MAPPING_GROUPNAME_HANDLER = "org.apache.ranger.usergroupsync.RegEx"; - private static final String SYNC_MAPPING_SEPARATOR = "ranger.usersync.mapping.regex.separator"; - private static final String DEFAULT_MAPPING_SEPARATOR = "/"; - private static final String ROLE_ASSIGNMENT_LIST_DELIMITER = "ranger.usersync.role.assignment.list.delimiter"; - private static final String USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER = "ranger.usersync.users.groups.assignment.list.delimiter"; - private static final String USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER = "ranger.usersync.username.groupname.assignment.list.delimiter"; - private static final String GROUP_BASED_ROLE_ASSIGNMENT_RULES = "ranger.usersync.group.based.role.assignment.rules"; - private static final String WHITELIST_USER_ROLE_ASSIGNMENT_RULES = "ranger.usersync.whitelist.users.role.assignment.rules"; - private static final String DEFAULT_WHITELIST_USER_ROLE_ASSIGNMENT_RULES = "&ROLE_SYS_ADMIN:u:admin,rangerusersync,rangertagsync&ROLE_KEY_ADMIN:u:keyadmin"; - private static final String USERSYNC_RANGER_COOKIE_ENABLED_PROP = "ranger.usersync.cookie.enabled"; - private static final String RANGER_ADMIN_COOKIE_NAME_PROPS = "ranger.usersync.dest.ranger.session.cookie.name"; - private static final String UGSYNC_METRICS_FILEPATH = "ranger.usersync.metrics.filepath"; - private static final String DEFAULT_UGSYNC_METRICS_FILEPATH = "/tmp/"; - private static final String UGSYNC_METRICS_FILENAME = "ranger.usersync.metrics.filename"; - private static final String DEFAULT_UGSYNC_METRICS_FILENAME = "ranger_usersync_metric.json"; - private static final String UGSYNC_METRICS_FREQUENCY_TIME_IN_MILLIS_PARAM = "ranger.usersync.metrics.frequencytimeinmillis"; - private static final long DEFAULT_UGSYNC_METRICS_FREQUENCY_TIME_IN_MILLIS = 10000L; - private static final String UGSYNC_DELETES_ENABLED = "ranger.usersync.deletes.enabled"; - private static final boolean DEFAULT_UGSYNC_DELETES_ENABLED = false; - private static final String UGSYNC_DELETES_FREQUENCY = "ranger.usersync.deletes.frequency"; - private static final long DEFAULT_UGSYNC_DELETES_FREQUENCY = 10L; // After every 10 sync cycles - private static final boolean DEFAULT_UGSYNC_NAME_VALIDATION_ENABLED = false; - private static final long UGSYNC_INIT_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE_FOR_HA = 5000L; - private static final boolean DEFAULT_UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED = true; + private static final String LGSYNC_USER_SEARCH_ENABLED = "ranger.usersync.user.searchenabled"; + + private static final String LGSYNC_GROUP_SEARCH_BASE = "ranger.usersync.group.searchbase"; + private static final String LGSYNC_GROUP_SEARCH_SCOPE = "ranger.usersync.group.searchscope"; + private static final String LGSYNC_GROUP_OBJECT_CLASS = "ranger.usersync.group.objectclass"; + private static final String LGSYNC_GROUP_SEARCH_FILTER = "ranger.usersync.group.searchfilter"; + private static final String LGSYNC_GROUP_NAME_ATTRIBUTE = "ranger.usersync.group.nameattribute"; + private static final String LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = "ranger.usersync.group.memberattributename"; + private static final String LGSYNC_GROUP_SEARCH_ENABLED = "ranger.usersync.group.searchenabled"; + private static final String LGSYNC_GROUP_SEARCH_FIRST_ENABLED = "ranger.usersync.group.search.first.enabled"; + private static final String LGSYNC_GROUPNAMES = "ranger.usersync.ldap.groupnames"; + private static final String LGSYNC_OTHER_GROUP_ATTRIBUTES = "ranger.usersync.ldap.group.otherattributes"; + private static final String LGSYNC_GROUP_CLOUDID_ATTRIBUTE = "ranger.usersync.ldap.group.cloudid.attribute"; + private static final String LGSYNC_GROUP_CLOUDID_ATTRIBUTE_DATATYPE = "ranger.usersync.ldap.group.cloudid.attribute.datatype"; + private static final String UGSYNC_GROUPNAME_CASE_CONVERSION_PARAM = "ranger.usersync.ldap.groupname.caseconversion"; + private static final String LGSYNC_GROUP_HIERARCHY_LEVELS = "ranger.usersync.ldap.grouphierarchylevels"; + + private static final String LGSYNC_LDAP_BIND_KEYSTORE = "ranger.usersync.credstore.filename"; + private static final String LGSYNC_LDAP_DELTASYNC_ENABLED = "ranger.usersync.ldap.deltasync"; + private static final String LGSYNC_LDAP_STARTTLS_ENABLED = "ranger.usersync.ldap.starttls"; + private static final String UGSYNC_SLEEP_LDAP_FORCE_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM_ENABLED = "ranger.usersync.ldap.force.sleeptimeinmillisbetweensynccycle.enabled"; + + /* LDAP Defaults */ + public static final String UGSYNC_NONE_CASE_CONVERSION_VALUE = "none"; + public static final String UGSYNC_LOWER_CASE_CONVERSION_VALUE = "lower"; + public static final String UGSYNC_UPPER_CASE_CONVERSION_VALUE = "upper"; + + private static final String DEFAULT_AUTHENTICATION_MECHANISM = "simple"; + private static final String DEFAULT_USER_OBJECT_CLASS = "person"; + private static final String DEFAULT_USER_NAME_ATTRIBUTE = "cn"; + private static final String DEFAULT_USER_GROUP_NAME_ATTRIBUTE = "memberof,ismemberof"; + private static final String DEFAULT_USER_CLOUDID_ATTRIBUTE = "objectid"; + private static final String DEFAULT_USER_CLOUDID_ATTRIBUTE_DATATYPE = "byte[]"; + private static final String DEFAULT_OTHER_USER_ATTRIBUTES = "userurincipaluame,"; + private static final String DEFAULT_UGSYNC_USERNAME_CASE_CONVERSION_VALUE = UGSYNC_NONE_CASE_CONVERSION_VALUE; + + private static final String DEFAULT_LGSYNC_GROUP_OBJECT_CLASS = "groupofnames"; + private static final String DEFAULT_LGSYNC_GROUP_NAME_ATTRIBUTE = "cn"; + private static final String DEFAULT_LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = "member"; + private static final String DEFAULT_GROUP_CLOUDID_ATTRIBUTE = "objectid"; + private static final String DEFAULT_GROUP_CLOUDID_ATTRIBUTE_DATATYPE = "byte[]"; + private static final String DEFAULT_OTHER_GROUP_ATTRIBUTES = "displayname,"; + private static final String DEFAULT_UGSYNC_GROUPNAME_CASE_CONVERSION_VALUE = UGSYNC_NONE_CASE_CONVERSION_VALUE; + private static final String DEFAULT_LGSYNC_REFERRAL = "ignore"; + private static final int DEFAULT_LGSYNC_GROUP_HIERARCHY_LEVELS = 0; + private static final int DEFAULT_LGSYNC_PAGED_RESULTS_SIZE = 500; + private static final boolean DEFAULT_LGSYNC_LDAP_DELTASYNC_ENABLED = false; + private static final boolean DEFAULT_LGSYNC_LDAP_STARTTLS_ENABLED = false; + private static final boolean DEFAULT_LGSYNC_PAGED_RESULTS_ENABLED = true; + private static final boolean DEFAULT_LGSYNC_GROUP_SEARCH_ENABLED = true; + private static final boolean DEFAULT_LGSYNC_USER_SEARCH_ENABLED = true; + private static final boolean DEFAULT_LGSYNC_GROUP_SEARCH_FIRST_ENABLED = true; + + /* Unix Configs */ + public static final String UGSYNC_MIN_USERID_PROP = "ranger.usersync.unix.minUserId"; + public static final String UGSYNC_MIN_GROUPID_PROP = "ranger.usersync.unix.minGroupId"; + public static final String UGSYNC_UNIX_PASSWORD_FILE = "ranger.usersync.unix.password.file"; + public static final String UGSYNC_UNIX_GROUP_FILE = "ranger.usersync.unix.group.file"; + + private static final String UGSYNC_GROUP_ENUMERATE_ENABLED = "ranger.usersync.group.enumerate"; + private static final String UGSYNC_GROUP_ENUMERATE_GROUPS = "ranger.usersync.group.enumerategroup"; + private static final String UGSYNC_UNIX_BACKEND = "ranger.usersync.unix.backend"; + + /* Unix Defaults */ + public static final String DEFAULT_UGSYNC_UNIX_GROUP_FILE = "/etc/group"; + public static final String DEFAULT_UGSYNC_UNIX_PASSWORD_FILE = "/etc/passwd"; + public static final String DEFAULT_UGSYNC_MIN_GROUPID = "0"; + + private static final String DEFAULT_UGSYNC_UNIX_BACKEND = "passwd"; + private static final String UGSYNC_UPDATE_MILLIS_MIN = "ranger.usersync.unix.updatemillismin"; + + /* File Sync Configs */ + public static final String UGSYNC_SOURCE_FILE_PROC = "ranger.usersync.filesource.file"; + public static final String UGSYNC_SOURCE_FILE_DELIMITER = "ranger.usersync.filesource.text.delimiter"; + public static final String UGSYNC_SOURCE_FILE_DELIMITERER = "ranger.usersync.filesource.text.delimiterer"; + + private static final String DEFAULT_USER_GROUP_TEXTFILE_DELIMITER = ","; + + /* RegEx */ + public static final String SYNC_MAPPING_USERNAME = "ranger.usersync.mapping.username.regex"; + public static final String SYNC_MAPPING_GROUPNAME = "ranger.usersync.mapping.groupname.regex"; + + private static final String SYNC_MAPPING_USERNAME_HANDLER = "ranger.usersync.mapping.username.handler"; + private static final String SYNC_MAPPING_GROUPNAME_HANDLER = "ranger.usersync.mapping.groupname.handler"; + private static final String SYNC_MAPPING_SEPARATOR = "ranger.usersync.mapping.regex.separator"; + + private static final String DEFAULT_SYNC_MAPPING_USERNAME_HANDLER = "org.apache.ranger.usergroupsync.RegEx"; + private static final String DEFAULT_SYNC_MAPPING_GROUPNAME_HANDLER = "org.apache.ranger.usergroupsync.RegEx"; + + private static final String DEFAULT_MAPPING_SEPARATOR = "/"; + + /* Role Assignments */ + private static final String ROLE_ASSIGNMENT_LIST_DELIMITER = "ranger.usersync.role.assignment.list.delimiter"; + private static final String USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER = "ranger.usersync.users.groups.assignment.list.delimiter"; + private static final String USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER = "ranger.usersync.username.groupname.assignment.list.delimiter"; + private static final String GROUP_BASED_ROLE_ASSIGNMENT_RULES = "ranger.usersync.group.based.role.assignment.rules"; + private static final String WHITELIST_USER_ROLE_ASSIGNMENT_RULES = "ranger.usersync.whitelist.users.role.assignment.rules"; + private static final String DEFAULT_WHITELIST_USER_ROLE_ASSIGNMENT_RULES = "&ROLE_SYS_ADMIN:u:admin,rangerusersync,rangertagsync&ROLE_KEY_ADMIN:u:keyadmin"; + + /* Metrics */ + public static final String UGSYNC_METRICS_ENABLED_PROP = "ranger.usersync.metrics.enabled"; + + private static final String UGSYNC_METRICS_FILENAME = "ranger.usersync.metrics.filename"; + private static final String DEFAULT_UGSYNC_METRICS_FILENAME = "ranger_usersync_metric.json"; + private static final String UGSYNC_METRICS_FILEPATH = "ranger.usersync.metrics.filepath"; + private static final String UGSYNC_METRICS_FREQUENCY_TIME_IN_MILLIS_PARAM = "ranger.usersync.metrics.frequencytimeinmillis"; + + private static final String DEFAULT_UGSYNC_METRICS_FILEPATH = "/tmp/"; + private static final long DEFAULT_UGSYNC_METRICS_FREQUENCY_TIME_IN_MILLIS = 10_000L; + + /* Policy Manager Configs */ + public static final String UGSYNC_PM_URL_PROP = "ranger.usersync.policymanager.baseURL"; + public static final String UGSYNC_MAX_RECORDS_PER_API_CALL_PROP = "ranger.usersync.policymanager.maxrecordsperapicall"; + public static final String UGSYNC_MOCK_RUN_PROP = "ranger.usersync.policymanager.mockrun"; + public static final String UGSYNC_TEST_RUN_PROP = "ranger.usersync.policymanager.testrun"; + + private static final String SYNC_POLICY_MGR_KEYSTORE = "ranger.usersync.policymgr.keystore"; + private static final String SYNC_POLICY_MGR_ALIAS = "ranger.usersync.policymgr.alias"; + private static final String SYNC_POLICY_MGR_PASSWORD = "ranger.usersync.policymgr.password"; + private static final String SYNC_POLICY_MGR_USERNAME = "ranger.usersync.policymgr.username"; + private static final String SYNC_POLICY_MGR_MAX_RETRY_ATTEMPTS = "ranger.usersync.policymgr.max.retry.attempts"; + private static final String SYNC_POLICY_MGR_RETRY_INTERVAL_MS = "ranger.usersync.policymgr.retry.interval.ms"; + private static final String DEFAULT_POLICYMGR_USERNAME = "rangerusersync"; + + /* Other Configs */ + public static final String UGSYNC_SERVER_HA_ENABLED_PARAM = "ranger-ugsync.server.ha.enabled"; + public static final String UGSYNC_NAME_VALIDATION_ENABLED = "ranger.usersync.name.validation.enabled"; + public static final String UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED = "ranger.usersync.syncsource.validation.enabled"; + private static final String UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM = "ranger.usersync.sleeptimeinmillisbetweensynccycle"; + + private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE = 60_000L; + private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_UNIX_DEFAULT_VALUE = 60_000L; + private static final long UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_LDAP_DEFAULT_VALUE = 3_600_000L; + private static final long DEFAULT_UGSYNC_UPDATE_MILLIS_MIN = 60_000; + private static final long UGSYNC_INIT_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE_FOR_HA = 5_000L; + private static final boolean DEFAULT_UGSYNC_NAME_VALIDATION_ENABLED = false; + private static final boolean DEFAULT_UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED = true; + + private static final String SYNC_SOURCE = "ranger.usersync.sync.source"; + private static final String LGSYNC_PAGED_RESULTS_ENABLED = "ranger.usersync.pagedresultsenabled"; + private static final String LGSYNC_PAGED_RESULTS_SIZE = "ranger.usersync.pagedresultssize"; + private static final String UGSYNC_DELETES_ENABLED = "ranger.usersync.deletes.enabled"; + private static final String UGSYNC_DELETES_FREQUENCY = "ranger.usersync.deletes.frequency"; + private static final String USERSYNC_RANGER_COOKIE_ENABLED_PROP = "ranger.usersync.cookie.enabled"; + private static final String RANGER_ADMIN_COOKIE_NAME_PROPS = "ranger.usersync.dest.ranger.session.cookie.name"; + + private static final boolean DEFAULT_UGSYNC_DELETES_ENABLED = false; + private static final long DEFAULT_UGSYNC_DELETES_FREQUENCY = 10L; // After every 10 sync cycles + + /* SSL Configs */ + private static final String SSL_KEYSTORE_FILE_TYPE_PARAM = "ranger.keystore.file.type"; + private static final String SSL_TRUSTSTORE_FILE_TYPE_PARAM = "ranger.truststore.file.type"; + private static final String SSL_KEYSTORE_PATH_PARAM = "ranger.usersync.keystore.file"; + private static final String SSL_KEYSTORE_PATH_PASSWORD_PARAM = "ranger.usersync.keystore.password"; + private static final String SSL_TRUSTSTORE_PATH_PARAM = "ranger.usersync.truststore.file"; + private static final String SSL_TRUSTSTORE_PATH_PASSWORD_PARAM = "ranger.usersync.truststore.password"; + private static final String SSL_KEYSTORE_PATH_PASSWORD_ALIAS = "usersync.ssl.key.password"; + private static final String SSL_TRUSTSTORE_PATH_PASSWORD_ALIAS = "usersync.ssl.truststore.password"; + private static volatile UserGroupSyncConfig me; private final Properties prop = new Properties(); private Configuration userGroupConfig; @@ -342,19 +379,15 @@ public String getSSLKeyStorePath() { } public String getSSLKeyStorePathPassword() { - if (prop == null) { - return null; - } if (prop.containsKey(LGSYNC_LDAP_BIND_KEYSTORE)) { String path = prop.getProperty(LGSYNC_LDAP_BIND_KEYSTORE); - String alias = SSL_KEYSTORE_PATH_PASSWORD_ALIAS; if (path != null) { - if (!path.trim().isEmpty() && !alias.trim().isEmpty()) { + if (!path.trim().isEmpty()) { if ("bcfks".equalsIgnoreCase(getSSLKeyStoreType())) { - String crendentialProviderPrefixBcfks = "bcfks" + "://file"; + String crendentialProviderPrefixBcfks = "bcfks://file"; path = crendentialProviderPrefixBcfks + path; } - String password = CredentialReader.getDecryptedString(path.trim(), alias.trim(), getSSLKeyStoreType()); + String password = CredentialReader.getDecryptedString(path.trim(), SSL_KEYSTORE_PATH_PASSWORD_ALIAS.trim(), getSSLKeyStoreType()); if (password != null && !password.trim().isEmpty() && !"none".equalsIgnoreCase(password.trim()) && !"_".equalsIgnoreCase(password.trim())) { prop.setProperty(SSL_KEYSTORE_PATH_PASSWORD_PARAM, password); } @@ -369,9 +402,6 @@ public String getSSLTrustStorePath() { } public String getSSLTrustStorePathPassword() { - if (prop == null) { - return null; - } if (prop.containsKey(LGSYNC_LDAP_BIND_KEYSTORE)) { String path = prop.getProperty(LGSYNC_LDAP_BIND_KEYSTORE); if (path != null) { @@ -402,7 +432,7 @@ public long getUpdateMillisMin() { public long getInitSleepTimeInMillisBetweenCycle() throws Throwable { long initSleepValue; - Configuration config = getUserGroupConfig(); + Configuration config = getUserGroupConfig(); if (config.getBoolean(UGSYNC_SERVER_HA_ENABLED_PARAM, false)) { initSleepValue = UGSYNC_INIT_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE_FOR_HA; } else { @@ -412,7 +442,7 @@ public long getInitSleepTimeInMillisBetweenCycle() throws Throwable { } public long getSleepTimeInMillisBetweenCycle() throws Throwable { - String val = prop.getProperty(UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM); + String val = prop.getProperty(UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM); boolean isLdapForce = Boolean.parseBoolean(prop.getProperty(UGSYNC_SLEEP_LDAP_FORCE_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM_ENABLED)); String className = getUserGroupSource().getClass().getName(); if (val == null) { @@ -427,7 +457,7 @@ public long getSleepTimeInMillisBetweenCycle() throws Throwable { if (LGSYNC_SOURCE_CLASS.equals(className)) { if (isLdapForce && ret < UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_LDAP_DEFAULT_VALUE) { minInterval = ret; - LOG.info("If you force the synchronization time of ldap users to be less than the default of 3600s, this setting [{}] millisec will take effect", minInterval); + LOG.info("If you force the synchronization time of ldap users to be less than the default of 3600s, this setting [{}] millisecond will take effect", minInterval); } else { minInterval = UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_LDAP_DEFAULT_VALUE; } @@ -437,7 +467,7 @@ public long getSleepTimeInMillisBetweenCycle() throws Throwable { minInterval = UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_MIN_VALUE; } if ((!isTestRunEnabled()) && (ret < minInterval)) { - LOG.info("Sleep Time Between Cycle can not be lower than [{}] millisec. resetting to min value.", minInterval); + LOG.info("Sleep Time Between Cycle can not be lower than [{}] milli sec. resetting to min value.", minInterval); ret = minInterval; } return ret; @@ -482,9 +512,6 @@ public String getLdapBindDn() throws Throwable { public String getLdapBindPassword() { //update credential from keystore - if (prop == null) { - return null; - } if (prop.containsKey(LGSYNC_LDAP_BIND_KEYSTORE)) { String path = prop.getProperty(LGSYNC_LDAP_BIND_KEYSTORE); if (path != null) { @@ -603,8 +630,8 @@ public Set getGroupNameSet() { } public Set getUserGroupNameAttributeSet() { - String uga = getUserGroupNameAttribute(); - StringTokenizer st = new StringTokenizer(uga, ","); + String uga = getUserGroupNameAttribute(); + StringTokenizer st = new StringTokenizer(uga, ","); Set userGroupNameAttributeSet = new HashSet<>(); while (st.hasMoreTokens()) { userGroupNameAttributeSet.add(st.nextToken().trim()); @@ -617,7 +644,7 @@ public Set getOtherUserAttributes() { if (otherAttributes == null || otherAttributes.trim().isEmpty()) { otherAttributes = DEFAULT_OTHER_USER_ATTRIBUTES; } - StringTokenizer st = new StringTokenizer(otherAttributes, ","); + StringTokenizer st = new StringTokenizer(otherAttributes, ","); Set otherUserAttributes = new HashSet<>(); while (st.hasMoreTokens()) { otherUserAttributes.add(st.nextToken().trim()); @@ -681,7 +708,7 @@ public void setPagedResultsEnabled(boolean pagedResultsEnabled) { public int getPagedResultsSize() { int pagedResultsSize; - String val = prop.getProperty(LGSYNC_PAGED_RESULTS_SIZE); + String val = prop.getProperty(LGSYNC_PAGED_RESULTS_SIZE); if (val == null || val.trim().isEmpty()) { pagedResultsSize = DEFAULT_LGSYNC_PAGED_RESULTS_SIZE; } else { @@ -843,7 +870,7 @@ public Set getOtherGroupAttributes() { if (otherAttributes == null || otherAttributes.trim().isEmpty()) { otherAttributes = DEFAULT_OTHER_GROUP_ATTRIBUTES; } - StringTokenizer st = new StringTokenizer(otherAttributes, ","); + StringTokenizer st = new StringTokenizer(otherAttributes, ","); Set otherGroupAttributes = new HashSet<>(); while (st.hasMoreTokens()) { otherGroupAttributes.add(st.nextToken().trim()); @@ -884,13 +911,13 @@ public String getProperty(String aPropertyName, String aDefaultValue) { public String getPolicyMgrPassword() { //update credential from keystore String password; - if (prop != null && prop.containsKey(SYNC_POLICY_MGR_KEYSTORE)) { + if (prop.containsKey(SYNC_POLICY_MGR_KEYSTORE)) { password = prop.getProperty(SYNC_POLICY_MGR_PASSWORD); if (password != null && !password.isEmpty()) { return password; } } - if (prop != null && prop.containsKey(SYNC_POLICY_MGR_KEYSTORE) && prop.containsKey(SYNC_POLICY_MGR_ALIAS)) { + if (prop.containsKey(SYNC_POLICY_MGR_KEYSTORE) && prop.containsKey(SYNC_POLICY_MGR_ALIAS)) { String path = prop.getProperty(SYNC_POLICY_MGR_KEYSTORE); String alias = prop.getProperty(SYNC_POLICY_MGR_ALIAS, "policymgr.user.password"); if (path != null && alias != null) { @@ -916,7 +943,7 @@ public String getPolicyMgrPassword() { public String getPolicyMgrUserName() { String userName = null; - if (prop != null && prop.containsKey(SYNC_POLICY_MGR_USERNAME)) { + if (prop.containsKey(SYNC_POLICY_MGR_USERNAME)) { userName = prop.getProperty(SYNC_POLICY_MGR_USERNAME); } if (userName == null || userName.isEmpty()) { @@ -935,7 +962,7 @@ public int getPolicyMgrRetryIntervalMs() { public String getSyncSource() { String syncSource = null; - if (prop != null && prop.containsKey(SYNC_SOURCE)) { + if (prop.containsKey(SYNC_SOURCE)) { syncSource = prop.getProperty(SYNC_SOURCE); if (syncSource == null || syncSource.trim().isEmpty()) { syncSource = null; @@ -948,7 +975,7 @@ public String getSyncSource() { public String getContextReferral() { String referral = "ignore"; - if (prop != null && prop.containsKey(LGSYNC_REFERRAL)) { + if (prop.containsKey(LGSYNC_REFERRAL)) { referral = prop.getProperty(LGSYNC_REFERRAL); if (referral == null || referral.trim().isEmpty()) { referral = DEFAULT_LGSYNC_REFERRAL; @@ -961,19 +988,17 @@ public String getContextReferral() { public List getAllRegexPatterns(String baseProperty) { List regexPatterns = new ArrayList<>(); - if (prop != null) { - String baseRegex = prop.getProperty(baseProperty); - if (baseRegex == null) { - return regexPatterns; - } - regexPatterns.add(baseRegex); - int i = 1; - String nextRegex = prop.getProperty(baseProperty + "." + i); - while (nextRegex != null) { - regexPatterns.add(nextRegex); - i++; - nextRegex = prop.getProperty(baseProperty + "." + i); - } + String baseRegex = prop.getProperty(baseProperty); + if (baseRegex == null) { + return regexPatterns; + } + regexPatterns.add(baseRegex); + int i = 1; + String nextRegex = prop.getProperty(baseProperty + "." + i); + while (nextRegex != null) { + regexPatterns.add(nextRegex); + i++; + nextRegex = prop.getProperty(baseProperty + "." + i); } return regexPatterns; } @@ -997,7 +1022,7 @@ public String getUserSyncMappingGroupNameHandler() { } public String getGroupRoleRules() { - if (prop != null && prop.containsKey(GROUP_BASED_ROLE_ASSIGNMENT_RULES)) { + if (prop.containsKey(GROUP_BASED_ROLE_ASSIGNMENT_RULES)) { String groupRoleRules = prop.getProperty(GROUP_BASED_ROLE_ASSIGNMENT_RULES); if (StringUtils.isNotBlank(groupRoleRules)) { return groupRoleRules.trim(); @@ -1007,7 +1032,7 @@ public String getGroupRoleRules() { } public String getWhileListUserRoleRules() { - if (prop != null && prop.containsKey(WHITELIST_USER_ROLE_ASSIGNMENT_RULES)) { + if (prop.containsKey(WHITELIST_USER_ROLE_ASSIGNMENT_RULES)) { String whiteListUserRoleRules = prop.getProperty(WHITELIST_USER_ROLE_ASSIGNMENT_RULES); if (StringUtils.isNotBlank(whiteListUserRoleRules)) { return whiteListUserRoleRules.trim(); @@ -1017,7 +1042,7 @@ public String getWhileListUserRoleRules() { } public String getUserGroupDelimiter() { - if (prop != null && prop.containsKey(USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER)) { + if (prop.containsKey(USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER)) { String userGroupDelimiter = prop.getProperty(USERS_GROUPS_ASSIGNMENT_LIST_DELIMITER); if (userGroupDelimiter != null && !userGroupDelimiter.isEmpty()) { return userGroupDelimiter; @@ -1027,7 +1052,7 @@ public String getUserGroupDelimiter() { } public String getUserGroupNameDelimiter() { - if (prop != null && prop.containsKey(USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER)) { + if (prop.containsKey(USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER)) { String userGroupNameDelimiter = prop.getProperty(USERNAME_GROUPNAME_ASSIGNMENT_LIST_DELIMITER); if (userGroupNameDelimiter != null && !userGroupNameDelimiter.isEmpty()) { return userGroupNameDelimiter; @@ -1051,7 +1076,7 @@ public String getRangerAdminCookieName() { } public String getRoleDelimiter() { - if (prop != null && prop.containsKey(ROLE_ASSIGNMENT_LIST_DELIMITER)) { + if (prop.containsKey(ROLE_ASSIGNMENT_LIST_DELIMITER)) { String roleDelimiter = prop .getProperty(ROLE_ASSIGNMENT_LIST_DELIMITER); if (roleDelimiter != null && !roleDelimiter.isEmpty()) { @@ -1227,7 +1252,7 @@ public String getRegexSeparator() { public boolean isSyncSourceValidationEnabled() { boolean isSyncSourceValidationEnabled = DEFAULT_UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED; - String val = prop.getProperty(UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED); + String val = prop.getProperty(UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED); if (StringUtils.isNotEmpty(val)) { isSyncSourceValidationEnabled = Boolean.parseBoolean(val); } @@ -1242,7 +1267,7 @@ private void init() { } private String getUserGroupSourceClassName() { - String val = prop.getProperty(UGSYNC_SOURCE_CLASS_PARAM); + String val = prop.getProperty(UGSYNC_SOURCE_CLASS_PARAM); String className; String syncSource; diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java index 233c9bf197..c0d2738293 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java @@ -55,8 +55,8 @@ public class FileSourceUserGroupBuilder extends AbstractUserGroupSource implemen private String currentSyncSource; private Map> sourceUsers; // Stores username and attr name & value pairs private Map> sourceGroups; // Stores groupname and attr name & value pairs - private Map> sourceGroupUsers; - private Map> user2GroupListMap = new HashMap<>(); + private Map> sourceGroupUsers; + private Map> user2GroupListMap = new HashMap<>(); private UgsyncAuditInfo ugsyncAuditInfo; private FileSyncSourceInfo fileSyncSourceInfo; private int deleteCycles; diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java index 78bb0883b2..704bac323c 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java @@ -62,12 +62,12 @@ public class PolicyMgrUserGroupBuilder extends AbstractUserGroupSource implement private static final Logger LOG = LoggerFactory.getLogger(PolicyMgrUserGroupBuilder.class); /* ***** POST APIs **** */ - private static final String PM_ADD_USERS_URI = "/service/xusers/ugsync/users"; - private static final String PM_ADD_GROUPS_URI = "/service/xusers/ugsync/groups/"; - private static final String PM_ADD_GROUP_USER_LIST_URI = "/service/xusers/ugsync/groupusers"; - private static final String PM_AUDIT_INFO_URI = "/service/xusers/ugsync/auditinfo/"; - private static final String PM_UPDATE_DELETED_USERS_URI = "/service/xusers/ugsync/users/visibility"; - private static final String PM_UPDATE_DELETED_GROUPS_URI = "/service/xusers/ugsync/groups/visibility"; + private static final String PM_ADD_USERS_URI = "/service/xusers/ugsync/users"; + private static final String PM_ADD_GROUPS_URI = "/service/xusers/ugsync/groups/"; + private static final String PM_ADD_GROUP_USER_LIST_URI = "/service/xusers/ugsync/groupusers"; + private static final String PM_AUDIT_INFO_URI = "/service/xusers/ugsync/auditinfo/"; + private static final String PM_UPDATE_DELETED_USERS_URI = "/service/xusers/ugsync/users/visibility"; + private static final String PM_UPDATE_DELETED_GROUPS_URI = "/service/xusers/ugsync/groups/visibility"; /* ******************* */ /* ***** GET APIs **** */ @@ -90,8 +90,10 @@ public class PolicyMgrUserGroupBuilder extends AbstractUserGroupSource implement private static final String ISVISIBLE = "1"; private static final String ISHIDDEN = "0"; private static final Pattern USER_OR_GROUP_NAME_VALIDATION_REGEX = Pattern.compile("^([A-Za-z0-9_]|[\u00C0-\u017F])([a-zA-Z0-9\\s,._\\-+/@= ]|[\u00C0-\u017F])+$", Pattern.CASE_INSENSITIVE); - private static String localHostname = "unknown"; - private static String errMsgForInactiveServer = "This userGroupSync server is not in active state. Cannot commit transaction!"; + + private static String localHostname = "unknown"; + private static String errMsgForInactiveServer = "This userGroupSync server is not in active state. Cannot commit transaction!"; + private volatile RangerUgSyncRESTClient ldapUgSyncClient; /* {key: user name in DB} */ private Map userCache; @@ -103,10 +105,10 @@ public class PolicyMgrUserGroupBuilder extends AbstractUserGroupSource implement private Map groupNameMap; /* {key: userDN, value: user name in DB} */ private Map userNameMap; + private Set computeRolesForUsers; private Map deltaGroups; private Map deltaUsers; private Map> deltaGroupUsers; - private Set computeRolesForUsers; private Map deletedGroups; private Map deletedUsers; private int noOfNewUsers; @@ -411,7 +413,8 @@ protected void setUserSyncNameValidationEnabled(String isNameValidationEnabled) private void buildUserGroupInfo() throws Throwable { if (authenticationType != null && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) && SecureClientLogin.isKerberosCredentialExists(principal, keytab)) { LOG.info(String.format("Using principal: %s and keytab: %s", principal, keytab)); - Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); + + Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); Boolean isInitDone = Subject.doAs(sub, new PrivilegedAction() { @Override public Boolean run() { @@ -427,7 +430,7 @@ public Boolean run() { } }); if (isInitDone.booleanValue() == false) { - String msg = ("Failed to build Users and Groups from Ranger admin"); + String msg = "Failed to build Users and Groups from Ranger admin"; LOG.error(msg); throw new Exception(msg); } @@ -440,6 +443,7 @@ public Boolean run() { private void buildGroupList() throws Throwable { LOG.debug("==> PolicyMgrUserGroupBuilder.buildGroupList()"); + int totalCount = 100; int retrievedCount = 0; @@ -465,9 +469,9 @@ private void buildGroupList() throws Throwable { } } LOG.debug(String.format("REST response from %s : %s", PM_GROUP_LIST_URI, response)); - GetXGroupListResponse groupList = JsonUtils.jsonToObject(response, GetXGroupListResponse.class); - totalCount = groupList.getTotalCount(); + GetXGroupListResponse groupList = JsonUtils.jsonToObject(response, GetXGroupListResponse.class); + totalCount = groupList.getTotalCount(); if (groupList.getXgroupInfoList() != null) { for (XGroupInfo g : groupList.getXgroupInfoList()) { @@ -513,8 +517,9 @@ private void buildUserList() throws Throwable { } LOG.debug(String.format("REST response from %s : %s", PM_USER_LIST_URI, response)); + GetXUserListResponse userList = JsonUtils.jsonToObject(response, GetXUserListResponse.class); - totalCount = userList.getTotalCount(); + totalCount = userList.getTotalCount(); if (userList.getXuserInfoList() != null) { for (XUserInfo u : userList.getXuserInfoList()) { @@ -551,6 +556,7 @@ private void buildGroupUserLinkList() throws Throwable { } } LOG.debug(String.format("REST response from %s : %s", PM_GET_ALL_GROUP_USER_MAP_LIST_URI, response)); + groupUsersCache = JsonUtils.jsonToObject(response, Map.class); if (MapUtils.isEmpty(groupUsersCache)) { groupUsersCache = new HashMap<>(); @@ -925,19 +931,22 @@ public Integer run() { private int getUsers(GetXUserListResponse xUserList) throws Throwable { LOG.debug("==> PolicyMgrUserGroupBuilder.getUsers()"); + int ret = 0; int totalCount = xUserList.getTotalCount(); int uploadedCount = 0; int pageSize = Integer.parseInt(recordsToPullPerCall); + while (uploadedCount < totalCount) { checkStatus(); GetXUserListResponse pagedXUserList = new GetXUserListResponse(); int pagedXUserListLen = uploadedCount + pageSize; - pagedXUserList.setXuserInfoList(xUserList.getXuserInfoList().subList(uploadedCount, - pagedXUserListLen > totalCount ? totalCount : pagedXUserListLen)); + + pagedXUserList.setXuserInfoList(xUserList.getXuserInfoList().subList(uploadedCount, pagedXUserListLen > totalCount ? totalCount : pagedXUserListLen)); pagedXUserList.setTotalCount(pageSize); if (pagedXUserList.getXuserInfoList().isEmpty()) { LOG.info("PolicyMgrUserGroupBuilder.getUsers() done updating users"); + return 1; } @@ -1028,8 +1037,8 @@ private int getGroups(GetXGroupListResponse xGroupList) throws Throwable { checkStatus(); GetXGroupListResponse pagedXGroupList = new GetXGroupListResponse(); int pagedXGroupListLen = uploadedCount + pageSize; - pagedXGroupList.setXgroupInfoList(xGroupList.getXgroupInfoList().subList(uploadedCount, - pagedXGroupListLen > totalCount ? totalCount : pagedXGroupListLen)); + + pagedXGroupList.setXgroupInfoList(xGroupList.getXgroupInfoList().subList(uploadedCount, pagedXGroupListLen > totalCount ? totalCount : pagedXGroupListLen)); pagedXGroupList.setTotalCount(pageSize); String response = getDataFromLdap(PM_ADD_GROUPS_URI, pagedXGroupList); @@ -1094,10 +1103,10 @@ private int getGroupUsers(List groupUserInfoList) throws Throwabl while (uploadedCount < totalCount) { checkStatus(); - int pagedGroupUserInfoListLen = uploadedCount + pageSize; + int pagedGroupUserInfoListLen = uploadedCount + pageSize; List pagedGroupUserInfoList = groupUserInfoList.subList(uploadedCount, pagedGroupUserInfoListLen > totalCount ? totalCount : pagedGroupUserInfoListLen); + String response = getDataFromLdap(PM_ADD_GROUP_USER_LIST_URI, pagedGroupUserInfoList); - String response = getDataFromLdap(PM_ADD_GROUP_USER_LIST_URI, pagedGroupUserInfoList); if (StringUtils.isNotEmpty(response)) { try { ret = Integer.valueOf(response); @@ -1153,20 +1162,22 @@ private String updateUsersRoles(UsersGroupRoleAssignments ugRoleAssignments) { checkStatus(); int pagedUgRoleAssignmentsListLen = uploadedCount + pageSize; UsersGroupRoleAssignments pagedUgRoleAssignmentsList = new UsersGroupRoleAssignments(); - pagedUgRoleAssignmentsList.setUsers(ugRoleAssignments.getUsers().subList(uploadedCount, - pagedUgRoleAssignmentsListLen > totalCount ? totalCount : pagedUgRoleAssignmentsListLen)); + + pagedUgRoleAssignmentsList.setUsers(ugRoleAssignments.getUsers().subList(uploadedCount, pagedUgRoleAssignmentsListLen > totalCount ? totalCount : pagedUgRoleAssignmentsListLen)); pagedUgRoleAssignmentsList.setGroupRoleAssignments(ugRoleAssignments.getGroupRoleAssignments()); pagedUgRoleAssignmentsList.setUserRoleAssignments(ugRoleAssignments.getUserRoleAssignments()); pagedUgRoleAssignmentsList.setWhiteListGroupRoleAssignments(ugRoleAssignments.getWhiteListGroupRoleAssignments()); pagedUgRoleAssignmentsList.setWhiteListUserRoleAssignments(ugRoleAssignments.getWhiteListUserRoleAssignments()); pagedUgRoleAssignmentsList.setReset(ugRoleAssignments.isReset()); + if ((uploadedCount + pageSize) >= totalCount) { // this is the last iteration of the loop pagedUgRoleAssignmentsList.setLastPage(true); } ClientResponse clientRes; - String url = PM_UPDATE_USERS_ROLES_URI; + String url = PM_UPDATE_USERS_ROLES_URI; String jsonString = JsonUtils.objectToJson(pagedUgRoleAssignmentsList); + LOG.debug(String.format("Paged RoleAssignments Request to %s: %s", url, jsonString)); if (isRangerCookieEnabled) { @@ -1345,8 +1356,7 @@ private String tryUploadEntityWithCred(Object obj, String apiURL) { } } } - if (clientResp.getStatus() != HttpServletResponse.SC_OK && clientResp.getStatus() != HttpServletResponse.SC_NO_CONTENT - && clientResp.getStatus() != HttpServletResponse.SC_BAD_REQUEST) { + if (clientResp.getStatus() != HttpServletResponse.SC_OK && clientResp.getStatus() != HttpServletResponse.SC_NO_CONTENT && clientResp.getStatus() != HttpServletResponse.SC_BAD_REQUEST) { sessionId = null; isValidRangerCookie = false; } @@ -1387,8 +1397,7 @@ private String tryGetEntityWithCred(String apiURL, int retrievedCount) { } } } - if (clientResp.getStatus() != HttpServletResponse.SC_OK && clientResp.getStatus() != HttpServletResponse.SC_NO_CONTENT - && clientResp.getStatus() != HttpServletResponse.SC_BAD_REQUEST) { + if (clientResp.getStatus() != HttpServletResponse.SC_OK && clientResp.getStatus() != HttpServletResponse.SC_NO_CONTENT && clientResp.getStatus() != HttpServletResponse.SC_BAD_REQUEST) { sessionId = null; isValidRangerCookie = false; } @@ -1451,6 +1460,7 @@ private void getRoleForUserGroups(String userGroupRolesData, Map String roleDelimiter = config.getRoleDelimiter(); String userGroupDelimiter = config.getUserGroupDelimiter(); String userNameDelimiter = config.getUserGroupNameDelimiter(); + roleDelimiter = StringUtils.isEmpty(roleDelimiter) ? "&" : roleDelimiter; userGroupDelimiter = StringUtils.isEmpty(userGroupDelimiter) ? ":" : userGroupDelimiter; userNameDelimiter = StringUtils.isEmpty(userNameDelimiter) ? "," : userNameDelimiter; @@ -1649,10 +1659,7 @@ private int updateDeletedUsers() throws Throwable { LOG.debug("==> PolicyMgrUserGroupBuilder.updateDeletedUsers({})", deletedUsers); int ret = 0; - if (authenticationType != null - && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) - && SecureClientLogin.isKerberosCredentialExists(principal, - keytab)) { + if (authenticationType != null && AUTH_KERBEROS.equalsIgnoreCase(authenticationType) && SecureClientLogin.isKerberosCredentialExists(principal, keytab)) { try { Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); ret = Subject.doAs(sub, new PrivilegedAction() { @@ -1679,6 +1686,7 @@ public Integer run() { private int getDeletedUsers() throws Throwable { LOG.debug("==> PolicyMgrUserGroupBuilder.getDeletedUsers()"); + checkStatus(); int ret = 0; String response = null; @@ -1696,7 +1704,9 @@ private int getDeletedUsers() throws Throwable { LOG.error("Failed to get response, Error is : ", t); } } + LOG.debug(String.format("REST response from %s : %s", PM_UPDATE_DELETED_USERS_URI, response)); + if (response != null) { try { ret = Integer.valueOf(response); diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java index a29bb5730c..7b28c551c6 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java @@ -57,35 +57,30 @@ public class UnixUserGroupBuilder implements UserGroupSource { static final String LINUX_GET_GROUP_CMD = "getent group %s"; // mainly for testing purposes, there might be a better way - static final String MAC_GET_ALL_USERS_CMD = "dscl . -readall /Users UniqueID PrimaryGroupID | " + - "awk 'BEGIN { OFS = \":\"; ORS=\"\\n\"; i=0;}" + - "/RecordName: / {name = $2;i = 0;}/PrimaryGroupID: / {gid = $2;}" + - "/^ / {if (i == 0) { i++; name = $1;}}" + - "/UniqueID: / {uid = $2;print name, \"*\", gid, uid;}'"; - static final String MAC_GET_ALL_GROUPS_CMD = "dscl . -list /Groups PrimaryGroupID | " + - "awk -v OFS=\":\" '{print $1, \"*\", $2, \"\"}'"; - static final String MAC_GET_GROUP_CMD = "dscl . -read /Groups/%1$s | paste -d, -s - | sed -e 's/:/|/g' | " + - "awk -v OFS=\":\" -v ORS=\"\\n\" -F, '{print \"%1$s\",\"*\",$6,$4}' | " + - "sed -e 's/:[^:]*| /:/g' | sed -e 's/ /,/g'"; - static final String BACKEND_PASSWD = "passwd"; + static final String MAC_GET_ALL_USERS_CMD = "dscl . -readall /Users UniqueID PrimaryGroupID | awk 'BEGIN { OFS = \":\"; ORS=\"\\n\"; i=0;}/RecordName: / {name = $2;i = 0;}/PrimaryGroupID: / {gid = $2;}/^ / {if (i == 0) { i++; name = $1;}}/UniqueID: / {uid = $2;print name, \"*\", gid, uid;}'"; + static final String MAC_GET_ALL_GROUPS_CMD = "dscl . -list /Groups PrimaryGroupID | awk -v OFS=\":\" '{print $1, \"*\", $2, \"\"}'"; + static final String MAC_GET_GROUP_CMD = "dscl . -read /Groups/%1$s | paste -d, -s - | sed -e 's/:/|/g' | awk -v OFS=\":\" -v ORS=\"\\n\" -F, '{print \"%1$s\",\"*\",$6,$4}' | sed -e 's/:[^:]*| /:/g' | sed -e 's/ /,/g'"; + static final String BACKEND_PASSWD = "passwd"; + + private final boolean enumerateGroupMembers; + private final UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); + private final int minimumUserId; + private final int minimumGroupId; + private final String unixPasswordFile; + private final String unixGroupFile; + private final long timeout; + Set allGroups = new HashSet<>(); private boolean useNss; - private final boolean enumerateGroupMembers; private boolean isUpdateSinkSucc = true; - private final UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); - private Map groupId2groupNameMap; + private Map groupId2groupNameMap; private Map> sourceUsers; // Stores username and attr name & value pairs private Map> sourceGroups; // Stores groupname and attr name & value pairs - private Map> sourceGroupUsers; - private Table groupUserTable; // groupname, username, group id - private final String unixPasswordFile; - private final String unixGroupFile; + private Map> sourceGroupUsers; + private Table groupUserTable; // groupname, username, group id private String currentSyncSource; private int deleteCycles; - private final int minimumUserId; - private final int minimumGroupId; private long lastUpdateTime; - private final long timeout; private long passwordFileModifiedAt; private long groupFileModifiedAt; private UgsyncAuditInfo ugsyncAuditInfo; @@ -99,13 +94,10 @@ public UnixUserGroupBuilder() { minimumGroupId = Integer.parseInt(config.getMinGroupId()); unixPasswordFile = config.getUnixPasswordFile(); unixGroupFile = config.getUnixGroupFile(); - - if (LOG.isDebugEnabled()) { - LOG.debug("Minimum UserId: {}, minimum GroupId: {}", minimumUserId, minimumGroupId); - } - - timeout = config.getUpdateMillisMin(); + timeout = config.getUpdateMillisMin(); enumerateGroupMembers = config.isGroupEnumerateEnabled(); + + LOG.debug("Minimum UserId: {}, minimum GroupId: {}", minimumUserId, minimumGroupId); } public static void main(String[] args) throws Throwable { diff --git a/ugsync/src/main/java/org/apache/ranger/usergroupsync/RegEx.java b/ugsync/src/main/java/org/apache/ranger/usergroupsync/RegEx.java index d991f856e5..aecb59dcd0 100644 --- a/ugsync/src/main/java/org/apache/ranger/usergroupsync/RegEx.java +++ b/ugsync/src/main/java/org/apache/ranger/usergroupsync/RegEx.java @@ -27,8 +27,8 @@ import java.util.regex.Pattern; public class RegEx extends AbstractMapper { - private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); - private LinkedHashMap replacementPattern; + private final UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); + private LinkedHashMap replacementPattern; public LinkedHashMap getReplacementPattern() { return replacementPattern; diff --git a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserSyncMetricsProducer.java b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserSyncMetricsProducer.java index e3af416c6c..8a7f220d54 100644 --- a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserSyncMetricsProducer.java +++ b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserSyncMetricsProducer.java @@ -29,6 +29,7 @@ public class UserSyncMetricsProducer implements Runnable { private static final Logger LOG = LoggerFactory.getLogger(UserSyncMetricsProducer.class); + private boolean shutdownFlag; public static void main(String[] args) { diff --git a/ugsync/src/test/java/org/apache/ranger/unixusersync/process/TestFileSourceUserGroupBuilder.java b/ugsync/src/test/java/org/apache/ranger/unixusersync/process/TestFileSourceUserGroupBuilder.java index dcedbb183d..08466d6881 100644 --- a/ugsync/src/test/java/org/apache/ranger/unixusersync/process/TestFileSourceUserGroupBuilder.java +++ b/ugsync/src/test/java/org/apache/ranger/unixusersync/process/TestFileSourceUserGroupBuilder.java @@ -26,7 +26,7 @@ import static org.junit.Assert.assertTrue; public class TestFileSourceUserGroupBuilder { - private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); + private final UserGroupSyncConfig config = UserGroupSyncConfig.getInstance(); @Test public void testUpdateSinkFromCsvFile() throws Throwable { diff --git a/ugsync/src/test/java/org/apache/ranger/usergroupsync/PolicyMgrUserGroupBuilderTest.java b/ugsync/src/test/java/org/apache/ranger/usergroupsync/PolicyMgrUserGroupBuilderTest.java index 7897439050..bbc8374160 100644 --- a/ugsync/src/test/java/org/apache/ranger/usergroupsync/PolicyMgrUserGroupBuilderTest.java +++ b/ugsync/src/test/java/org/apache/ranger/usergroupsync/PolicyMgrUserGroupBuilderTest.java @@ -39,7 +39,7 @@ public PolicyMgrUserGroupBuilderTest() { } @Override - public void init() throws Throwable { + public void init() { allGroups = new HashSet<>(); allUsers = new HashSet<>(); groupUsers = new HashMap<>(); @@ -48,16 +48,16 @@ public void init() throws Throwable { } @Override - public void addOrUpdateUsersGroups(Map> sourceGroups, Map> sourceUsers, Map> sourceGroupUsers, boolean computeDeletes) throws Throwable { - for (String userdn : sourceUsers.keySet()) { - String username = userNameTransform(sourceUsers.get(userdn).get("original_name")); + public void addOrUpdateUsersGroups(Map> sourceGroups, Map> sourceUsers, Map> sourceGroupUsers, boolean computeDeletes) { + for (String userDn : sourceUsers.keySet()) { + String username = userNameTransform(sourceUsers.get(userDn).get("original_name")); allUsers.add(username); if (!isValidString(username)) { invalidUsers.add(username); } } - for (String groupdn : sourceGroups.keySet()) { - String groupname = groupNameTransform(sourceGroups.get(groupdn).get("original_name")); + for (String groupDn : sourceGroups.keySet()) { + String groupname = groupNameTransform(sourceGroups.get(groupDn).get("original_name")); allGroups.add(groupname); if (!isValidString(groupname)) { invalidGroups.add(groupname); diff --git a/ugsync/src/test/java/org/apache/ranger/usergroupsync/TestRegEx.java b/ugsync/src/test/java/org/apache/ranger/usergroupsync/TestRegEx.java index 560615a46d..7e9856c3af 100644 --- a/ugsync/src/test/java/org/apache/ranger/usergroupsync/TestRegEx.java +++ b/ugsync/src/test/java/org/apache/ranger/usergroupsync/TestRegEx.java @@ -38,22 +38,22 @@ public class TestRegEx { List groupRegexPatterns; @Before - public void setUp() throws Exception { + public void setUp() { userNameRegEx = new RegEx(); groupNameRegEx = new RegEx(); - userRegexPatterns = new ArrayList(); - groupRegexPatterns = new ArrayList(); + userRegexPatterns = new ArrayList<>(); + groupRegexPatterns = new ArrayList<>(); } @Test - public void testUserNameTransform() throws Throwable { + public void testUserNameTransform() { userRegexPatterns.add("s/\\s/_/"); userNameRegEx.populateReplacementPatterns(userNameBaseProperty, userRegexPatterns, mappingSeparator); assertEquals("test_user", userNameRegEx.transform("test user")); } @Test - public void testGroupNameTransform() throws Throwable { + public void testGroupNameTransform() { groupRegexPatterns.add("s/\\s/_/g"); groupRegexPatterns.add("s/_/\\$/g"); groupNameRegEx.populateReplacementPatterns(groupNameBaseProperty, groupRegexPatterns, mappingSeparator); @@ -67,7 +67,7 @@ public void testEmptyTransform() { } @Test - public void testTransform() throws Throwable { + public void testTransform() { userRegexPatterns.add("s/\\s/_/g"); groupRegexPatterns.add("s/\\s/_/g"); userNameRegEx.populateReplacementPatterns(userNameBaseProperty, userRegexPatterns, mappingSeparator); @@ -77,7 +77,7 @@ public void testTransform() throws Throwable { } @Test - public void testTransform1() throws Throwable { + public void testTransform1() { userRegexPatterns.add("s/\\\\/ /g"); userRegexPatterns.add("s//_/g"); userNameRegEx.populateReplacementPatterns(userNameBaseProperty, userRegexPatterns, mappingSeparator); @@ -90,10 +90,10 @@ public void testTransform1() throws Throwable { } @Test - public void testTransformWithSeparators() throws Throwable { + public void testTransformWithSeparators() { String[] separators = {"%", "#", "&", "!", "@", "-", "~", "=", ",", " "}; for (String separator : separators) { - userRegexPatterns = new ArrayList(); + userRegexPatterns = new ArrayList<>(); userRegexPatterns.add(String.format("s%sdark%sDE/dark%sg", separator, separator, separator)); userNameRegEx.populateReplacementPatterns(userNameBaseProperty, userRegexPatterns, separator); assertEquals("DE/dark_knight_admin", userNameRegEx.transform("dark_knight_admin")); @@ -101,7 +101,7 @@ public void testTransformWithSeparators() throws Throwable { } @Test - public void testUsernamePrefix() throws Throwable { + public void testUsernamePrefix() { // appends PR/ to the beginning String separator = "#"; userRegexPatterns = Collections.singletonList("s#^(.*)#PR/$1#g"); @@ -113,7 +113,7 @@ public void testUsernamePrefix() throws Throwable { } @Test - public void testUsernameSuffix() throws Throwable { + public void testUsernameSuffix() { // appends _ty to the end String separator = "#"; userRegexPatterns = Collections.singletonList("s#^(.*)#$1_ty#g"); diff --git a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamLoginModule.java b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamLoginModule.java index 7b33823787..84d8a7ff73 100644 --- a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamLoginModule.java +++ b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamLoginModule.java @@ -42,6 +42,7 @@ public class PamLoginModule implements LoginModule { public static final String SERVICE_KEY = "ranger.pam.service"; + private PAM pam; private Subject subject; private CallbackHandler callbackHandler; diff --git a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamPrincipal.java b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamPrincipal.java index 969323b035..c6fb51c1a1 100644 --- a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamPrincipal.java +++ b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/PamPrincipal.java @@ -26,13 +26,13 @@ import java.util.Set; public class PamPrincipal implements Principal { - private String userName; - private String gecos; - private String homeDir; - private String shell; - private int uid; - private int gid; - private Set groups; + private final String userName; + private final String gecos; + private final String homeDir; + private final String shell; + private final Set groups; + private final int uid; + private final int gid; public PamPrincipal(UnixUser user) { super(); diff --git a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/RemoteUnixLoginModule.java b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/RemoteUnixLoginModule.java index 57bb32412b..7c3c1f6b0e 100644 --- a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/RemoteUnixLoginModule.java +++ b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/RemoteUnixLoginModule.java @@ -46,7 +46,6 @@ import java.net.Socket; import java.security.KeyStore; import java.security.SecureRandom; -import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.util.Arrays; import java.util.Map; @@ -64,6 +63,7 @@ public class RemoteUnixLoginModule implements LoginModule { private static final String SERVER_CERT_VALIDATION_PARAM = "ranger.unixauth.server.cert.validation"; private static final String JAAS_ENABLED_PARAM = "ranger.unixauth.remote.login.enabled"; private static final String SSL_ALGORITHM = "TLSv1.2"; + private String userName; private String remoteHostName; private String loginGroups; @@ -194,11 +194,7 @@ public void initParams(Properties options) { } val = (String) options.get(DEBUG_PARAM); - if (val != null && (!val.equalsIgnoreCase("false"))) { - debug = true; - } else { - debug = false; - } + debug = val != null && (!val.equalsIgnoreCase("false")); remoteHostName = (String) options.get(REMOTE_LOGIN_HOST_PARAM); log("RemoteHostName:" + remoteHostName); @@ -284,16 +280,8 @@ private String getLoginReplyFromAuthService(String aUserName, char[] modifiedPas if (keyStorePath != null) { KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType()); - InputStream in = null; - - in = getFileInputStream(keyStorePath); - - try { + try (InputStream in = getFileInputStream(keyStorePath)) { ks.load(in, keyStorePathPassword.toCharArray()); - } finally { - if (in != null) { - in.close(); - } } KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); @@ -306,34 +294,26 @@ private String getLoginReplyFromAuthService(String aUserName, char[] modifiedPas TrustManager[] tm = null; if (serverCertValidation) { - KeyStore trustStoreKeyStore = null; + KeyStore trustStoreKeyStore; if (trustStorePath != null) { trustStoreKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); - InputStream in = null; - - in = getFileInputStream(trustStorePath); - - try { + try (InputStream in = getFileInputStream(trustStorePath)) { trustStoreKeyStore.load(in, trustStorePathPassword.toCharArray()); trustManagerFactory.init(trustStoreKeyStore); tm = trustManagerFactory.getTrustManagers(); - } finally { - if (in != null) { - in.close(); - } } } } else { TrustManager ignoreValidationTM = new X509TrustManager() { - public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { + public void checkClientTrusted(X509Certificate[] chain, String authType) { // Ignore Server Certificate Validation } - public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { + public void checkServerTrusted(X509Certificate[] chain, String authType) { // Ignore Server Certificate Validation } @@ -359,7 +339,6 @@ public X509Certificate[] getAcceptedIssuers() { OutputStreamWriter writer = new OutputStreamWriter(sslsocket.getOutputStream()); writer.write(loginData); - writer.flush(); BufferedReader reader = new BufferedReader(new InputStreamReader(sslsocket.getInputStream())); @@ -367,7 +346,6 @@ public X509Certificate[] getAcceptedIssuers() { ret = reader.readLine(); reader.close(); - writer.close(); } finally { if (sslsocket != null) { @@ -386,7 +364,7 @@ public X509Certificate[] getAcceptedIssuers() { } private InputStream getFileInputStream(String path) throws FileNotFoundException { - InputStream ret = null; + InputStream ret; File f = new File(path); diff --git a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UnixGroupPrincipal.java b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UnixGroupPrincipal.java index 129330b11f..19ca032871 100644 --- a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UnixGroupPrincipal.java +++ b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UnixGroupPrincipal.java @@ -23,8 +23,9 @@ import java.security.Principal; public class UnixGroupPrincipal implements Principal, Serializable { - private static final long serialVersionUID = 8137147441841439754L; - private String groupName; + private static final long serialVersionUID = 8137147441841439754L; + + private final String groupName; public UnixGroupPrincipal(String groupName) { this.groupName = groupName; diff --git a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UnixUserPrincipal.java b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UnixUserPrincipal.java index 49da784619..d280e689b6 100644 --- a/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UnixUserPrincipal.java +++ b/unixauthclient/src/main/java/org/apache/ranger/authentication/unix/jaas/UnixUserPrincipal.java @@ -23,8 +23,9 @@ import java.security.Principal; public class UnixUserPrincipal implements Principal, Serializable { - private static final long serialVersionUID = -3568658536591178268L; - private String userName; + private static final long serialVersionUID = -3568658536591178268L; + + private final String userName; public UnixUserPrincipal(String userName) { this.userName = userName; diff --git a/unixauthservice/src/main/java/org/apache/ranger/authentication/PasswordValidator.java b/unixauthservice/src/main/java/org/apache/ranger/authentication/PasswordValidator.java index a6d6dd4d92..0df92cafdf 100644 --- a/unixauthservice/src/main/java/org/apache/ranger/authentication/PasswordValidator.java +++ b/unixauthservice/src/main/java/org/apache/ranger/authentication/PasswordValidator.java @@ -33,11 +33,9 @@ public class PasswordValidator implements Runnable { private static final Logger LOG = LoggerFactory.getLogger(PasswordValidator.class); - private static String validatorProgram; - private static List adminUserList; - private static String adminRoleNames; + private static String validatorProgram; private Socket client; @@ -71,7 +69,7 @@ public static void setAdminRoleNames(String adminRoleNames) { @Override public void run() { - BufferedReader reader = null; + BufferedReader reader; PrintWriter writer = null; String userName = null; @@ -95,15 +93,14 @@ public void run() { writer.flush(); LOG.error("Response [{}] for user: {} as ValidatorProgram is not defined in configuration", res, userName); } else { - BufferedReader pReader = null; - PrintWriter pWriter = null; + BufferedReader pReader; + PrintWriter pWriter; Process p = null; try { p = Runtime.getRuntime().exec(validatorProgram); pReader = new BufferedReader(new InputStreamReader(p.getInputStream())); - pWriter = new PrintWriter(new OutputStreamWriter(p.getOutputStream())); pWriter.println(request); diff --git a/unixauthservice/src/main/java/org/apache/ranger/authentication/UnixAuthenticationService.java b/unixauthservice/src/main/java/org/apache/ranger/authentication/UnixAuthenticationService.java index 97c07072cd..cc9b2497d0 100644 --- a/unixauthservice/src/main/java/org/apache/ranger/authentication/UnixAuthenticationService.java +++ b/unixauthservice/src/main/java/org/apache/ranger/authentication/UnixAuthenticationService.java @@ -54,6 +54,7 @@ public class UnixAuthenticationService { private static final Logger LOG = LoggerFactory.getLogger(UnixAuthenticationService.class); + private static final String serviceName = "UnixAuthenticationService"; private static final String SSL_ALGORITHM = "TLSv1.2"; private static final String REMOTE_LOGIN_AUTH_SERVICE_PORT_PARAM = "ranger.usersync.port"; @@ -69,17 +70,20 @@ public class UnixAuthenticationService { private static final String SSL_ENABLED_PARAM = "ranger.usersync.ssl"; private static final String CREDSTORE_FILENAME_PARAM = "ranger.usersync.credstore.filename"; private static final String[] UGSYNC_CONFIG_XML_FILES = {"ranger-ugsync-default.xml", "ranger-ugsync-site.xml"}; + private static boolean enableUnixAuth; - private String keyStorePath; - private String keyStoreType; - private List enabledProtocolsList; - private List enabledCipherSuiteList; - private String keyStorePathPassword; - private String trustStorePath; - private String trustStorePathPassword; - private String trustStoreType; - private List adminUserList = new ArrayList(); - private String adminRoleNames; + + private final List adminUserList = new ArrayList<>(); + + private String adminRoleNames; + private String keyStorePath; + private String keyStorePathPassword; + private String keyStoreType; + private String trustStorePath; + private String trustStorePathPassword; + private String trustStoreType; + private List enabledProtocolsList; + private List enabledCipherSuiteList; private UserSyncHAInitializerImpl userSyncHAInitializerImpl; private int portNum; private boolean sslEnabled; @@ -88,7 +92,7 @@ public UnixAuthenticationService() { } public static void main(String[] args) { - enableUnixAuth = Arrays.stream(args).anyMatch(arg -> "-enableUnixAuth".equalsIgnoreCase(arg)); + enableUnixAuth = Arrays.stream(args).anyMatch("-enableUnixAuth"::equalsIgnoreCase); UnixAuthenticationService service = new UnixAuthenticationService(); service.userSyncHAInitializerImpl = UserSyncHAInitializerImpl.getInstance(UserGroupSyncConfig.getInstance().getUserGroupConfig()); service.run(); @@ -125,19 +129,11 @@ public void startService() throws Throwable { if (keyStorePath != null && !keyStorePath.isEmpty()) { KeyStore ks = KeyStore.getInstance(keyStoreType); - InputStream in = null; - - in = getFileInputStream(keyStorePath); - - try { + try (InputStream in = getFileInputStream(keyStorePath)) { if (keyStorePathPassword == null) { keyStorePathPassword = ""; } ks.load(in, keyStorePathPassword.toCharArray()); - } finally { - if (in != null) { - in.close(); - } } KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); @@ -152,19 +148,11 @@ public void startService() throws Throwable { if (trustStorePath != null && !trustStorePath.isEmpty()) { trustStoreKeyStore = KeyStore.getInstance(trustStoreType); - InputStream in = null; - - in = getFileInputStream(trustStorePath); - - try { + try (InputStream in = getFileInputStream(trustStorePath)) { if (trustStorePathPassword == null) { trustStorePathPassword = ""; } trustStoreKeyStore.load(in, trustStorePathPassword.toCharArray()); - } finally { - if (in != null) { - in.close(); - } } } @@ -182,7 +170,7 @@ public void startService() throws Throwable { if (sslEnabled) { SSLServerSocket secureSocket = (SSLServerSocket) socket; String[] protocols = secureSocket.getEnabledProtocols(); - Set allowedProtocols = new HashSet(); + Set allowedProtocols = new HashSet<>(); for (String ep : protocols) { if (enabledProtocolsList.contains(ep.toUpperCase())) { LOG.info("Enabling Protocol: [{}]", ep); @@ -196,17 +184,13 @@ public void startService() throws Throwable { secureSocket.setEnabledProtocols(allowedProtocols.toArray(new String[0])); } String[] enabledCipherSuites = secureSocket.getEnabledCipherSuites(); - Set allowedCipherSuites = new HashSet(); + Set allowedCipherSuites = new HashSet<>(); for (String enabledCipherSuite : enabledCipherSuites) { if (enabledCipherSuiteList.contains(enabledCipherSuite)) { - if (LOG.isDebugEnabled()) { - LOG.debug("Enabling CipherSuite : [{}]", enabledCipherSuite); - } + LOG.debug("Enabling CipherSuite : [{}]", enabledCipherSuite); allowedCipherSuites.add(enabledCipherSuite); } else { - if (LOG.isDebugEnabled()) { - LOG.debug("Disabling CipherSuite : [{}]", enabledCipherSuite); - } + LOG.debug("Disabling CipherSuite : [{}]", enabledCipherSuite); } } if (!allowedCipherSuites.isEmpty()) { @@ -214,7 +198,7 @@ public void startService() throws Throwable { } } - Socket client = null; + Socket client; try { while ((client = socket.accept()) != null) { @@ -234,12 +218,15 @@ private void startUnixUserGroupSyncProcess() { UserGroupSync syncProc = new UserGroupSync(); Thread newSyncProcThread = new Thread(syncProc); newSyncProcThread.setName("UnixUserSyncThread"); - // If this thread is set as daemon, then the entire process will terminate if enableUnixAuth is false - // Therefore this is marked as non-daemon thread. Don't change the following line + + /* If this thread is set as daemon, then the entire process will terminate if enableUnixAuth is false + Therefore this is marked as non-daemon thread. Don't change the following line + */ newSyncProcThread.setDaemon(false); newSyncProcThread.start(); LOG.info("UnixUserSyncThread started"); LOG.info("creating UserSyncMetricsProducer thread with default metrics location : {}", System.getProperty("logdir")); + //Start the user sync metrics boolean isUserSyncMetricsEnabled = UserGroupSyncConfig.getInstance().isUserSyncMetricsEnabled(); if (isUserSyncMetricsEnabled) { @@ -255,7 +242,7 @@ private void startUnixUserGroupSyncProcess() { } //TODO: add more validation code - private void init() throws Throwable { + private void init() { Properties prop = new Properties(); for (String fn : UGSYNC_CONFIG_XML_FILES) { @@ -263,9 +250,8 @@ private void init() throws Throwable { } String credStoreFileName = prop.getProperty(CREDSTORE_FILENAME_PARAM); - - keyStorePath = prop.getProperty(SSL_KEYSTORE_PATH_PARAM); - + keyStorePath = prop.getProperty(SSL_KEYSTORE_PATH_PARAM); + trustStorePath = prop.getProperty(SSL_TRUSTSTORE_PATH_PARAM); keyStoreType = prop.getProperty(SSL_KEYSTORE_FILE_TYPE_PARAM, KeyStore.getDefaultType()); trustStoreType = prop.getProperty(SSL_TRUSTSTORE_FILE_TYPE_PARAM, KeyStore.getDefaultType()); @@ -287,19 +273,19 @@ private void init() throws Throwable { String crendentialProviderPrefixBcfks = "bcfks://file"; credStoreFileName = crendentialProviderPrefixBcfks + credStoreFileName; } + keyStorePathPassword = CredentialReader.getDecryptedString(credStoreFileName, SSL_KEYSTORE_PATH_PASSWORD_ALIAS, keyStoreType); trustStorePathPassword = CredentialReader.getDecryptedString(credStoreFileName, SSL_TRUSTSTORE_PATH_PASSWORD_ALIAS, trustStoreType); + portNum = Integer.parseInt(prop.getProperty(REMOTE_LOGIN_AUTH_SERVICE_PORT_PARAM)); + String validatorProg = prop.getProperty(CRED_VALIDATOR_PROG); - trustStorePath = prop.getProperty(SSL_TRUSTSTORE_PATH_PARAM); - portNum = Integer.parseInt(prop.getProperty(REMOTE_LOGIN_AUTH_SERVICE_PORT_PARAM)); - String validatorProg = prop.getProperty(CRED_VALIDATOR_PROG); if (validatorProg != null) { PasswordValidator.setValidatorProgram(validatorProg); } String adminUsers = prop.getProperty(ADMIN_USER_LIST_PARAM); - if (adminUsers != null && adminUsers.trim().length() > 0) { + if (adminUsers != null && !adminUsers.trim().isEmpty()) { for (String u : adminUsers.split(",")) { LOG.info("Adding Admin User: {}", u.trim()); adminUserList.add(u.trim()); @@ -320,12 +306,12 @@ private void init() throws Throwable { String defaultEnabledProtocols = "TLSv1.2"; String enabledProtocols = prop.getProperty("ranger.usersync.https.ssl.enabled.protocols", defaultEnabledProtocols); String enabledCipherSuites = prop.getProperty("ranger.usersync.https.ssl.enabled.cipher.suites", ""); - enabledProtocolsList = new ArrayList(Arrays.asList(enabledProtocols.toUpperCase().trim().split("\\s*,\\s*"))); - enabledCipherSuiteList = new ArrayList(Arrays.asList(enabledCipherSuites.toUpperCase().trim().split("\\s*,\\s*"))); + enabledProtocolsList = new ArrayList<>(Arrays.asList(enabledProtocols.toUpperCase().trim().split("\\s*,\\s*"))); + enabledCipherSuiteList = new ArrayList<>(Arrays.asList(enabledCipherSuites.toUpperCase().trim().split("\\s*,\\s*"))); } private InputStream getFileInputStream(String path) throws FileNotFoundException { - InputStream ret = null; + InputStream ret; File f = new File(path); From b32d01dbaa510d9aa5670e0548299c0512563b75 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 21 Dec 2024 19:23:47 -0800 Subject: [PATCH 4/4] Remove isDebugEnabled() in UserGroupSync.java --- .../ranger/usergroupsync/UserGroupSync.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java index 523582069b..d8b6b305d4 100644 --- a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java +++ b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java @@ -52,24 +52,23 @@ public void run() { LOG.info("initializing source: {}", ugSource.getClass().getName()); ugSource.init(); - LOG.info("Begin: initial load of user/group from source==>sink"); + LOG.info("Begin: initial load of user/group from source ==> sink"); syncUserGroup(); - LOG.info("End: initial load of user/group from source==>sink"); + LOG.info("End: initial load of user/group from source ==> sink"); initPending = false; LOG.info("Done initializing user/group source and sink"); } else { - if (LOG.isDebugEnabled()) { - LOG.debug("Sleeping for [{}] milliSeconds as this server is running in passive mode", initSleepTimeBetweenCycleInMillis); - } + LOG.debug("Sleeping for [{}] milliSeconds as this server is running in passive mode", initSleepTimeBetweenCycleInMillis); + Thread.sleep(initSleepTimeBetweenCycleInMillis); } } catch (Throwable t) { LOG.error("Failed to initialize UserGroup source/sink. Will retry after {} milliseconds. Error details: ", sleepTimeBetweenCycleInMillis, t); + try { - if (LOG.isDebugEnabled()) { - LOG.debug("Sleeping for [{}] milliSeconds", sleepTimeBetweenCycleInMillis); - } + LOG.debug("Sleeping for [{}] milliSeconds", sleepTimeBetweenCycleInMillis); + Thread.sleep(sleepTimeBetweenCycleInMillis); } catch (Exception e) { LOG.error("Failed to wait for [{}] milliseconds before attempting to initialize UserGroup source/sink", sleepTimeBetweenCycleInMillis, e); @@ -79,9 +78,8 @@ public void run() { while (true) { try { - if (LOG.isDebugEnabled()) { - LOG.debug("Sleeping for {} milliSeconds", sleepTimeBetweenCycleInMillis); - } + LOG.debug("Sleeping for {} milliSeconds", sleepTimeBetweenCycleInMillis); + Thread.sleep(sleepTimeBetweenCycleInMillis); } catch (InterruptedException e) { LOG.error("Failed to wait for [{}] milliseconds before attempting to synchronize UserGroup information", sleepTimeBetweenCycleInMillis, e); @@ -89,9 +87,11 @@ public void run() { try { if (UserGroupSyncConfig.isUgsyncServiceActive()) { - LOG.info("Begin: update user/group from source==>sink"); + LOG.info("Begin: update user/group from source ==> sink"); + syncUserGroup(); - LOG.info("End: update user/group from source==>sink"); + + LOG.info("End: update user/group from source ==> sink"); } else { LOG.info("Sleeping for [{}] milliSeconds as this server is running in passive mode", sleepTimeBetweenCycleInMillis); } @@ -100,7 +100,7 @@ public void run() { } } } catch (Throwable t) { - LOG.error("UserGroupSync thread got an error", t); + LOG.error("UserGroupSync thread got an error ", t); } finally { LOG.info("Shutting down the UserGroupSync thread"); }