From 09a1b1146fd3f289fb5671dae1a8a7c47c881f50 Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Fri, 23 Dec 2022 17:10:54 +0530 Subject: [PATCH] Optimized the code for user report generation (#164) --- .../common/util/CbExtServerProperties.java | 11 + .../sunbird/common/util/IndexerService.java | 237 ++++++++++-------- .../profile/service/ProfileServiceImpl.java | 122 +++++---- .../user/report/UserReportService.java | 9 +- .../user/report/UserReportServiceImpl.java | 76 +++--- src/main/resources/application.properties | 1 + 6 files changed, 265 insertions(+), 191 deletions(-) diff --git a/src/main/java/org/sunbird/common/util/CbExtServerProperties.java b/src/main/java/org/sunbird/common/util/CbExtServerProperties.java index 7f64d2a13..8c56b252b 100644 --- a/src/main/java/org/sunbird/common/util/CbExtServerProperties.java +++ b/src/main/java/org/sunbird/common/util/CbExtServerProperties.java @@ -430,6 +430,9 @@ public class CbExtServerProperties { @Value("${course.url}") private String courseLinkUrl; + + @Value("${es.user.report.include.fields}") + private String esUserReportIncludeFields; public String getUserAssessmentSubmissionDuration() { return userAssessmentSubmissionDuration; @@ -1573,4 +1576,12 @@ public String getCourseLinkUrl() { public void setCourseLinkUrl(String courseLinkUrl) { this.courseLinkUrl = courseLinkUrl; } + + public String[] getEsUserReportIncludeFields() { + return esUserReportIncludeFields.split(",", -1); + } + + public void setEsUserReportIncludeFields(String esUserReportIncludeFields) { + this.esUserReportIncludeFields = esUserReportIncludeFields; + } } \ No newline at end of file diff --git a/src/main/java/org/sunbird/common/util/IndexerService.java b/src/main/java/org/sunbird/common/util/IndexerService.java index 8770d3337..722fd3be7 100644 --- a/src/main/java/org/sunbird/common/util/IndexerService.java +++ b/src/main/java/org/sunbird/common/util/IndexerService.java @@ -31,113 +31,117 @@ @Service public class IndexerService { - private Logger logger = LoggerFactory.getLogger(IndexerService.class); - - @Autowired - @Qualifier("esClient") - private RestHighLevelClient esClient; - - @Autowired - @Qualifier("sbEsClient") - private RestHighLevelClient sbEsClient; - - /** - * @param index name of index - * @param indexType index type - * @param entityId entity Id - * @param indexDocument index Document - * @return status - */ - public RestStatus addEntity(String index, String indexType, String entityId, Map indexDocument) { - logger.info("addEntity starts with index {} and entityId {}", index, entityId); - IndexResponse response = null; - try { - if(!StringUtils.isEmpty(entityId)){ - response = esClient.index(new IndexRequest(index, indexType, entityId).source(indexDocument), RequestOptions.DEFAULT); - }else{ - response = esClient.index(new IndexRequest(index, indexType).source(indexDocument), RequestOptions.DEFAULT); - } - } catch (IOException e) { - logger.error("Exception in adding record to ElasticSearch", e); - } - if (null == response) - return null; - return response.status(); - } - - /** - * @param index name of index - * @param indexType index type - * @param entityId entity Id - * @param indexDocument index Document - * @return status - */ - public RestStatus updateEntity(String index, String indexType, String entityId, Map indexDocument) { - logger.info("updateEntity starts with index {} and entityId {}", index, entityId); - UpdateResponse response = null; - try { - response = esClient.update(new UpdateRequest(index.toLowerCase(), indexType, entityId).doc(indexDocument), RequestOptions.DEFAULT); - } catch (IOException e) { - logger.error("Exception in updating a record to ElasticSearch", e); - } - if (null == response) - return null; - return response.status(); - } - - /** - * @param index name of index - * @param indexType index type - * @param entityId entity Id - * @return status - */ - public Map readEntity(String index, String indexType, String entityId){ - logger.info("readEntity starts with index {} and entityId {}", index, entityId); - GetResponse response = null; - try { - response = esClient.get(new GetRequest(index, indexType, entityId), RequestOptions.DEFAULT); - } catch (IOException e) { - logger.error("Exception in getting the record from ElasticSearch", e); - } - if(null == response) - return null; - return response.getSourceAsMap(); - } - - /** - * Search the document in es based on provided information - * - * @param indexName es index name - * @param type index type - * @param searchSourceBuilder source builder - * @return es search response - * @throws IOException - */ - public SearchResponse getEsResult(String indexName, String type, SearchSourceBuilder searchSourceBuilder, boolean isSunbirdES) throws IOException { - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices(indexName); - if (!StringUtils.isEmpty(type)) - searchRequest.types(type); - searchRequest.source(searchSourceBuilder); - return getEsResult(searchRequest, isSunbirdES); - } - - public RestStatus BulkInsert(List indexRequestList) { - BulkResponse restStatus = null; - if (!CollectionUtils.isEmpty(indexRequestList)) { - BulkRequest bulkRequest = new BulkRequest(); - indexRequestList.forEach(bulkRequest::add); - try { - restStatus = esClient.bulk(bulkRequest, RequestOptions.DEFAULT); - } catch (IOException e) { - logger.error("Exception while doing the bulk operation in ElasticSearch", e); - } - } - if(null == restStatus) - return null; - return restStatus.status(); - } - + private Logger logger = LoggerFactory.getLogger(IndexerService.class); + + @Autowired + @Qualifier("esClient") + private RestHighLevelClient esClient; + + @Autowired + @Qualifier("sbEsClient") + private RestHighLevelClient sbEsClient; + + /** + * @param index name of index + * @param indexType index type + * @param entityId entity Id + * @param indexDocument index Document + * @return status + */ + public RestStatus addEntity(String index, String indexType, String entityId, Map indexDocument) { + logger.info("addEntity starts with index {} and entityId {}", index, entityId); + IndexResponse response = null; + try { + if (!StringUtils.isEmpty(entityId)) { + response = esClient.index(new IndexRequest(index, indexType, entityId).source(indexDocument), + RequestOptions.DEFAULT); + } else { + response = esClient.index(new IndexRequest(index, indexType).source(indexDocument), + RequestOptions.DEFAULT); + } + } catch (IOException e) { + logger.error("Exception in adding record to ElasticSearch", e); + } + if (null == response) + return null; + return response.status(); + } + + /** + * @param index name of index + * @param indexType index type + * @param entityId entity Id + * @param indexDocument index Document + * @return status + */ + public RestStatus updateEntity(String index, String indexType, String entityId, Map indexDocument) { + logger.info("updateEntity starts with index {} and entityId {}", index, entityId); + UpdateResponse response = null; + try { + response = esClient.update(new UpdateRequest(index.toLowerCase(), indexType, entityId).doc(indexDocument), + RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("Exception in updating a record to ElasticSearch", e); + } + if (null == response) + return null; + return response.status(); + } + + /** + * @param index name of index + * @param indexType index type + * @param entityId entity Id + * @return status + */ + public Map readEntity(String index, String indexType, String entityId) { + logger.info("readEntity starts with index {} and entityId {}", index, entityId); + GetResponse response = null; + try { + response = esClient.get(new GetRequest(index, indexType, entityId), RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("Exception in getting the record from ElasticSearch", e); + } + if (null == response) + return null; + return response.getSourceAsMap(); + } + + /** + * Search the document in es based on provided information + * + * @param indexName es index name + * @param type index type + * @param searchSourceBuilder source builder + * @return es search response + * @throws IOException + */ + public SearchResponse getEsResult(String indexName, String type, SearchSourceBuilder searchSourceBuilder, + boolean isSunbirdES) throws IOException { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(indexName); + if (!StringUtils.isEmpty(type)) + searchRequest.types(type); + searchRequest.source(searchSourceBuilder); + return getEsResult(searchRequest, isSunbirdES); + } + + public RestStatus BulkInsert(List indexRequestList) { + BulkResponse restStatus = null; + if (!CollectionUtils.isEmpty(indexRequestList)) { + BulkRequest bulkRequest = new BulkRequest(); + indexRequestList.forEach(bulkRequest::add); + try { + restStatus = esClient.bulk(bulkRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("Exception while doing the bulk operation in ElasticSearch", e); + } + } + if (null == restStatus) + return null; + return restStatus.status(); + } + public long getDocumentCount(String index, SearchSourceBuilder searchSourceBuilder) { try { CountRequest countRequest = new CountRequest().indices(index); @@ -149,9 +153,24 @@ public long getDocumentCount(String index, SearchSourceBuilder searchSourceBuild return 0l; } } - + + public long getDocumentCount(String index, boolean isSunbirdES) { + try { + CountRequest countRequest = new CountRequest().indices(index); + if (isSunbirdES) { + return sbEsClient.count(countRequest, RequestOptions.DEFAULT).getCount(); + } else { + return esClient.count(countRequest, RequestOptions.DEFAULT).getCount(); + } + + } catch (Exception e) { + + } + return 0l; + } + private SearchResponse getEsResult(SearchRequest searchRequest, boolean isSbES) throws IOException { - if(isSbES) { + if (isSbES) { return sbEsClient.search(searchRequest, RequestOptions.DEFAULT); } else { return esClient.search(searchRequest, RequestOptions.DEFAULT); diff --git a/src/main/java/org/sunbird/profile/service/ProfileServiceImpl.java b/src/main/java/org/sunbird/profile/service/ProfileServiceImpl.java index 4cd26592a..9bf421f5f 100644 --- a/src/main/java/org/sunbird/profile/service/ProfileServiceImpl.java +++ b/src/main/java/org/sunbird/profile/service/ProfileServiceImpl.java @@ -19,6 +19,7 @@ import org.apache.commons.collections.ListUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.poi.ss.usermodel.Workbook; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -44,6 +45,7 @@ import org.sunbird.common.util.IndexerService; import org.sunbird.common.util.ProjectUtil; import org.sunbird.common.util.PropertiesCache; +import org.sunbird.core.cipher.DecryptServiceImpl; import org.sunbird.org.service.ExtendedOrgService; import org.sunbird.storage.service.StorageServiceImpl; import org.sunbird.user.report.UserReportService; @@ -89,6 +91,9 @@ public class ProfileServiceImpl implements ProfileService { @Autowired UserReportService reportService; + @Autowired + DecryptServiceImpl decryptService; + private Logger log = LoggerFactory.getLogger(getClass().getName()); private ObjectMapper ob = new ObjectMapper(); @@ -1370,15 +1375,47 @@ public SBApiResponse getUserReport() { List fields = new ArrayList(); fields.addAll(Constants.USER_ENROLMENT_REPORT_FIELDS); fields.add(Constants.ROLES); + int index = 0; + int size = 500; + boolean isCompleted = false; + List> resultArray = new ArrayList<>(); Map> userInfoMap = new HashMap>(); + Map result; + + final BoolQueryBuilder finalQuery = QueryBuilders.boolQuery(); + finalQuery.must(QueryBuilders.termQuery(Constants.STATUS, 1)); + SearchSourceBuilder sourceBuilder = null; + Workbook wb = null; + long userCount = 0l; + do { + sourceBuilder = new SearchSourceBuilder().query(finalQuery).from(index).size(size); + sourceBuilder.fetchSource(serverConfig.getEsUserReportIncludeFields(), new String[] {}); + SearchResponse searchResponse = indexerService.getEsResult(serverConfig.getSbEsUserProfileIndex(), + serverConfig.getEsProfileIndexType(), sourceBuilder, true); + + if (index == 0) { + userCount = searchResponse.getHits().getTotalHits(); + log.info(String.format("Number of users in ES index : %s", userCount)); + wb = reportService.createReportWorkbook(fields); + } + for (SearchHit hit : searchResponse.getHits()) { + result = hit.getSourceAsMap(); + resultArray.add(result); + } + processUserDetails(resultArray, userInfoMap); + wb = reportService.appendData(wb, fields, userInfoMap); + resultArray.clear(); + userInfoMap.clear(); - cassandraOperation.getAllRecords(Constants.SUNBIRD_KEY_SPACE_NAME, Constants.TABLE_USER, Constants.USER_ENROLMENT_REPORT_FIELDS, - Constants.USER_ID, userInfoMap); + index = (int) Math.min(userCount, index + size); - userUtilityService.enrichUserInfo(fields, userInfoMap); - enrichUserRoles(userInfoMap); - reportService.generateUserReport(userInfoMap, fields, response); + if (index == userCount) { + isCompleted = true; + } + } while (!isCompleted); + + reportService.completeReportWorkbook(wb, response); } catch (Exception e) { log.error("Failed to generate user report. Exception: ", e); @@ -1391,6 +1428,38 @@ public SBApiResponse getUserReport() { return response; } + private void processUserDetails(List> userMapList, + Map> userInfoMap) { + for (Map user : userMapList) { + Map userInfo = new HashMap(); + userInfo.put(Constants.USER_ID, (String) user.get(Constants.USER_ID)); + userInfo.put(Constants.FIRSTNAME, (String) user.get(Constants.FIRSTNAME)); + userInfo.put(Constants.LASTNAME, (String) user.get(Constants.LASTNAME)); + userInfo.put(Constants.ROOT_ORG_ID, (String) user.get(Constants.ROOT_ORG_ID)); + userInfo.put(Constants.CHANNEL, (String) user.get(Constants.CHANNEL)); + if (StringUtils.isNotBlank((String) user.get(Constants.EMAIL))) { + String value = decryptService.decryptString((String) user.get(Constants.EMAIL)); + userInfo.put(Constants.EMAIL, value); + } + if (StringUtils.isNotBlank((String) user.get(Constants.PHONE))) { + userInfo.put(Constants.PHONE, decryptService.decryptString((String) user.get(Constants.PHONE))); + } + String strRoles = ""; + List> roles = (List>) user.get(Constants.ROLES); + for (Map role : roles) { + String strRole = (String) role.get(Constants.ROLE); + if (StringUtils.isNotBlank(strRoles)) { + strRoles = strRoles.concat(", ").concat(strRole); + } + else { + strRoles = StringUtils.isBlank(strRole) ? "" : strRole; + } + } + userInfo.put(Constants.ROLES, strRoles); + userInfoMap.put(userInfo.get(Constants.USER_ID), userInfo); + } + } + private void enrichUserDetails(List userIdList, Map> userInfoMap, Map orgInfoMap) { long startTime = System.currentTimeMillis(); @@ -1462,47 +1531,4 @@ private void copyReportDetails(Map enrolmentReport, Map> userInfoMap) { - Iterator userIdSet = userInfoMap.keySet().iterator(); - while (userIdSet.hasNext()) { - int i = 0; - List userIds = new ArrayList(); - while (i++ < 10 && userIdSet.hasNext()) { - userIds.add(userIdSet.next()); - } - Map propertyMap = new HashMap<>(); - propertyMap.put(Constants.USER_ID, userIds); - - List> userRoleList = cassandraOperation.getRecordsByProperties( - Constants.KEYSPACE_SUNBIRD, Constants.TABLE_USER_ROLES, propertyMap, - Arrays.asList(Constants.USER_ID, Constants.ROLE, Constants.SCOPE)); - - for (Map userRole : userRoleList) { - String userId = (String) userRole.get(Constants.USER_ID); - try { - String dbScope = (String) userRole.get(Constants.SCOPE); - if (dbScope != null && dbScope.length() > 0) { - List> scopeList = ob.readValue(dbScope, ArrayList.class); - for (Map scopeObj : scopeList) { - String orgId = scopeObj.get("organisationId"); - if (StringUtils.isNotBlank(orgId) - && orgId.equalsIgnoreCase(userInfoMap.get(userId).get(Constants.ROOT_ORG_ID))) { - String existingRoles = userInfoMap.get(userId).get(Constants.ROLES); - if (StringUtils.isNotBlank(existingRoles)) { - existingRoles = existingRoles.concat(", ") - .concat((String) userRole.get(Constants.ROLE)); - } else { - existingRoles = (String) userRole.get(Constants.ROLE); - } - userInfoMap.get(userId).put(Constants.ROLES, existingRoles); - } - } - } - } catch (Exception e) { - log.error("Failed to process role. Exception: ", e); - } - } - } - } } diff --git a/src/main/java/org/sunbird/user/report/UserReportService.java b/src/main/java/org/sunbird/user/report/UserReportService.java index 4efe3269f..f3f96bd40 100644 --- a/src/main/java/org/sunbird/user/report/UserReportService.java +++ b/src/main/java/org/sunbird/user/report/UserReportService.java @@ -3,6 +3,7 @@ import java.util.List; import java.util.Map; +import org.apache.poi.ss.usermodel.Workbook; import org.sunbird.common.model.SBApiResponse; public interface UserReportService { @@ -10,6 +11,10 @@ public interface UserReportService { public void generateUserEnrolmentReport(Map> userEnrolmentMap, List fields, SBApiResponse response); - public void generateUserReport(Map> userInfoMap, List fields, - SBApiResponse response); + public Workbook createReportWorkbook(List fields); + + public Workbook appendData(Workbook wb, List fields, Map> userInfoMap); + + public void completeReportWorkbook(Workbook wb, SBApiResponse response); + } diff --git a/src/main/java/org/sunbird/user/report/UserReportServiceImpl.java b/src/main/java/org/sunbird/user/report/UserReportServiceImpl.java index acd21987e..9d0a8203e 100644 --- a/src/main/java/org/sunbird/user/report/UserReportServiceImpl.java +++ b/src/main/java/org/sunbird/user/report/UserReportServiceImpl.java @@ -80,42 +80,57 @@ public void generateUserEnrolmentReport(Map> userEnr (System.currentTimeMillis() - startTime) / 1000)); } - public void generateUserReport(Map> userInfoMap, List fields, - SBApiResponse response) { - log.info("UserReportServiceImpl:: generateUserReport started"); - long startTime = System.currentTimeMillis(); + public Workbook createReportWorkbook(List fields) { + try { + Workbook wb = new XSSFWorkbook(); + Sheet sheet = wb.createSheet("KarmayogiBharat User Details"); + int rowNum = 0; + int cellNum = 0; + Row row = sheet.createRow(rowNum++); + for (String fieldName : fields) { + Cell cell = row.createCell(cellNum++); + cell.setCellValue(fieldName); + } + return wb; + } catch (Exception e) { + log.error("Failed to create a workbook ", e); + } + return null; + } - Workbook wb = new XSSFWorkbook(); - Sheet sheet = wb.createSheet("KarmayogiBharat User Details"); + public Workbook appendData(Workbook wb, List fields, Map> userInfoMap) { int rowNum = 0; - Row row = sheet.createRow(rowNum++); int cellNum = 0; - for (String fieldName : fields) { - Cell cell = row.createCell(cellNum++); - cell.setCellValue(fieldName); - } - - Iterator>> it = userInfoMap.entrySet().iterator(); - while (it.hasNext()) { - Entry> entry = it.next(); - Row rowE = sheet.createRow(rowNum++); - cellNum = 0; - for (String field : fields) { - Cell cell = rowE.createCell(cellNum++); - String cellVal = entry.getValue().get(field); - if (StringUtils.isNotBlank(cellVal)) { - cell.setCellValue(cellVal); - } else { - cell.setCellValue(""); + try { + Sheet sheet = wb.getSheet("KarmayogiBharat User Details"); + rowNum = sheet.getLastRowNum()+1; + Iterator>> it = userInfoMap.entrySet().iterator(); + while (it.hasNext()) { + Entry> entry = it.next(); + Row rowE = sheet.createRow(rowNum++); + cellNum = 0; + for (String field : fields) { + Cell cell = rowE.createCell(cellNum++); + String cellVal = entry.getValue().get(field); + if (StringUtils.isNotBlank(cellVal)) { + cell.setCellValue(cellVal); + } else { + cell.setCellValue(""); + } } } + } catch (Exception e) { + log.error("Failed to write data into the User Report excel. Exception: ", e); + wb = null; } + return wb; + } - String fileName = userStorePath + "/userReport-" + java.time.LocalDate.now() + System.currentTimeMillis() - + ".xlsx"; - log.info("Constructed File Name -> " + fileName); - + public void completeReportWorkbook(Workbook wb, SBApiResponse response) { try { + String fileName = userStorePath + "/userReport-" + java.time.LocalDate.now() + System.currentTimeMillis() + + ".xlsx"; + log.info("Constructed File Name -> " + fileName); File file = new File(fileName); file.createNewFile(); OutputStream fileOut = new FileOutputStream(file, false); @@ -123,10 +138,7 @@ public void generateUserReport(Map> userInfoMap, Lis wb.close(); response.getResult().put(Constants.FILE_NAME, fileName); } catch (Exception e) { - log.error("Failed to write the workbook created for UserEnrolment Report. Exception: ", e); + log.error("Failed to write data into User Report excel file. Exception: e", e); } - - log.info(String.format("UserReportServiceImpl:: generateUserReport started and it took %s seconds to complete.", - (System.currentTimeMillis() - startTime) / 1000)); } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 4ce0c6b38..a009db28d 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -60,6 +60,7 @@ es.profile.index.type=_doc es.profile.source.fields=photo,id,employmentDetails,personalDetails es.user.auto.complete.search.fields=profileDetails.personalDetails.primaryEmail,profileDetails.personalDetails.firstname,profileDetails.personalDetails.surname es.user.auto.complete.include.fields=lastName,maskedPhone,rootOrgName,roles,channel,prevUsedPhone,updatedDate,stateValidated,isDeleted,organisations,managedBy,countryCode,flagsValue,id,recoveryEmail,identifier,updatedBy,phoneVerified,locationIds,recoveryPhone,rootOrgId,userId,userSubType,prevUsedEmail,emailVerified,firstName,profileLocation,createdDate,framework,tncAcceptedOn,allTncAccepted,profileDetails,createdBy,phone,profileUserType,dob,userType,tncAcceptedVersion,status +es.user.report.include.fields=userId,firstName,lastName,email,phone,rootOrgId,channel,roles #workallocation es config