diff --git a/managed/devops/bin/node_utils.sh b/managed/devops/bin/node_utils.sh index 2afacca46ada..a077603deab6 100644 --- a/managed/devops/bin/node_utils.sh +++ b/managed/devops/bin/node_utils.sh @@ -61,7 +61,7 @@ check_file_exists() { fi } -find_paths_in_dir() { +get_paths_and_sizes() { remote_dir_path=$1 shift max_depth=$1 @@ -70,12 +70,13 @@ find_paths_in_dir() { shift temp_file_path=$1 - find "$remote_dir_path" -maxdepth "$max_depth" -type "$file_type" > "$temp_file_path" + find "$remote_dir_path" -maxdepth "$max_depth" -type "$file_type" \ + -exec ls -ltp {} + | awk '{print $5, $9}' > "$temp_file_path" } -# This function returns a list of file paths and their respective sizes, in a given directory. +# This function returns a list of file names and their respective sizes, in a given directory. # Sorts the list by modification time, with newest first. -get_paths_and_sizes() { +get_paths_and_sizes_within_dates() { remote_dir_path=$1 shift temp_file_path=$1 diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateSupportBundle.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateSupportBundle.java index 1b78449b14e7..cc72ca9099a0 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateSupportBundle.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateSupportBundle.java @@ -31,6 +31,7 @@ import com.yugabyte.yw.common.operator.OperatorStatusUpdaterFactory; import com.yugabyte.yw.common.supportbundle.SupportBundleComponent; import com.yugabyte.yw.common.supportbundle.SupportBundleComponentFactory; +import com.yugabyte.yw.common.utils.Pair; import com.yugabyte.yw.controllers.handlers.UniverseInfoHandler; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.SupportBundle; @@ -43,14 +44,12 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.text.SimpleDateFormat; -import java.util.Calendar; import java.util.Date; import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.time.DateUtils; import play.libs.Json; @Slf4j @@ -60,7 +59,7 @@ public class CreateSupportBundle extends AbstractTaskBase { @Inject private UniverseInfoHandler universeInfoHandler; @Inject private SupportBundleComponentFactory supportBundleComponentFactory; @Inject private SupportBundleUtil supportBundleUtil; - @Inject private Config config; + @Inject private Config staticConfig; @Inject private NodeUniverseManager nodeUniverseManager; @Inject RuntimeConfGetter confGetter; @@ -113,30 +112,10 @@ public Path generateBundle(SupportBundle supportBundle) throws Exception { Files.createDirectories(bundlePath); log.debug("Fetching Universe {} logs", universe.getName()); - // Simplified the following 4 cases to extract appropriate start and end date - // 1. If both of the dates are given and valid - // 2. If only the start date is valid, filter from startDate till the end - // 3. If only the end date is valid, filter from the beginning till endDate - // 4. Default : If no dates are specified, download all the files from last n days - Date startDate, endDate; - boolean startDateIsValid = supportBundleUtil.isValidDate(supportBundle.getStartDate()); - boolean endDateIsValid = supportBundleUtil.isValidDate(supportBundle.getEndDate()); - if (!startDateIsValid && !endDateIsValid) { - int default_date_range = config.getInt("yb.support_bundle.default_date_range"); - endDate = supportBundleUtil.getTodaysDate(); - startDate = - DateUtils.truncate( - supportBundleUtil.getDateNDaysAgo(endDate, default_date_range), - Calendar.DAY_OF_MONTH); - } else { - // Strip the date object of the time and set only the date. - // This will ensure that we collect files inclusive of the start date. - startDate = - startDateIsValid - ? DateUtils.truncate(supportBundle.getStartDate(), Calendar.DAY_OF_MONTH) - : new Date(Long.MIN_VALUE); - endDate = endDateIsValid ? supportBundle.getEndDate() : new Date(Long.MAX_VALUE); - } + Pair datePair = + supportBundleUtil.getValidStartAndEndDates( + staticConfig, supportBundle.getStartDate(), supportBundle.getEndDate()); + Date startDate = datePair.getFirst(), endDate = datePair.getSecond(); // Add the supportBundle metadata into the bundle try { diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/SupportBundleComponentDownload.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/SupportBundleComponentDownload.java index 23dab7e30ef3..e8f0b6878cfd 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/SupportBundleComponentDownload.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/SupportBundleComponentDownload.java @@ -64,10 +64,10 @@ public void run() { } catch (Exception e) { // Log the error and continue with the rest of support bundle collection. log.error( - "Error occurred in support bundle collection for component '{}' on {} node: {}", + "Error occurred in support bundle collection for component '{}' on {} node", taskParams().supportBundleComponent.getClass().getSimpleName(), - (taskParams().node == null) ? "YBA" : taskParams().node.getNodeName(), - e.getMessage()); + (taskParams().node == null) ? "YBA" : taskParams().node.getNodeName()); + e.printStackTrace(); } } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/NodeUniverseManager.java b/managed/src/main/java/com/yugabyte/yw/common/NodeUniverseManager.java index 987f632029c4..79aa9958ea16 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/NodeUniverseManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/NodeUniverseManager.java @@ -12,7 +12,6 @@ import com.yugabyte.yw.common.config.RuntimeConfGetter; import com.yugabyte.yw.common.config.UniverseConfKeys; import com.yugabyte.yw.common.gflags.GFlagsUtil; -import com.yugabyte.yw.common.utils.Pair; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.Cluster; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.UserIntent; @@ -27,7 +26,6 @@ import java.io.IOException; import java.io.PrintWriter; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -35,12 +33,12 @@ import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.MapUtils; import org.apache.commons.io.FileUtils; @@ -730,16 +728,16 @@ public boolean isNodeReachable(NodeDetails node, Universe universe, long timeout } /** - * Gets a list of all the absolute file paths at a given remote directory + * Gets a map of all the absolute file paths to sizes at a given remote directory * * @param node * @param universe * @param remoteDirPath * @param maxDepth * @param fileType - * @return list of strings of all the absolute file paths + * @return map of absolute file paths to file sizes */ - public List getNodeFilePaths( + public Map getNodeFilePathAndSizes( NodeDetails node, Universe universe, String remoteDirPath, int maxDepth, String fileType) { String localTempFilePath = getLocalTmpDir() + "/" + UUID.randomUUID().toString() + "-source-files-unfiltered.txt"; @@ -750,7 +748,7 @@ public List getNodeFilePaths( + "-source-files-unfiltered.txt"; List findCommandParams = new ArrayList<>(); - findCommandParams.add("find_paths_in_dir"); + findCommandParams.add("get_paths_and_sizes"); findCommandParams.add(remoteDirPath); findCommandParams.add(String.valueOf(maxDepth)); findCommandParams.add(fileType); @@ -768,18 +766,27 @@ public List getNodeFilePaths( // Populate the text file into array. List nodeFilePathStrings = Arrays.asList(); + // LinkedHashMap to maintain insertion order. + // Files are ordered most recent to least. + Map nodeFilePathSizeMap = new LinkedHashMap<>(); try { nodeFilePathStrings = Files.readAllLines(Paths.get(localTempFilePath)); + for (String outputLine : nodeFilePathStrings) { + String[] outputLineSplit = outputLine.split("\\s+", 2); + if (!StringUtils.isBlank(outputLine) && outputLineSplit.length == 2) { + nodeFilePathSizeMap.put(outputLineSplit[1], Long.valueOf(outputLineSplit[0])); + } + } } catch (IOException e) { log.error("Error occurred", e); } finally { FileUtils.deleteQuietly(new File(localTempFilePath)); } - return nodeFilePathStrings.stream().map(Paths::get).collect(Collectors.toList()); + return nodeFilePathSizeMap; } /** - * Returns a list of file sizes (in bytes) and their names present in a remote directory on the + * Returns a map of file names to their sizes (in bytes) present in a remote directory on the * node. This function creates a temp file with these sizes and names and copies the temp file * from remote to local. Then reads and processes this info from the local temp file. This is done * so that this operation is scalable for large number of files present on the node. @@ -787,9 +794,9 @@ public List getNodeFilePaths( * @param node * @param universe * @param remoteDirPath - * @return the list of pairs (size, name) + * @return a map of filenames to filesizes */ - public List> getNodeFilePathsAndSize( + public Map getNodeFilePathsAndSizeWithinDates( NodeDetails node, Universe universe, String remoteDirPath, Date startDate, Date endDate) { String randomUUIDStr = UUID.randomUUID().toString(); String localTempFilePath = @@ -799,7 +806,7 @@ public List> getNodeFilePathsAndSize( SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); List findCommandParams = new ArrayList<>(); - findCommandParams.add("get_paths_and_sizes"); + findCommandParams.add("get_paths_and_sizes_within_dates"); findCommandParams.add(remoteDirPath); findCommandParams.add(remoteTempFilePath); findCommandParams.add(formatter.format(startDate)); @@ -817,15 +824,16 @@ public List> getNodeFilePathsAndSize( // Populate the text file into array. List nodeFilePathStrings = Arrays.asList(); - List> nodeFileSizePathStrings = new ArrayList<>(); + // LinkedHashMap to maintain insertion order. + // Files are ordered most recent to least. + Map nodeFilePathSizeMap = new LinkedHashMap<>(); try { nodeFilePathStrings = Files.readAllLines(Paths.get(localTempFilePath)); log.debug("List of files found on the node '{}': '{}'", node.nodeName, nodeFilePathStrings); for (String outputLine : nodeFilePathStrings) { String[] outputLineSplit = outputLine.split("\\s+", 2); if (!StringUtils.isBlank(outputLine) && outputLineSplit.length == 2) { - nodeFileSizePathStrings.add( - new Pair<>(Long.valueOf(outputLineSplit[0]), outputLineSplit[1])); + nodeFilePathSizeMap.put(outputLineSplit[1], Long.valueOf(outputLineSplit[0])); } } } catch (IOException e) { @@ -833,7 +841,7 @@ public List> getNodeFilePathsAndSize( } finally { FileUtils.deleteQuietly(new File(localTempFilePath)); } - return nodeFileSizePathStrings; + return nodeFilePathSizeMap; } public enum UniverseNodeAction { diff --git a/managed/src/main/java/com/yugabyte/yw/common/SupportBundleUtil.java b/managed/src/main/java/com/yugabyte/yw/common/SupportBundleUtil.java index 411b50527c1f..85f97b9196f3 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/SupportBundleUtil.java +++ b/managed/src/main/java/com/yugabyte/yw/common/SupportBundleUtil.java @@ -17,6 +17,7 @@ import com.typesafe.config.Config; import com.yugabyte.yw.common.KubernetesManager.RoleData; import com.yugabyte.yw.common.RedactingService.RedactionTarget; +import com.yugabyte.yw.common.utils.Pair; import com.yugabyte.yw.controllers.handlers.UniverseInfoHandler; import com.yugabyte.yw.forms.UniverseResp; import com.yugabyte.yw.models.Audit; @@ -50,6 +51,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Calendar; import java.util.Collections; import java.util.Comparator; import java.util.Date; @@ -130,6 +132,34 @@ public Date getDateNMinutesAgo(Date date, int minutes) { return dateNMinutesAgo; } + /** + * Simplified the following 4 cases to extract appropriate start and end date 1. If both of the + * dates are given and valid 2. If only the start date is valid, filter from startDate till the + * end 3. If only the end date is valid, filter from the beginning till endDate 4. Default : If no + * dates are specified, download all the files from last n days + */ + public Pair getValidStartAndEndDates(Config config, Date sDate, Date eDate) + throws Exception { + Date startDate, endDate; + boolean startDateIsValid = isValidDate(sDate); + boolean endDateIsValid = isValidDate(eDate); + if (!startDateIsValid && !endDateIsValid) { + int default_date_range = config.getInt("yb.support_bundle.default_date_range"); + endDate = getTodaysDate(); + startDate = + DateUtils.truncate(getDateNDaysAgo(endDate, default_date_range), Calendar.DAY_OF_MONTH); + } else { + // Strip the date object of the time and set only the date. + // This will ensure that we collect files inclusive of the start date. + startDate = + startDateIsValid + ? DateUtils.truncate(sDate, Calendar.DAY_OF_MONTH) + : new Date(Long.MIN_VALUE); + endDate = endDateIsValid ? eDate : new Date(Long.MAX_VALUE); + } + return new Pair(startDate, endDate); + } + public List sortDatesWithPattern(List datesList, String sdfPattern) { // Sort the list of dates based on the given 'SimpleDateFormat' pattern List sortedList = new ArrayList(datesList); @@ -158,11 +188,11 @@ public int compare(String o1, String o2) { * @param regexList list of regex strings to match against any of them * @return list of paths after regex filtering */ - public List filterList(List list, List regexList) { - List result = new ArrayList(); - for (Path entry : list) { + public List filterList(List list, List regexList) { + List result = new ArrayList(); + for (String entry : list) { for (String regex : regexList) { - if (entry.toString().matches(regex)) { + if (entry.matches(regex)) { result.add(entry); } } @@ -282,12 +312,12 @@ public Date extractDateFromFileNameAndRegex(String fileName, List fileRe * @return list of paths after filtering based on dates. * @throws ParseException */ - public List filterFilePathsBetweenDates( - List logFilePaths, List fileRegexList, Date startDate, Date endDate) + public List filterFilePathsBetweenDates( + List logFilePaths, List fileRegexList, Date startDate, Date endDate) throws ParseException { // Final filtered log paths - List filteredLogFilePaths = new ArrayList<>(); + List filteredLogFilePaths = new ArrayList<>(); // Initial filtering of the file names based on regex logFilePaths = filterList(logFilePaths, fileRegexList); @@ -300,22 +330,21 @@ public List filterFilePathsBetweenDates( // "/mnt/d0/master/logs/log.INFO.20221121-000000.log"]} // The reason we don't use a map of > is because we need to return the // entire path. - Map> fileTypeToDate = + Map> fileTypeToDate = logFilePaths.stream() .collect( - Collectors.groupingBy( - p -> extractFileTypeFromFileNameAndRegex(p.toString(), fileRegexList))); + Collectors.groupingBy(p -> extractFileTypeFromFileNameAndRegex(p, fileRegexList))); // Loop through each file type for (String fileType : fileTypeToDate.keySet()) { // Sort the files in descending order of extracted date Collections.sort( fileTypeToDate.get(fileType), - new Comparator() { + new Comparator() { @Override - public int compare(Path path1, Path path2) { - Date date1 = extractDateFromFileNameAndRegex(path1.toString(), fileRegexList); - Date date2 = extractDateFromFileNameAndRegex(path2.toString(), fileRegexList); + public int compare(String path1, String path2) { + Date date1 = extractDateFromFileNameAndRegex(path1, fileRegexList); + Date date2 = extractDateFromFileNameAndRegex(path2, fileRegexList); return date2.compareTo(date1); } }); @@ -323,9 +352,8 @@ public int compare(Path path1, Path path2) { // Filter file paths according to start and end dates // Add filtered date paths to final list Date extraStartDate = null; - for (Path filePathToCheck : fileTypeToDate.get(fileType)) { - Date dateToCheck = - extractDateFromFileNameAndRegex(filePathToCheck.toString(), fileRegexList); + for (String filePathToCheck : fileTypeToDate.get(fileType)) { + Date dateToCheck = extractDateFromFileNameAndRegex(filePathToCheck, fileRegexList); if (checkDateBetweenDates(dateToCheck, startDate, endDate)) { filteredLogFilePaths.add(filePathToCheck); } @@ -337,7 +365,6 @@ public int compare(Path path1, Path path2) { } } } - return filteredLogFilePaths; } @@ -791,7 +818,7 @@ public void saveMetadata(Customer customer, String destDir, JsonNode jsonData, S public void getCustomerMetadata(Customer customer, String destDir) { // Gather metadata. JsonNode jsonData = - RedactingService.filterSecretFields(Json.toJson(customer), RedactionTarget.APIS); + RedactingService.filterSecretFields(Json.toJson(customer), RedactionTarget.LOGS); // Save the above collected metadata. saveMetadata(customer, destDir, jsonData, "customer.json"); @@ -802,7 +829,7 @@ public void getUniversesMetadata(Customer customer, String destDir) { List universes = customer.getUniverses().stream().map(u -> new UniverseResp(u)).collect(Collectors.toList()); JsonNode jsonData = - RedactingService.filterSecretFields(Json.toJson(universes), RedactionTarget.APIS); + RedactingService.filterSecretFields(Json.toJson(universes), RedactionTarget.LOGS); // Save the above collected metadata. saveMetadata(customer, destDir, jsonData, "universes.json"); @@ -813,7 +840,7 @@ public void getProvidersMetadata(Customer customer, String destDir) { List providers = Provider.getAll(customer.getUuid()); providers.forEach(CloudInfoInterface::mayBeMassageResponse); JsonNode jsonData = - RedactingService.filterSecretFields(Json.toJson(providers), RedactionTarget.APIS); + RedactingService.filterSecretFields(Json.toJson(providers), RedactionTarget.LOGS); // Save the above collected metadata. saveMetadata(customer, destDir, jsonData, "providers.json"); @@ -823,7 +850,7 @@ public void getUsersMetadata(Customer customer, String destDir) { // Gather metadata. List users = Users.getAll(customer.getUuid()); JsonNode jsonData = - RedactingService.filterSecretFields(Json.toJson(users), RedactionTarget.APIS); + RedactingService.filterSecretFields(Json.toJson(users), RedactionTarget.LOGS); // Save the above collected metadata. saveMetadata(customer, destDir, jsonData, "users.json"); @@ -867,7 +894,7 @@ public void getInstanceTypeMetadata(Customer customer, String destDir) { .filter(it -> providerUUIDs.contains(it.getProvider().getUuid())) .collect(Collectors.toList()); JsonNode jsonData = - RedactingService.filterSecretFields(Json.toJson(instanceTypes), RedactionTarget.APIS); + RedactingService.filterSecretFields(Json.toJson(instanceTypes), RedactionTarget.LOGS); // Save the above collected metadata. saveMetadata(customer, destDir, jsonData, "instance_type.json"); diff --git a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/ApplicationLogsComponent.java b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/ApplicationLogsComponent.java index 25903c39feaf..a3320b5029f0 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/ApplicationLogsComponent.java +++ b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/ApplicationLogsComponent.java @@ -8,6 +8,7 @@ import com.yugabyte.yw.commissioner.BaseTaskDependencies; import com.yugabyte.yw.commissioner.tasks.params.SupportBundleTaskParams; import com.yugabyte.yw.common.SupportBundleUtil; +import com.yugabyte.yw.forms.SupportBundleFormData; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.helpers.NodeDetails; @@ -17,12 +18,14 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; @@ -69,8 +72,38 @@ public void downloadComponentBetweenDates( Date startDate, Date endDate, NodeDetails node) - throws IOException, ParseException { + throws Exception { + // Create "application_logs" folder inside the support bundle folder + String destDir = bundlePath.toString() + "/application_logs"; + Files.createDirectories(Paths.get(destDir)); + File dest = new File(destDir); + + // Set of absolute paths to be copied to the support bundle directory + Set filteredLogFiles = + getFilesListWithSizes(customer, null, universe, startDate, endDate, node).keySet(); + + // Copy individual files from source directory to the support bundle folder + for (String filteredLogFile : filteredLogFiles) { + Path sourceFilePath = Paths.get(filteredLogFile); + Path destFilePath = + Paths.get(dest.toString(), Paths.get(filteredLogFile).getFileName().toString()); + Files.copy(sourceFilePath, destFilePath, StandardCopyOption.REPLACE_EXISTING); + } + + log.debug("Downloaded application logs to {}, between {} and {}", destDir, startDate, endDate); + } + + public Map getFilesListWithSizes( + Customer customer, + SupportBundleFormData bundleData, + Universe universe, + Date startDate, + Date endDate, + NodeDetails node) + throws Exception { + + Map res = new HashMap<>(); // Get application configured locations String appHomeDir = config.getString("application.home"); log.info("[ApplicationLogsComponent] appHomeDir = '{}'", appHomeDir); @@ -80,11 +113,7 @@ public void downloadComponentBetweenDates( log.info("[ApplicationLogsComponent] logDir = '{}'", logDir); log.info("[ApplicationLogsComponent] logDirAbsolute = '{}'", logDirAbsolute); - // Create "application_logs" folder inside the support bundle folder - String destDir = bundlePath.toString() + "/application_logs"; - Files.createDirectories(Paths.get(destDir)); File source = new File(logDirAbsolute); - File dest = new File(destDir); // Get all the log file names present in source directory List logFiles = new ArrayList<>(); @@ -116,12 +145,7 @@ public void downloadComponentBetweenDates( String applicationLogsRegexPattern = config.getString("yb.support_bundle.application_logs_regex_pattern"); logFiles = - supportBundleUtil - .filterList( - logFiles.stream().map(Paths::get).collect(Collectors.toList()), - Arrays.asList(applicationLogsRegexPattern)) - .stream() - .map(Path::toString) + supportBundleUtil.filterList(logFiles, Arrays.asList(applicationLogsRegexPattern)).stream() .collect(Collectors.toList()); String applicationLogsSdfPattern = @@ -138,14 +162,10 @@ public void downloadComponentBetweenDates( } } - // Copy individual files from source directory to the support bundle folder - for (String filteredLogFile : filteredLogFiles) { - Path sourceFilePath = Paths.get(source.toString(), filteredLogFile); - Path destFilePath = Paths.get(dest.toString(), filteredLogFile); - Files.copy(sourceFilePath, destFilePath, StandardCopyOption.REPLACE_EXISTING); + for (String logFile : filteredLogFiles) { + Path absolutePath = Paths.get(source.toString(), logFile); + res.put(absolutePath.toString(), absolutePath.toFile().length()); } - - log.debug("Downloaded application logs to {}, between {} and {}", destDir, startDate, endDate); + return res; } - ; } diff --git a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/ConsensusMetaComponent.java b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/ConsensusMetaComponent.java index 07366b897c53..3971cb78fc09 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/ConsensusMetaComponent.java +++ b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/ConsensusMetaComponent.java @@ -7,13 +7,17 @@ import com.yugabyte.yw.common.NodeUniverseManager; import com.yugabyte.yw.common.SupportBundleUtil; import com.yugabyte.yw.controllers.handlers.UniverseInfoHandler; +import com.yugabyte.yw.forms.SupportBundleFormData; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.helpers.NodeDetails; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -77,4 +81,24 @@ public void downloadComponentBetweenDates( throws Exception { this.downloadComponent(supportBundleTaskParams, customer, universe, bundlePath, node); } + + public Map getFilesListWithSizes( + Customer customer, + SupportBundleFormData bundleData, + Universe universe, + Date startDate, + Date endDate, + NodeDetails node) + throws Exception { + + String mountPath = + supportBundleUtil.getDataDirPath(universe, node, nodeUniverseManager, config); + String nodeHomeDir = mountPath + "/yb-data"; + Map res = new HashMap<>(); + for (String path : sourceNodeFiles) { + String fullPath = Paths.get(nodeHomeDir, path).toString(); + res.putAll(nodeUniverseManager.getNodeFilePathAndSizes(node, universe, fullPath, 1, "f")); + } + return res; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/CoreFilesComponent.java b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/CoreFilesComponent.java index 804f9b1812d5..df24a9654eea 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/CoreFilesComponent.java +++ b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/CoreFilesComponent.java @@ -7,8 +7,8 @@ import com.yugabyte.yw.commissioner.tasks.params.SupportBundleTaskParams; import com.yugabyte.yw.common.NodeUniverseManager; import com.yugabyte.yw.common.SupportBundleUtil; -import com.yugabyte.yw.common.utils.Pair; import com.yugabyte.yw.controllers.handlers.UniverseInfoHandler; +import com.yugabyte.yw.forms.SupportBundleFormData; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.helpers.NodeDetails; @@ -16,7 +16,9 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Date; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -64,22 +66,15 @@ public void downloadComponentBetweenDates( bundlePath = Paths.get(bundlePath.toAbsolutePath().toString(), "cores"); Files.createDirectories(bundlePath); - // Get and filter the core files list based on the 2 params given in the request body. - List> fileSizeNameList = - nodeUniverseManager - .getNodeFilePathsAndSize(node, universe, coresDir, startDate, endDate) - .stream() - .limit(supportBundleTaskParams.bundleData.maxNumRecentCores) - .filter(p -> p.getFirst() <= supportBundleTaskParams.bundleData.maxCoreFileSize) - .collect(Collectors.toList()); - - // Filter the core files list based on the 2 params given in the request body. - List sourceNodeFiles = - fileSizeNameList.stream().map(p -> p.getSecond()).collect(Collectors.toList()); + Map fileNameSizeMap = + getFilesListWithSizes( + customer, supportBundleTaskParams.bundleData, universe, startDate, endDate, node); + + List sourceNodeFiles = fileNameSizeMap.keySet().stream().collect(Collectors.toList()); // Check if YBA node has enough space to store all cores files from the DB nodes. long YbaDiskSpaceFreeInBytes = Files.getFileStore(bundlePath).getUsableSpace(); - long coreFilesSize = fileSizeNameList.stream().mapToLong(Pair::getFirst).sum(); + long coreFilesSize = fileNameSizeMap.values().stream().mapToLong(Long::longValue).sum(); // Throw error if YBA node doesn't have enough space to download the core files from this DB // node. Exception is caught in CreateSupportBundle.java to continue with rest of support bundle @@ -109,4 +104,46 @@ public void downloadComponentBetweenDates( this.getClass().getSimpleName(), true /* skipUntar */); } + + public Map getFilesListWithSizes( + Customer customer, + SupportBundleFormData bundleData, + Universe universe, + Date startDate, + Date endDate, + NodeDetails node) + throws Exception { + + String nodeHomeDir = nodeUniverseManager.getYbHomeDir(node, universe); + String coresDir = nodeHomeDir + "/cores/"; + + // Get and filter the core files list based on the 2 params given in the request body. + Map fileNameSizeMap = + nodeUniverseManager.getNodeFilePathsAndSizeWithinDates( + node, universe, coresDir, startDate, endDate); + fileNameSizeMap = + filterFirstNEntries( + fileNameSizeMap, bundleData.maxNumRecentCores, bundleData.maxCoreFileSize); + return fileNameSizeMap; + } + + /** Returns a map with the first numEntries which are <= given maxSize. */ + private Map filterFirstNEntries( + Map map, int numEntries, long maxSize) { + LinkedHashMap result = new LinkedHashMap<>(); + if (numEntries <= 0) { + return result; + } + int count = 0; + for (Map.Entry entry : map.entrySet()) { + if (entry.getValue() <= maxSize) { + result.put(entry.getKey(), entry.getValue()); + count++; + if (count == numEntries) { + break; + } + } + } + return result; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/ErrorFilesComponent.java b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/ErrorFilesComponent.java index b2e9d75aea61..7d92e5e3e3aa 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/ErrorFilesComponent.java +++ b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/ErrorFilesComponent.java @@ -6,13 +6,16 @@ import com.yugabyte.yw.common.NodeUniverseManager; import com.yugabyte.yw.common.SupportBundleUtil; import com.yugabyte.yw.controllers.handlers.UniverseInfoHandler; +import com.yugabyte.yw.forms.SupportBundleFormData; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.helpers.NodeDetails; import java.nio.file.Path; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -68,4 +71,21 @@ public void downloadComponentBetweenDates( throws Exception { this.downloadComponent(supportBundleTaskParams, customer, universe, bundlePath, node); } + + // This component collects two files only, so instead of getting the actual file + // sizes from db nodes just adding 5KB for each file. Shouldn't affect end + // result too much. + public Map getFilesListWithSizes( + Customer customer, + SupportBundleFormData bundleData, + Universe universe, + Date startDate, + Date endDate, + NodeDetails node) + throws Exception { + + Map res = new HashMap<>(); + res.put("Sample File", 10000L); + return res; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/GFlagsComponent.java b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/GFlagsComponent.java index 9a8f887f2e55..5c6f6414165d 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/GFlagsComponent.java +++ b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/GFlagsComponent.java @@ -6,13 +6,16 @@ import com.yugabyte.yw.common.NodeUniverseManager; import com.yugabyte.yw.common.SupportBundleUtil; import com.yugabyte.yw.controllers.handlers.UniverseInfoHandler; +import com.yugabyte.yw.forms.SupportBundleFormData; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.helpers.NodeDetails; import java.nio.file.Path; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -68,4 +71,21 @@ public void downloadComponentBetweenDates( throws Exception { this.downloadComponent(supportBundleTaskParams, customer, universe, bundlePath, node); } + + // This component collects two conf files only, so instead of getting the actual file + // sizes from db nodes just adding 5KB for each file. Shouldn't affect end + // result too much. + public Map getFilesListWithSizes( + Customer customer, + SupportBundleFormData bundleData, + Universe universe, + Date startDate, + Date endDate, + NodeDetails node) + throws Exception { + + Map res = new HashMap<>(); + res.put("Sample File", 10000L); + return res; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/InstanceComponent.java b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/InstanceComponent.java index 7522c276a03d..777f04e6f42f 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/InstanceComponent.java +++ b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/InstanceComponent.java @@ -7,13 +7,16 @@ import com.yugabyte.yw.common.NodeUniverseManager; import com.yugabyte.yw.common.SupportBundleUtil; import com.yugabyte.yw.controllers.handlers.UniverseInfoHandler; +import com.yugabyte.yw.forms.SupportBundleFormData; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.helpers.NodeDetails; import java.nio.file.Path; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -77,4 +80,21 @@ public void downloadComponentBetweenDates( throws Exception { this.downloadComponent(supportBundleTaskParams, customer, universe, bundlePath, node); } + + // This component collects two files only, so instead of getting the actual file + // sizes from db nodes just adding 5KB for each file. Shouldn't affect end + // result too much. + public Map getFilesListWithSizes( + Customer customer, + SupportBundleFormData bundleData, + Universe universe, + Date startDate, + Date endDate, + NodeDetails node) + throws Exception { + + Map res = new HashMap<>(); + res.put("Sample File", 10000L); + return res; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/K8sInfoComponent.java b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/K8sInfoComponent.java index f93f628dfa66..74b02e51c570 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/K8sInfoComponent.java +++ b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/K8sInfoComponent.java @@ -21,6 +21,7 @@ import com.yugabyte.yw.common.SupportBundleUtil; import com.yugabyte.yw.common.SupportBundleUtil.KubernetesCluster; import com.yugabyte.yw.common.SupportBundleUtil.KubernetesResourceType; +import com.yugabyte.yw.forms.SupportBundleFormData; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.Cluster; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.ClusterType; @@ -452,4 +453,20 @@ public void downloadComponentBetweenDates( throws Exception { this.downloadComponent(supportBundleTaskParams, customer, universe, bundlePath, node); } + + // This component collects few yaml conf files only, so instead of getting the actual file + // sizes from db nodes just adding 300KB. Shouldn't affect end + // result too much. + public Map getFilesListWithSizes( + Customer customer, + SupportBundleFormData bundleData, + Universe universe, + Date startDate, + Date endDate, + NodeDetails node) + throws Exception { + Map res = new HashMap(); + res.put("default value", 300000L); + return res; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/NodeAgentComponent.java b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/NodeAgentComponent.java index 32c6c9afcabd..cdea51884059 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/NodeAgentComponent.java +++ b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/NodeAgentComponent.java @@ -8,6 +8,7 @@ import com.yugabyte.yw.common.NodeUniverseManager; import com.yugabyte.yw.common.SupportBundleUtil; import com.yugabyte.yw.controllers.handlers.UniverseInfoHandler; +import com.yugabyte.yw.forms.SupportBundleFormData; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.NodeAgent; import com.yugabyte.yw.models.Universe; @@ -15,11 +16,12 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @Slf4j @@ -47,37 +49,23 @@ public void downloadComponent( Path bundlePath, NodeDetails node) throws Exception { - if (node.cloudInfo == null || StringUtils.isBlank(node.cloudInfo.private_ip)) { - log.info("Skipping node-agent log download as node IP is not available"); + Map logsPathSizeMap = + getFilesListWithSizes(customer, null, universe, null, null, node); + if (logsPathSizeMap.isEmpty()) { return; } Optional optional = NodeAgent.maybeGetByIp(node.cloudInfo.private_ip); - if (!optional.isPresent()) { - log.info("Skipping node-agent log download as node-agent is not installed"); - return; - } Path nodeAgentHome = Paths.get(optional.get().getHome()); // Get target file path String nodeName = node.getNodeName(); Path nodeTargetFile = Paths.get(bundlePath.toString(), getClass().getSimpleName() + ".tar.gz"); log.debug( - "Gathering universe logs for node: {}, source path: {}, target path: {} ", + "Gathering node agent logs for node: {}, source path: {}, target path: {} ", nodeName, nodeAgentHome, nodeTargetFile); - Path nodeAgentLogDirPath = nodeAgentHome.resolve("logs"); - if (!nodeUniverseManager.checkNodeIfFileExists( - node, universe, nodeAgentLogDirPath.toString())) { - log.info("Skipping node-agent log download as {} does not exists", nodeAgentLogDirPath); - return; - } List nodeAgentLogFilePaths = - nodeUniverseManager.getNodeFilePaths( - node, universe, nodeAgentLogDirPath.toString(), /*maxDepth*/ 1, /*fileType*/ "f"); - if (CollectionUtils.isEmpty(nodeAgentLogFilePaths)) { - log.info("Skipping node-agent log download as no file exists in {}", nodeAgentLogDirPath); - return; - } + logsPathSizeMap.keySet().stream().map(Paths::get).collect(Collectors.toList()); // Relativize from the parent to include node-agent folder in the tgz. Path nodeAgentHomeParent = nodeAgentHome.getParent(); List relativeLogFilePaths = @@ -112,4 +100,35 @@ public void downloadComponentBetweenDates( // Simply return all the logs files as this method is just an overkill for now. downloadComponent(supportBundleTaskParams, customer, universe, bundlePath, node); } + + public Map getFilesListWithSizes( + Customer customer, + SupportBundleFormData bundleData, + Universe universe, + Date startDate, + Date endDate, + NodeDetails node) + throws Exception { + Map res = new HashMap<>(); + if (node.cloudInfo == null || StringUtils.isBlank(node.cloudInfo.private_ip)) { + log.info("Skipping node-agent log download as node IP is not available"); + return res; + } + Optional optional = NodeAgent.maybeGetByIp(node.cloudInfo.private_ip); + if (!optional.isPresent()) { + log.info("Skipping node-agent log download as node-agent is not installed"); + return res; + } + Path nodeAgentHome = Paths.get(optional.get().getHome()); + Path nodeAgentLogDirPath = nodeAgentHome.resolve("logs"); + if (!nodeUniverseManager.checkNodeIfFileExists( + node, universe, nodeAgentLogDirPath.toString())) { + log.info("Skipping node-agent log download as {} does not exists", nodeAgentLogDirPath); + return res; + } + res = + nodeUniverseManager.getNodeFilePathAndSizes( + node, universe, nodeAgentLogDirPath.toString(), /* maxDepth */ 1, /* fileType */ "f"); + return res; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/OutputFilesComponent.java b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/OutputFilesComponent.java index 11f3a2e6f0e4..47bdb0367cf8 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/OutputFilesComponent.java +++ b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/OutputFilesComponent.java @@ -6,13 +6,16 @@ import com.yugabyte.yw.common.NodeUniverseManager; import com.yugabyte.yw.common.SupportBundleUtil; import com.yugabyte.yw.controllers.handlers.UniverseInfoHandler; +import com.yugabyte.yw.forms.SupportBundleFormData; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.helpers.NodeDetails; import java.nio.file.Path; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -68,4 +71,21 @@ public void downloadComponentBetweenDates( throws Exception { this.downloadComponent(supportBundleTaskParams, customer, universe, bundlePath, node); } + + // This component collects two files only, so instead of getting the actual file + // sizes from db nodes just adding 5KB for each file. Shouldn't affect end + // result too much. + public Map getFilesListWithSizes( + Customer customer, + SupportBundleFormData bundleData, + Universe universe, + Date startDate, + Date endDate, + NodeDetails node) + throws Exception { + + Map res = new HashMap<>(); + res.put("Sample File", 10000L); + return res; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/PrometheusMetricsComponent.java b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/PrometheusMetricsComponent.java index cc254bb2ddd9..29bc8f9bcc21 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/PrometheusMetricsComponent.java +++ b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/PrometheusMetricsComponent.java @@ -23,6 +23,8 @@ import java.time.Duration; import java.util.Date; import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import play.libs.Json; @@ -201,21 +203,7 @@ public void downloadComponent( log.debug("Collecting the following Prometheus metrics: {}", data.prometheusMetricsTypes); - // validate the start & end dates of prometheus metrics dump - // 1. If both the dates are given; Continue - // 2. If no dates are specified, download all the exports from last 'x' duration - boolean startDateIsValid = supportBundleUtil.isValidDate(data.promDumpStartDate); - boolean endDateIsValid = supportBundleUtil.isValidDate(data.promDumpEndDate); - if (!startDateIsValid && !endDateIsValid) { - int defaultPromDumpRange = - confGetter.getGlobalConf(GlobalConfKeys.supportBundleDefaultPromDumpRange); - log.debug( - "'promDumpStartDate' and 'promDumpEndDate' are not valid. Defaulting the duration to {}", - defaultPromDumpRange); - data.promDumpEndDate = data.endDate; - data.promDumpStartDate = - supportBundleUtil.getDateNMinutesAgo(data.promDumpEndDate, defaultPromDumpRange); - } + dateValidation(data); // loop through the requested metric types for (PrometheusMetricsType type : data.prometheusMetricsTypes) { @@ -254,4 +242,76 @@ public void downloadComponentBetweenDates( throws Exception { this.downloadComponent(supportBundleTaskParams, customer, universe, bundlePath, node); } + + // It's hard to know exact sizes before actually collecting the data. + // So we add estimates based on date collected from various dev portal LRUs with varying node + // numbers. + // Can adjust the estimates later if they are too off. + public Map getFilesListWithSizes( + Customer customer, + SupportBundleFormData bundleData, + Universe universe, + Date startDate, + Date endDate, + NodeDetails node) + throws Exception { + + dateValidation(bundleData); + Long totalMins = + TimeUnit.MILLISECONDS.toMinutes( + bundleData.promDumpEndDate.getTime() - bundleData.promDumpStartDate.getTime()); + // By default we collect prom metrics in 15 min batches. + Long timeMultiplier = (totalMins + 14) / 15; + Map res = new HashMap(); + Long sum = 0L, numNodes = 1L * universe.getNodes().size(); + for (PrometheusMetricsType type : bundleData.prometheusMetricsTypes) { + switch (type) { + // 600KB per db node per 15 mins + case NODE_EXPORT: + sum += 600000 * numNodes * timeMultiplier; + break; + // 3MB per master per 15 mins + case MASTER_EXPORT: + sum += 3000000L * universe.getMasters().size() * timeMultiplier; + break; + // 110KB per 15 mins + case PLATFORM: + sum += 110000L * timeMultiplier; + break; + // 400KB per universe per 15 mins + case PROMETHEUS: + sum += 400000L * Universe.find.all().size() * timeMultiplier; + break; + // 3MB per tserver per 15 mins + case TSERVER_EXPORT: + sum += 3000000L * universe.getTServers().size() * timeMultiplier; + break; + // 300KB per node per 15 mins + case YSQL_EXPORT: + // Intentional fallthrough + case CQL_EXPORT: + sum += 300000L * numNodes * timeMultiplier; + } + } + res.put("promSizeEstimate", sum); + return res; + } + + // validate the start & end dates of prometheus metrics dump + // 1. If both the dates are given; Continue + // 2. If no dates are specified, download all the exports from last 'x' duration + private void dateValidation(SupportBundleFormData data) { + boolean startDateIsValid = supportBundleUtil.isValidDate(data.promDumpStartDate); + boolean endDateIsValid = supportBundleUtil.isValidDate(data.promDumpEndDate); + if (!startDateIsValid && !endDateIsValid) { + int defaultPromDumpRange = + confGetter.getGlobalConf(GlobalConfKeys.supportBundleDefaultPromDumpRange); + log.debug( + "'promDumpStartDate' and 'promDumpEndDate' are not valid. Defaulting the duration to {}", + defaultPromDumpRange); + data.promDumpEndDate = data.endDate; + data.promDumpStartDate = + supportBundleUtil.getDateNMinutesAgo(data.promDumpEndDate, defaultPromDumpRange); + } + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/SupportBundleComponent.java b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/SupportBundleComponent.java index 41b791e645ba..68c94b1c1219 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/SupportBundleComponent.java +++ b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/SupportBundleComponent.java @@ -1,11 +1,13 @@ package com.yugabyte.yw.common.supportbundle; import com.yugabyte.yw.commissioner.tasks.params.SupportBundleTaskParams; +import com.yugabyte.yw.forms.SupportBundleFormData; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.helpers.NodeDetails; import java.nio.file.Path; import java.util.Date; +import java.util.Map; public interface SupportBundleComponent { @@ -26,4 +28,13 @@ void downloadComponentBetweenDates( Date endDate, NodeDetails node) throws Exception; + + public Map getFilesListWithSizes( + Customer customer, + SupportBundleFormData bundleData, + Universe universe, + Date startDate, + Date endDate, + NodeDetails node) + throws Exception; } diff --git a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/TabletMetaComponent.java b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/TabletMetaComponent.java index ba6bfa7cb9af..d47b46026285 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/TabletMetaComponent.java +++ b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/TabletMetaComponent.java @@ -7,13 +7,17 @@ import com.yugabyte.yw.common.NodeUniverseManager; import com.yugabyte.yw.common.SupportBundleUtil; import com.yugabyte.yw.controllers.handlers.UniverseInfoHandler; +import com.yugabyte.yw.forms.SupportBundleFormData; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.helpers.NodeDetails; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -77,4 +81,24 @@ public void downloadComponentBetweenDates( throws Exception { this.downloadComponent(supportBundleTaskParams, customer, universe, bundlePath, node); } + + public Map getFilesListWithSizes( + Customer customer, + SupportBundleFormData bundleData, + Universe universe, + Date startDate, + Date endDate, + NodeDetails node) + throws Exception { + + String mountPath = + supportBundleUtil.getDataDirPath(universe, node, nodeUniverseManager, config); + String nodeHomeDir = mountPath + "/yb-data"; + Map res = new HashMap<>(); + for (String path : sourceNodeFiles) { + String fullPath = Paths.get(nodeHomeDir, path).toString(); + res.putAll(nodeUniverseManager.getNodeFilePathAndSizes(node, universe, fullPath, 1, "f")); + } + return res; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/UniverseLogsComponent.java b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/UniverseLogsComponent.java index 5a6e761618c3..748f62242337 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/UniverseLogsComponent.java +++ b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/UniverseLogsComponent.java @@ -11,19 +11,19 @@ import com.yugabyte.yw.common.config.RuntimeConfGetter; import com.yugabyte.yw.common.config.UniverseConfKeys; import com.yugabyte.yw.controllers.handlers.UniverseInfoHandler; +import com.yugabyte.yw.forms.SupportBundleFormData; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.UserIntent; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.helpers.NodeDetails; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -98,55 +98,13 @@ public void downloadComponentBetweenDates( startDate, endDate); - // Get the regex patterns used to filter file names - String universeLogsRegexPattern = - confGetter.getConfForScope(universe, UniverseConfKeys.universeLogsRegexPattern); - String postgresLogsRegexPattern = - confGetter.getConfForScope(universe, UniverseConfKeys.postgresLogsRegexPattern); - String connectionPoolingLogsRegexPattern = - confGetter.getConfForScope(universe, UniverseConfKeys.connectionPoolingLogsRegexPattern); - List fileRegexList = - Arrays.asList( - universeLogsRegexPattern, postgresLogsRegexPattern, connectionPoolingLogsRegexPattern); - - // Get and filter master log files that fall within given dates - String masterLogsPath = nodeHomeDir + "/master/logs"; - // Update logs path if overriden via Gflag. - String master_log_dir = getOverridenGflagValue(universe, ServerType.MASTER, LOG_DIR_GFLAG); - if (master_log_dir != null) { - masterLogsPath = master_log_dir; - } - List masterLogFilePaths = new ArrayList<>(); - if (nodeUniverseManager.checkNodeIfFileExists(node, universe, masterLogsPath)) { - masterLogFilePaths = - nodeUniverseManager.getNodeFilePaths( - node, universe, masterLogsPath, /*maxDepth*/ 1, /*fileType*/ "f"); - masterLogFilePaths = - supportBundleUtil.filterFilePathsBetweenDates( - masterLogFilePaths, fileRegexList, startDate, endDate); - } - - // Get and filter tserver log files that fall within given dates - String tserverLogsPath = nodeHomeDir + "/tserver/logs"; - // Update logs path if overriden via Gflag. - String ts_log_dir = getOverridenGflagValue(universe, ServerType.TSERVER, LOG_DIR_GFLAG); - if (ts_log_dir != null) { - tserverLogsPath = ts_log_dir; - } - List tserverLogFilePaths = new ArrayList<>(); - if (nodeUniverseManager.checkNodeIfFileExists(node, universe, tserverLogsPath)) { - tserverLogFilePaths = - nodeUniverseManager.getNodeFilePaths( - node, universe, tserverLogsPath, /*maxDepth*/ 1, /*fileType*/ "f"); - tserverLogFilePaths = - supportBundleUtil.filterFilePathsBetweenDates( - tserverLogFilePaths, fileRegexList, startDate, endDate); - } - // Combine both master and tserver files to download all the files together List allLogFilePaths = - Stream.concat(masterLogFilePaths.stream(), tserverLogFilePaths.stream()) - .map(filePath -> Paths.get(nodeHomeDir).relativize(filePath)) + getFilesListWithSizes( + customer, supportBundleTaskParams.bundleData, universe, startDate, endDate, node) + .keySet() + .stream() + .map(filePath -> Paths.get(nodeHomeDir).relativize(Paths.get(filePath))) .map(Path::toString) .collect(Collectors.toList()); @@ -194,4 +152,78 @@ private String getOverridenGflagValue(Universe universe, ServerType serverType, } return ret; } + + public Map getFilesListWithSizes( + Customer customer, + SupportBundleFormData bundleData, + Universe universe, + Date startDate, + Date endDate, + NodeDetails node) + throws Exception { + // Get source file path prefix + String mountPath = + supportBundleUtil.getDataDirPath(universe, node, nodeUniverseManager, config); + String nodeHomeDir = mountPath + "/yb-data"; + + // Get the regex patterns used to filter file names + String universeLogsRegexPattern = + confGetter.getConfForScope(universe, UniverseConfKeys.universeLogsRegexPattern); + String postgresLogsRegexPattern = + confGetter.getConfForScope(universe, UniverseConfKeys.postgresLogsRegexPattern); + String connectionPoolingLogsRegexPattern = + confGetter.getConfForScope(universe, UniverseConfKeys.connectionPoolingLogsRegexPattern); + List fileRegexList = + Arrays.asList( + universeLogsRegexPattern, postgresLogsRegexPattern, connectionPoolingLogsRegexPattern); + + // Get and filter master log files that fall within given dates + String masterLogsPath = nodeHomeDir + "/master/logs"; + // Update logs path if overriden via Gflag. + String master_log_dir = getOverridenGflagValue(universe, ServerType.MASTER, LOG_DIR_GFLAG); + if (master_log_dir != null) { + masterLogsPath = master_log_dir; + } + Map finalMap = new HashMap<>(); + if (nodeUniverseManager.checkNodeIfFileExists(node, universe, masterLogsPath)) { + Map masterLogsPathSizeMap = + nodeUniverseManager.getNodeFilePathAndSizes( + node, universe, masterLogsPath, /*maxDepth*/ 1, /*fileType*/ "f"); + List filteredMasterLogs = + supportBundleUtil.filterFilePathsBetweenDates( + masterLogsPathSizeMap.keySet().stream().collect(Collectors.toList()), + fileRegexList, + startDate, + endDate); + // Add filtered paths to the final map. + for (String path : filteredMasterLogs) { + finalMap.put(path, masterLogsPathSizeMap.get(path)); + } + } + + // Get and filter tserver log files that fall within given dates + String tserverLogsPath = nodeHomeDir + "/tserver/logs"; + // Update logs path if overriden via Gflag. + String ts_log_dir = getOverridenGflagValue(universe, ServerType.TSERVER, LOG_DIR_GFLAG); + if (ts_log_dir != null) { + tserverLogsPath = ts_log_dir; + } + + if (nodeUniverseManager.checkNodeIfFileExists(node, universe, tserverLogsPath)) { + Map tserverLogsPathSizeMap = + nodeUniverseManager.getNodeFilePathAndSizes( + node, universe, tserverLogsPath, /* maxDepth */ 1, /* fileType */ "f"); + List filteredTserverLogs = + supportBundleUtil.filterFilePathsBetweenDates( + tserverLogsPathSizeMap.keySet().stream().collect(Collectors.toList()), + fileRegexList, + startDate, + endDate); + // Add filtered paths to the final map. + for (String path : filteredTserverLogs) { + finalMap.put(path, tserverLogsPathSizeMap.get(path)); + } + } + return finalMap; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/YbaMetadataComponent.java b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/YbaMetadataComponent.java index d820c60258f0..e51ef8cb559f 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/YbaMetadataComponent.java +++ b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/YbaMetadataComponent.java @@ -8,16 +8,23 @@ import com.yugabyte.yw.commissioner.BaseTaskDependencies; import com.yugabyte.yw.commissioner.tasks.params.SupportBundleTaskParams; import com.yugabyte.yw.common.SupportBundleUtil; +import com.yugabyte.yw.common.config.GlobalConfKeys; +import com.yugabyte.yw.common.config.RuntimeConfGetter; +import com.yugabyte.yw.forms.SupportBundleFormData; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.helpers.NodeDetails; +import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.text.ParseException; import java.util.Date; +import java.util.HashMap; +import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; @Slf4j @Singleton @@ -26,6 +33,7 @@ public class YbaMetadataComponent implements SupportBundleComponent { protected final Config config; private final SupportBundleUtil supportBundleUtil; public final String YBA_METADATA_FOLDER = "metadata"; + @Inject RuntimeConfGetter confGetter; @Inject public YbaMetadataComponent( @@ -64,4 +72,35 @@ public void downloadComponentBetweenDates( // Gather and save the YBA metadata. supportBundleUtil.gatherAndSaveAllMetadata(customer, destDir, startDate, endDate); } + + // Collect the metadata in a temp dir and return the size of the directory. + // Its only a matter of few db queries so shouldn't take too long. + public Map getFilesListWithSizes( + Customer customer, + SupportBundleFormData bundleData, + Universe universe, + Date startDate, + Date endDate, + NodeDetails node) + throws Exception { + + // Create YBA_METADATA_FOLDER folder inside the support bundle folder. + String destDir = getLocalTmpDir() + "/" + YBA_METADATA_FOLDER; + File tmpMetadataDir = Files.createDirectories(Paths.get(destDir)).toFile(); + + // Gather and save the YBA metadata. + supportBundleUtil.gatherAndSaveAllMetadata(customer, destDir, startDate, endDate); + Map res = new HashMap<>(); + res.put(destDir, FileUtils.sizeOfDirectory(tmpMetadataDir)); + FileUtils.deleteDirectory(tmpMetadataDir); + return res; + } + + private String getLocalTmpDir() { + String localTmpDir = confGetter.getGlobalConf(GlobalConfKeys.ybTmpDirectoryPath); + if (localTmpDir == null || localTmpDir.isEmpty()) { + localTmpDir = "/tmp"; + } + return localTmpDir; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/YbcLogsComponent.java b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/YbcLogsComponent.java index 66029464d938..c0a53659825f 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/YbcLogsComponent.java +++ b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/YbcLogsComponent.java @@ -9,16 +9,18 @@ import com.yugabyte.yw.common.NodeUniverseManager; import com.yugabyte.yw.common.SupportBundleUtil; import com.yugabyte.yw.controllers.handlers.UniverseInfoHandler; +import com.yugabyte.yw.forms.SupportBundleFormData; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.helpers.NodeDetails; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -92,26 +94,18 @@ public void downloadComponentBetweenDates( startDate, endDate); - String ybcLogsRegexPattern = config.getString("yb.support_bundle.ybc_logs_regex_pattern"); + List filteredPaths = + getFilesListWithSizes(customer, null, universe, startDate, endDate, node).keySet().stream() + .collect(Collectors.toList()); - // Get and filter YB-Controller log files that fall within given dates - String ybcLogsPath = nodeHomeDir + "/controller/logs"; - List ybcLogFilePaths = new ArrayList<>(); - if (nodeUniverseManager.checkNodeIfFileExists(node, universe, ybcLogsPath)) { - // Gets the absolute paths of the ybc logs - ybcLogFilePaths = - nodeUniverseManager.getNodeFilePaths( - node, universe, ybcLogsPath, /*maxDepth*/ 1, /*fileType*/ "f"); - ybcLogFilePaths = - supportBundleUtil.filterFilePathsBetweenDates( - ybcLogFilePaths, Arrays.asList(ybcLogsRegexPattern), startDate, endDate); - } + if (filteredPaths.size() > 0) { - if (ybcLogFilePaths.size() > 0) { - List ybcLogFilePathString = - ybcLogFilePaths.stream() - .map(filePath -> Paths.get(nodeHomeDir).relativize(filePath)) + filteredPaths = + filteredPaths.stream() + .peek(s -> log.error("file path before relativing = {}", s)) + .map(filePath -> Paths.get(nodeHomeDir).relativize(Paths.get(filePath))) .map(Path::toString) + .peek(s -> log.error("file path after relativing = {}", s)) .collect(Collectors.toList()); // Download all logs batch wise @@ -123,7 +117,7 @@ public void downloadComponentBetweenDates( node, nodeTargetFile, nodeHomeDir, - ybcLogFilePathString, + filteredPaths, this.getClass().getSimpleName(), false); @@ -159,4 +153,39 @@ public void downloadComponentBetweenDates( endDate); } } + + public Map getFilesListWithSizes( + Customer customer, + SupportBundleFormData bundleData, + Universe universe, + Date startDate, + Date endDate, + NodeDetails node) + throws Exception { + Map res = new HashMap<>(); + + String mountPath = + supportBundleUtil.getDataDirPath(universe, node, nodeUniverseManager, config); + String nodeHomeDir = mountPath + "/ybc-data"; + String ybcLogsRegexPattern = config.getString("yb.support_bundle.ybc_logs_regex_pattern"); + + // Get and filter YB-Controller log files that fall within given dates + String ybcLogsPath = nodeHomeDir + "/controller/logs"; + if (nodeUniverseManager.checkNodeIfFileExists(node, universe, ybcLogsPath)) { + // Gets the absolute paths and sizes of the ybc logs + Map ybcLogPathSizeMap = + nodeUniverseManager.getNodeFilePathAndSizes( + node, universe, ybcLogsPath, /* maxDepth */ 1, /* fileType */ "f"); + List filteredPaths = + supportBundleUtil.filterFilePathsBetweenDates( + ybcLogPathSizeMap.keySet().stream().collect(Collectors.toList()), + Arrays.asList(ybcLogsRegexPattern), + startDate, + endDate); + for (String path : filteredPaths) { + res.put(path, ybcLogPathSizeMap.get(path)); + } + } + return res; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/controllers/SupportBundleController.java b/managed/src/main/java/com/yugabyte/yw/controllers/SupportBundleController.java index 29e1e4de1c84..b81f079d1014 100644 --- a/managed/src/main/java/com/yugabyte/yw/controllers/SupportBundleController.java +++ b/managed/src/main/java/com/yugabyte/yw/controllers/SupportBundleController.java @@ -6,20 +6,18 @@ import com.google.inject.Inject; import com.typesafe.config.Config; import com.yugabyte.yw.commissioner.Commissioner; -import com.yugabyte.yw.commissioner.Common.CloudType; import com.yugabyte.yw.commissioner.tasks.params.SupportBundleTaskParams; import com.yugabyte.yw.common.PlatformServiceException; import com.yugabyte.yw.common.SupportBundleUtil; import com.yugabyte.yw.common.Util; -import com.yugabyte.yw.common.config.GlobalConfKeys; -import com.yugabyte.yw.common.config.RuntimeConfGetter; -import com.yugabyte.yw.common.config.RuntimeConfigFactory; import com.yugabyte.yw.common.rbac.PermissionInfo.Action; import com.yugabyte.yw.common.rbac.PermissionInfo.ResourceType; +import com.yugabyte.yw.controllers.handlers.SupportBundleHandler; import com.yugabyte.yw.forms.PlatformResults; import com.yugabyte.yw.forms.PlatformResults.YBPSuccess; import com.yugabyte.yw.forms.PlatformResults.YBPTask; import com.yugabyte.yw.forms.SupportBundleFormData; +import com.yugabyte.yw.forms.SupportBundleSizeEstimateResponse; import com.yugabyte.yw.models.Audit; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.CustomerTask; @@ -61,8 +59,7 @@ public class SupportBundleController extends AuthenticatedController { @Inject Commissioner commissioner; @Inject SupportBundleUtil supportBundleUtil; - @Inject private RuntimeConfigFactory runtimeConfigFactory; - @Inject private RuntimeConfGetter confGetter; + @Inject SupportBundleHandler sbHandler; @Inject Config config; @ApiOperation( @@ -99,68 +96,10 @@ public Result create(UUID customerUUID, UUID universeUUID, Http.Request request) log.info( "Trying to create support bundle while universe {} is " + "in a locked/paused state or has backup running.", - universe.getUniverseUUID()); + universe.getName()); } - // Support bundle for onprem and k8s universes was originally behind a runtime flag. - // Now both are enabled by default. - CloudType cloudType = universe.getUniverseDetails().getPrimaryCluster().userIntent.providerType; - Boolean k8sEnabled = confGetter.getGlobalConf(GlobalConfKeys.supportBundleK8sEnabled); - Boolean onpremEnabled = confGetter.getGlobalConf(GlobalConfKeys.supportBundleOnPremEnabled); - Boolean allowCoresCollection = - confGetter.getGlobalConf(GlobalConfKeys.supportBundleAllowCoresCollection); - if (CloudType.onprem.equals(cloudType) && !onpremEnabled) { - throw new PlatformServiceException( - BAD_REQUEST, - "Creating support bundle for on-prem universes is not enabled. " - + "Please set onprem_enabled=true to create support bundle"); - } - if (CloudType.kubernetes.equals(cloudType) && !k8sEnabled) { - throw new PlatformServiceException( - BAD_REQUEST, - "Creating support bundle for k8s universes is not enabled. " - + "Please set k8s_enabled=true to create support bundle"); - } - - if (cloudType != CloudType.kubernetes - && bundleData.components.contains(ComponentType.K8sInfo)) { - bundleData.components.remove(ComponentType.K8sInfo); - log.warn( - "Component 'K8sInfo' is only applicable for kubernetes universes, not cloud type = " - + cloudType.toString() - + ". Continuing without it."); - } - - if (bundleData.components.contains(ComponentType.CoreFiles) && !allowCoresCollection) { - throw new PlatformServiceException( - BAD_REQUEST, - "Core file collection is disabled globally. Either remove core files component from" - + " bundle creation, or enable runtime config" - + " 'yb.support_bundle.allow_cores_collection'."); - } - - if (bundleData.components.contains(ComponentType.PrometheusMetrics) - && ((bundleData.promDumpStartDate == null) ^ (bundleData.promDumpEndDate == null))) { - throw new PlatformServiceException( - BAD_REQUEST, - "Either define both 'promDumpStartDate' and 'promDumpEndDate', or neither (Will default" - + " to 'yb.support_bundle.default_prom_dump_range' in this case)"); - } - - if (bundleData.startDate != null - && bundleData.endDate != null - && !supportBundleUtil.checkDatesValid(bundleData.startDate, bundleData.endDate)) { - throw new PlatformServiceException(BAD_REQUEST, "'startDate' should be before the 'endDate'"); - } - - if (bundleData.components.contains(ComponentType.PrometheusMetrics) - && bundleData.promDumpStartDate != null - && bundleData.promDumpEndDate != null - && !supportBundleUtil.checkDatesValid( - bundleData.promDumpStartDate, bundleData.promDumpEndDate)) { - throw new PlatformServiceException( - BAD_REQUEST, "'promDumpStartDate' should be before the 'promDumpEndDate'"); - } + sbHandler.bundleDataVaidation(bundleData, universe); SupportBundle supportBundle = SupportBundle.create(bundleData, universe); SupportBundleTaskParams taskParams = @@ -320,4 +259,35 @@ public Result getComponents(UUID customerUUID) { EnumSet components = EnumSet.allOf(ComponentType.class); return PlatformResults.withData(components); } + + @ApiOperation( + value = "Estimate support bundle size for specific universe", + nickname = "estimateSupportBundleSize", + response = SupportBundleSizeEstimateResponse.class) + @ApiImplicitParams( + @ApiImplicitParam( + name = "supportBundle", + value = "support bundle info", + paramType = "body", + dataType = "com.yugabyte.yw.forms.SupportBundleFormData", + required = true)) + @AuthzPath({ + @RequiredPermissionOnResource( + requiredPermission = + @PermissionAttribute(resourceType = ResourceType.UNIVERSE, action = Action.READ), + resourceLocation = @Resource(path = Util.UNIVERSES, sourceType = SourceType.ENDPOINT)), + }) + public Result estimateSize(UUID customerUUID, UUID universeUUID, Http.Request request) + throws Exception { + JsonNode requestBody = request.body().asJson(); + SupportBundleFormData bundleData = + formFactory.getFormDataOrBadRequest(requestBody, SupportBundleFormData.class); + + Customer customer = Customer.getOrBadRequest(customerUUID); + Universe universe = Universe.getOrBadRequest(universeUUID, customer); + + sbHandler.bundleDataVaidation(bundleData, universe); + + return PlatformResults.withData(sbHandler.estimateBundleSize(customer, bundleData, universe)); + } } diff --git a/managed/src/main/java/com/yugabyte/yw/controllers/handlers/SupportBundleHandler.java b/managed/src/main/java/com/yugabyte/yw/controllers/handlers/SupportBundleHandler.java new file mode 100644 index 000000000000..3da41648015b --- /dev/null +++ b/managed/src/main/java/com/yugabyte/yw/controllers/handlers/SupportBundleHandler.java @@ -0,0 +1,225 @@ +package com.yugabyte.yw.controllers.handlers; + +import static play.mvc.Http.Status.BAD_REQUEST; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.typesafe.config.Config; +import com.yugabyte.yw.commissioner.Common.CloudType; +import com.yugabyte.yw.common.NodeUniverseManager; +import com.yugabyte.yw.common.PlatformExecutorFactory; +import com.yugabyte.yw.common.PlatformServiceException; +import com.yugabyte.yw.common.SupportBundleUtil; +import com.yugabyte.yw.common.config.GlobalConfKeys; +import com.yugabyte.yw.common.config.RuntimeConfGetter; +import com.yugabyte.yw.common.supportbundle.SupportBundleComponent; +import com.yugabyte.yw.common.supportbundle.SupportBundleComponentFactory; +import com.yugabyte.yw.common.utils.Pair; +import com.yugabyte.yw.forms.SupportBundleFormData; +import com.yugabyte.yw.forms.SupportBundleSizeEstimateResponse; +import com.yugabyte.yw.models.Customer; +import com.yugabyte.yw.models.Universe; +import com.yugabyte.yw.models.helpers.BundleDetails; +import com.yugabyte.yw.models.helpers.BundleDetails.ComponentType; +import com.yugabyte.yw.models.helpers.NodeDetails; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; + +@Singleton +@Slf4j +public class SupportBundleHandler { + + @Inject SupportBundleUtil supportBundleUtil; + @Inject RuntimeConfGetter confGetter; + @Inject SupportBundleComponentFactory componentFactory; + @Inject Config staticConf; + @Inject PlatformExecutorFactory executorFactory; + @Inject NodeUniverseManager nodeUniverseManager; + + /** + * Validate support bundle form and throw BAD_REQUEST if validation fails. + * + * @param bundleData + * @param universe + */ + public void bundleDataVaidation(SupportBundleFormData bundleData, Universe universe) { + // Support bundle for onprem and k8s universes was originally behind a runtime flag. + // Now both are enabled by default. + CloudType cloudType = universe.getUniverseDetails().getPrimaryCluster().userIntent.providerType; + Boolean k8sEnabled = confGetter.getGlobalConf(GlobalConfKeys.supportBundleK8sEnabled); + Boolean onpremEnabled = confGetter.getGlobalConf(GlobalConfKeys.supportBundleOnPremEnabled); + Boolean allowCoresCollection = + confGetter.getGlobalConf(GlobalConfKeys.supportBundleAllowCoresCollection); + if (CloudType.onprem.equals(cloudType) && !onpremEnabled) { + throw new PlatformServiceException( + BAD_REQUEST, + "Creating support bundle for on-prem universes is not enabled. " + + "Please set onprem_enabled=true to create support bundle"); + } + if (CloudType.kubernetes.equals(cloudType) && !k8sEnabled) { + throw new PlatformServiceException( + BAD_REQUEST, + "Creating support bundle for k8s universes is not enabled. " + + "Please set k8s_enabled=true to create support bundle"); + } + + if (cloudType != CloudType.kubernetes + && bundleData.components.contains(ComponentType.K8sInfo)) { + bundleData.components.remove(ComponentType.K8sInfo); + log.warn( + "Component 'K8sInfo' is only applicable for kubernetes universes, not cloud type = " + + cloudType.toString() + + ". Continuing without it."); + } + + if (bundleData.components.contains(ComponentType.CoreFiles) && !allowCoresCollection) { + throw new PlatformServiceException( + BAD_REQUEST, + "Core file collection is disabled globally. Either remove core files component from" + + " bundle creation, or enable runtime config" + + " 'yb.support_bundle.allow_cores_collection'."); + } + + if (bundleData.components.contains(ComponentType.PrometheusMetrics) + && ((bundleData.promDumpStartDate == null) ^ (bundleData.promDumpEndDate == null))) { + throw new PlatformServiceException( + BAD_REQUEST, + "Either define both 'promDumpStartDate' and 'promDumpEndDate', or neither (Will default" + + " to 'yb.support_bundle.default_prom_dump_range' in this case)"); + } + + if (bundleData.startDate != null + && bundleData.endDate != null + && !supportBundleUtil.checkDatesValid(bundleData.startDate, bundleData.endDate)) { + throw new PlatformServiceException(BAD_REQUEST, "'startDate' should be before the 'endDate'"); + } + + if (bundleData.components.contains(ComponentType.PrometheusMetrics) + && bundleData.promDumpStartDate != null + && bundleData.promDumpEndDate != null + && !supportBundleUtil.checkDatesValid( + bundleData.promDumpStartDate, bundleData.promDumpEndDate)) { + throw new PlatformServiceException( + BAD_REQUEST, "'promDumpStartDate' should be before the 'promDumpEndDate'"); + } + } + + /** + * Calculate the size of a support bundle. For each given component we collect file sizes in + * parallel. Only few node level components actually require connecting to the db nodes. When + * required, only a single "find | ls" command is run on the node per component so this should be + * safe and shouldn't affect cpu/memory consumption on the node. + * + * @return a map containing sizes for all components and the totalSize + */ + public SupportBundleSizeEstimateResponse estimateBundleSize( + Customer customer, SupportBundleFormData bundleData, Universe universe) throws Exception { + + // Map to track sizes for all components. + Map> resp = new HashMap<>(); + Pair datePair = + supportBundleUtil.getValidStartAndEndDates( + staticConf, bundleData.startDate, bundleData.endDate); + Date startDate = datePair.getFirst(), endDate = datePair.getSecond(); + + // Threadpool with same configs as default task threadpool. + ThreadPoolExecutor threadpool = + executorFactory.createExecutor("task", Executors.defaultThreadFactory()); + try { + Set reachableNodes = getReachableNodes(threadpool, universe); + + // Submit tasks to the threadpool to collect file size for all components in parallel. + List, Future>>> futures = + new ArrayList<>(); + for (ComponentType componentType : bundleData.components) { + if (componentType.getComponentLevel().equals(BundleDetails.ComponentLevel.NodeLevel)) { + // For node level components, collect sizes from all nodes. + for (NodeDetails node : reachableNodes) { + SupportBundleComponent component = componentFactory.getComponent(componentType); + Callable> callable = + () -> { + return component.getFilesListWithSizes( + customer, bundleData, universe, startDate, endDate, node); + }; + Pair, Future>> taskPair = + new Pair<>( + new Pair<>(node.getNodeName(), componentType), threadpool.submit(callable)); + futures.add(taskPair); + } + } else { + SupportBundleComponent component = componentFactory.getComponent(componentType); + Callable> callable = + () -> { + return component.getFilesListWithSizes( + customer, bundleData, universe, startDate, endDate, null); + }; + Pair, Future>> taskPair = + new Pair<>(new Pair<>("YBA", componentType), threadpool.submit(callable)); + futures.add(taskPair); + } + } + + for (Pair, Future>> p : futures) { + String nodeName = p.getFirst().getFirst(); + ComponentType component = p.getFirst().getSecond(); + try { + Map componentSizeMap = p.getSecond().get(); + Long componentSize = componentSizeMap.values().stream().mapToLong(Long::longValue).sum(); + Map nodeComponentSizeMap = resp.getOrDefault(nodeName, new HashMap<>()); + nodeComponentSizeMap.put(component.toString(), componentSize); + resp.put(nodeName, nodeComponentSizeMap); + } catch (InterruptedException | ExecutionException e) { + log.error("Error while getting file sizes for component: ", component.toString()); + e.printStackTrace(); + } + } + } finally { + threadpool.shutdown(); + } + return new SupportBundleSizeEstimateResponse(resp); + } + + /** Check for reachable nodes in a universe in parallel. */ + private Set getReachableNodes(ThreadPoolExecutor threadpool, Universe universe) { + Set reachableNodes = new HashSet<>(); + List>> futures = new ArrayList<>(); + for (NodeDetails node : universe.getNodes()) { + Callable callable = + () -> { + return nodeUniverseManager.isNodeReachable( + node, + universe, + confGetter.getGlobalConf(GlobalConfKeys.supportBundleNodeCheckTimeoutSec)); + }; + futures.add(new Pair>(node, threadpool.submit(callable))); + } + // Wait for tasks to complete. + for (Pair> p : futures) { + try { + if (p.getSecond().get()) { + reachableNodes.add(p.getFirst()); + } + } catch (InterruptedException | ExecutionException e) { + log.error("Error while collecting reachable nodes for universe: {}", universe.getName()); + e.printStackTrace(); + } + } + log.info( + "Reachable nodes for universe {} = {}", + universe.getName(), + reachableNodes.stream().map(NodeDetails::getNodeName).collect(Collectors.joining(", "))); + return reachableNodes; + } +} diff --git a/managed/src/main/java/com/yugabyte/yw/forms/SupportBundleSizeEstimateResponse.java b/managed/src/main/java/com/yugabyte/yw/forms/SupportBundleSizeEstimateResponse.java new file mode 100644 index 000000000000..f877da676912 --- /dev/null +++ b/managed/src/main/java/com/yugabyte/yw/forms/SupportBundleSizeEstimateResponse.java @@ -0,0 +1,17 @@ +package com.yugabyte.yw.forms; + +import io.swagger.annotations.ApiModelProperty; +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class SupportBundleSizeEstimateResponse { + + @ApiModelProperty( + value = + "A map of universe node names to component sizes for the given support bundle payload." + + " Global component sizes are mapped to \"YBA\" node.") + private Map> data; +} diff --git a/managed/src/main/resources/swagger-strict.json b/managed/src/main/resources/swagger-strict.json index 092ad9558ae9..c92fa0ecff6c 100644 --- a/managed/src/main/resources/swagger-strict.json +++ b/managed/src/main/resources/swagger-strict.json @@ -12904,6 +12904,22 @@ "required" : [ "components", "endDate", "startDate" ], "type" : "object" }, + "SupportBundleSizeEstimateResponse" : { + "properties" : { + "data" : { + "additionalProperties" : { + "additionalProperties" : { + "format" : "int64", + "type" : "integer" + }, + "type" : "object" + }, + "description" : "A map of universe node names to component sizes for the given support bundle payload. Global component sizes are mapped to \"YBA\" node.", + "type" : "object" + } + }, + "type" : "object" + }, "SuppressHealthCheckNotificationsConfig" : { "properties" : { "suppressAllUniverses" : { @@ -27023,6 +27039,58 @@ "tags" : [ "Support Bundle management" ] } }, + "/api/v1/customers/{cUUID}/universes/{uniUUID}/support_bundle/estimate_size" : { + "get" : { + "description" : "", + "operationId" : "estimateSupportBundleSize", + "parameters" : [ { + "format" : "uuid", + "in" : "path", + "name" : "cUUID", + "required" : true, + "type" : "string" + }, { + "format" : "uuid", + "in" : "path", + "name" : "uniUUID", + "required" : true, + "type" : "string" + }, { + "in" : "query", + "name" : "request", + "required" : false + }, { + "description" : "support bundle info", + "in" : "body", + "name" : "supportBundle", + "required" : true, + "schema" : { + "$ref" : "#/definitions/SupportBundleFormData" + } + } ], + "responses" : { + "200" : { + "description" : "successful operation", + "schema" : { + "$ref" : "#/definitions/SupportBundleSizeEstimateResponse" + } + } + }, + "responsesObject" : { + "200" : { + "description" : "successful operation", + "schema" : { + "$ref" : "#/definitions/SupportBundleSizeEstimateResponse" + } + } + }, + "security" : [ { + "apiKeyAuth" : [ ] + } ], + "summary" : "Estimate support bundle size for specific universe", + "tags" : [ "Support Bundle management" ] + } + }, "/api/v1/customers/{cUUID}/universes/{uniUUID}/support_bundle/{sbUUID}" : { "delete" : { "description" : "", diff --git a/managed/src/main/resources/swagger.json b/managed/src/main/resources/swagger.json index f46d1d4f6be7..bb096b98b12e 100644 --- a/managed/src/main/resources/swagger.json +++ b/managed/src/main/resources/swagger.json @@ -13021,6 +13021,22 @@ "required" : [ "components", "endDate", "startDate" ], "type" : "object" }, + "SupportBundleSizeEstimateResponse" : { + "properties" : { + "data" : { + "additionalProperties" : { + "additionalProperties" : { + "format" : "int64", + "type" : "integer" + }, + "type" : "object" + }, + "description" : "A map of universe node names to component sizes for the given support bundle payload. Global component sizes are mapped to \"YBA\" node.", + "type" : "object" + } + }, + "type" : "object" + }, "SuppressHealthCheckNotificationsConfig" : { "properties" : { "suppressAllUniverses" : { @@ -28677,6 +28693,58 @@ "tags" : [ "Support Bundle management" ] } }, + "/api/v1/customers/{cUUID}/universes/{uniUUID}/support_bundle/estimate_size" : { + "get" : { + "description" : "", + "operationId" : "estimateSupportBundleSize", + "parameters" : [ { + "format" : "uuid", + "in" : "path", + "name" : "cUUID", + "required" : true, + "type" : "string" + }, { + "format" : "uuid", + "in" : "path", + "name" : "uniUUID", + "required" : true, + "type" : "string" + }, { + "in" : "query", + "name" : "request", + "required" : false + }, { + "description" : "support bundle info", + "in" : "body", + "name" : "supportBundle", + "required" : true, + "schema" : { + "$ref" : "#/definitions/SupportBundleFormData" + } + } ], + "responses" : { + "200" : { + "description" : "successful operation", + "schema" : { + "$ref" : "#/definitions/SupportBundleSizeEstimateResponse" + } + } + }, + "responsesObject" : { + "200" : { + "description" : "successful operation", + "schema" : { + "$ref" : "#/definitions/SupportBundleSizeEstimateResponse" + } + } + }, + "security" : [ { + "apiKeyAuth" : [ ] + } ], + "summary" : "Estimate support bundle size for specific universe", + "tags" : [ "Support Bundle management" ] + } + }, "/api/v1/customers/{cUUID}/universes/{uniUUID}/support_bundle/{sbUUID}" : { "delete" : { "description" : "", diff --git a/managed/src/main/resources/v1.routes b/managed/src/main/resources/v1.routes index 4a68cf61a1c6..09efd4ac2f41 100644 --- a/managed/src/main/resources/v1.routes +++ b/managed/src/main/resources/v1.routes @@ -554,6 +554,7 @@ PUT /customers/:cUUID/universes/:uniUUID/update_scheduled_script c GET /customers/:cUUID/support_bundle/components com.yugabyte.yw.controllers.SupportBundleController.getComponents(cUUID: java.util.UUID) POST /customers/:cUUID/universes/:uniUUID/support_bundle com.yugabyte.yw.controllers.SupportBundleController.create(cUUID: java.util.UUID, uniUUID: java.util.UUID, request: Request) GET /customers/:cUUID/universes/:uniUUID/support_bundle com.yugabyte.yw.controllers.SupportBundleController.list(cUUID: java.util.UUID, uniUUID: java.util.UUID) +GET /customers/:cUUID/universes/:uniUUID/support_bundle/estimate_size com.yugabyte.yw.controllers.SupportBundleController.estimateSize(cUUID: java.util.UUID, uniUUID: java.util.UUID, request: Request) GET /customers/:cUUID/universes/:uniUUID/support_bundle/:sbUUID com.yugabyte.yw.controllers.SupportBundleController.get(cUUID: java.util.UUID, uniUUID: java.util.UUID, sbUUID: java.util.UUID) GET /customers/:cUUID/universes/:uniUUID/support_bundle/:sbUUID/download com.yugabyte.yw.controllers.SupportBundleController.download(cUUID: java.util.UUID, uniUUID: java.util.UUID, sbUUID: java.util.UUID) DELETE /customers/:cUUID/universes/:uniUUID/support_bundle/:sbUUID com.yugabyte.yw.controllers.SupportBundleController.delete(cUUID: java.util.UUID, uniUUID: java.util.UUID, sbUUID: java.util.UUID, request: Request) diff --git a/managed/src/test/java/com/yugabyte/yw/common/SupportBundleUtilTest.java b/managed/src/test/java/com/yugabyte/yw/common/SupportBundleUtilTest.java index 5f62a30305c6..c0f410c9d2e4 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/SupportBundleUtilTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/SupportBundleUtilTest.java @@ -19,7 +19,6 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.stream.Collectors; import junitparams.JUnitParamsRunner; import junitparams.Parameters; import org.junit.Before; @@ -108,11 +107,9 @@ public void testFilterList() throws ParseException { "application-log-2022-02-03.gz", "application-log-2022-02-04.gz", "application-log-2022-02-05.gz"); - List outputList = - supportBundleUtil.filterList( - unfilteredList.stream().map(Paths::get).collect(Collectors.toList()), - Arrays.asList(testRegexPattern)); - assertEquals(outputList, filteredList.stream().map(Paths::get).collect(Collectors.toList())); + List outputList = + supportBundleUtil.filterList(unfilteredList, Arrays.asList(testRegexPattern)); + assertEquals(outputList, filteredList); } @Test @@ -290,13 +287,9 @@ public void testFilterFilePathsBetweenDates() throws ParseException { "/mnt/disk0/yb-data/tserver/logs/yb-tserver.yb-dev-sahith-new-yb-tserver-1.root." + "log.WARNING.20221122-000000.00"); - List unFilteredLogFilePathList = - unfilteredLogFilePaths.stream().map(Paths::get).collect(Collectors.toList()); - List expectedLogFilePathList = - expectedLogFilePaths.stream().map(Paths::get).collect(Collectors.toList()); - List filteredLogFilePaths = + List filteredLogFilePaths = supportBundleUtil.filterFilePathsBetweenDates( - unFilteredLogFilePathList, + unfilteredLogFilePaths, Arrays.asList( universe_logs_regex_pattern, postgres_logs_regex_pattern, @@ -305,8 +298,8 @@ public void testFilterFilePathsBetweenDates() throws ParseException { endDate); assertTrue( - expectedLogFilePathList.containsAll(filteredLogFilePaths) - && filteredLogFilePaths.containsAll(expectedLogFilePathList)); + expectedLogFilePaths.containsAll(filteredLogFilePaths) + && filteredLogFilePaths.containsAll(expectedLogFilePaths)); } @Test diff --git a/managed/src/test/java/com/yugabyte/yw/common/supportbundle/ApplicationLogsComponentTest.java b/managed/src/test/java/com/yugabyte/yw/common/supportbundle/ApplicationLogsComponentTest.java index 070a01fe9c83..cf6dc1a623a7 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/supportbundle/ApplicationLogsComponentTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/supportbundle/ApplicationLogsComponentTest.java @@ -17,7 +17,6 @@ import java.io.File; import java.io.IOException; import java.nio.file.Paths; -import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Date; @@ -81,7 +80,7 @@ public void tearDown() throws IOException { } @Test - public void testDownloadComponentBetweenDatesTillCurrentDay() throws IOException, ParseException { + public void testDownloadComponentBetweenDatesTillCurrentDay() throws Exception { // Define start and end dates to filter Date startDate = dateFormat.parse("2022-03-06"); Date endDate = mockSupportBundleUtil.getTodaysDate(); @@ -109,7 +108,7 @@ public void testDownloadComponentBetweenDatesTillCurrentDay() throws IOException } @Test - public void testDownloadComponentBetweenDatesWithOlderDates() throws IOException, ParseException { + public void testDownloadComponentBetweenDatesWithOlderDates() throws Exception { // Define start and end dates to filter Date startDate = dateFormat.parse("2022-03-06"); Date endDate = dateFormat.parse("2022-03-07"); @@ -133,7 +132,7 @@ public void testDownloadComponentBetweenDatesWithOlderDates() throws IOException } @Test - public void testDownloadComponentBetweenDatesPartialBounds() throws IOException, ParseException { + public void testDownloadComponentBetweenDatesPartialBounds() throws Exception { // Define start and end dates to filter Date startDate = dateFormat.parse("2022-03-01"); Date endDate = dateFormat.parse("2022-03-05"); @@ -156,7 +155,7 @@ public void testDownloadComponentBetweenDatesPartialBounds() throws IOException, } @Test - public void testDownloadComponentBetweenDatesOutOfBounds() throws IOException, ParseException { + public void testDownloadComponentBetweenDatesOutOfBounds() throws Exception { // Define start and end dates to filter Date startDate = dateFormat.parse("2022-03-01"); Date endDate = dateFormat.parse("2022-03-03"); diff --git a/managed/src/test/java/com/yugabyte/yw/common/supportbundle/CoreFilesComponentTest.java b/managed/src/test/java/com/yugabyte/yw/common/supportbundle/CoreFilesComponentTest.java index a949715026be..3b509bb73160 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/supportbundle/CoreFilesComponentTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/supportbundle/CoreFilesComponentTest.java @@ -21,7 +21,6 @@ import com.yugabyte.yw.common.ModelFactory; import com.yugabyte.yw.common.NodeUniverseManager; import com.yugabyte.yw.common.SupportBundleUtil; -import com.yugabyte.yw.common.utils.Pair; import com.yugabyte.yw.controllers.handlers.UniverseInfoHandler; import com.yugabyte.yw.forms.SupportBundleFormData; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams; @@ -37,7 +36,9 @@ import java.util.Arrays; import java.util.Date; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import org.apache.commons.io.FileUtils; import org.junit.After; import org.junit.Before; @@ -99,15 +100,14 @@ public void setUp() throws Exception { doCallRealMethod() .when(mockSupportBundleUtil) .batchWiseDownload(any(), any(), any(), any(), any(), any(), any(), any(), any(), eq(true)); - List> fileSizeNameList = - Arrays.asList( - new Pair<>(101L, "core_test.2"), - new Pair<>(15L, "core_test.3"), - new Pair<>(100L, "core_test.1"), - new Pair<>(10L, "core_test.4")); - doReturn(fileSizeNameList) + Map fileNameSizeMap = new LinkedHashMap<>(); + fileNameSizeMap.put("core_test.2", 101L); + fileNameSizeMap.put("core_test.3", 15L); + fileNameSizeMap.put("core_test.1", 100L); + fileNameSizeMap.put("core_test.4", 10L); + doReturn(fileNameSizeMap) .when(mockNodeUniverseManager) - .getNodeFilePathsAndSize(any(), any(), any(), any(), any()); + .getNodeFilePathsAndSizeWithinDates(any(), any(), any(), any(), any()); when(mockUniverseInfoHandler.downloadNodeFile(any(), any(), any(), any(), any(), any())) .thenAnswer( @@ -195,7 +195,8 @@ public void testDownloadComponentWithCoreFiltering() throws Exception { captorSourceNodeFiles.capture(), any(), eq(true)); - List expectedSourceNodeFiles = Arrays.asList("core_test.3", "core_test.1"); + List expectedSourceNodeFiles = + Arrays.asList("core_test.3", "core_test.1", "core_test.4"); System.out.println("CAPTOR: " + captorSourceNodeFiles.getValue().toString()); assertEquals(expectedSourceNodeFiles, captorSourceNodeFiles.getValue()); diff --git a/managed/src/test/java/com/yugabyte/yw/common/supportbundle/NodeAgentComponentTest.java b/managed/src/test/java/com/yugabyte/yw/common/supportbundle/NodeAgentComponentTest.java index f2bdab498a7c..6c29dee94580 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/supportbundle/NodeAgentComponentTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/supportbundle/NodeAgentComponentTest.java @@ -35,9 +35,10 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.stream.Collectors; +import java.util.Map; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -107,9 +108,6 @@ public void setUp() throws Exception { nodeAgentLogDir + "/node-agent.log", nodeAgentLogDir + "/node_agent-2023-08-07T22-50-15.473.log.gz"); - List fakeLogFilePathList = - fakeLogsList.stream().map(Paths::get).collect(Collectors.toList()); - // Mock all the invocations with fake data when(mockSupportBundleUtil.unGzip(any(), any())).thenCallRealMethod(); when(mockSupportBundleUtil.unTar(any(), any())).thenCallRealMethod(); @@ -118,8 +116,13 @@ public void setUp() throws Exception { .batchWiseDownload( any(), any(), any(), any(), any(), any(), any(), any(), any(), eq(false)); - when(mockNodeUniverseManager.getNodeFilePaths(any(), any(), any(), eq(1), eq("f"))) - .thenReturn(fakeLogFilePathList); + Map fakeLogPathSizeMap = new HashMap<>(); + for (String path : fakeLogsList) { + fakeLogPathSizeMap.put(path, 10L); + } + + when(mockNodeUniverseManager.getNodeFilePathAndSizes(any(), any(), any(), eq(1), eq("f"))) + .thenReturn(fakeLogPathSizeMap); // Generate a fake shell response containing the output of the "check file exists" script // Mocks the server response as "file existing" when(mockNodeUniverseManager.checkNodeIfFileExists(any(), any(), any())).thenReturn(true); diff --git a/managed/src/test/java/com/yugabyte/yw/common/supportbundle/UniverseLogsComponentTest.java b/managed/src/test/java/com/yugabyte/yw/common/supportbundle/UniverseLogsComponentTest.java index d2444817e9b8..a871c41c5e09 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/supportbundle/UniverseLogsComponentTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/supportbundle/UniverseLogsComponentTest.java @@ -32,9 +32,10 @@ import java.nio.file.Paths; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.stream.Collectors; +import java.util.Map; import org.apache.commons.io.FileUtils; import org.junit.After; import org.junit.Before; @@ -123,10 +124,12 @@ public void setUp() throws Exception { // Generate a fake shell response containing the entire list of file paths // Mocks the server response - List fakeLogFilePathList = - fakeLogsList.stream().map(Paths::get).collect(Collectors.toList()); - when(mockNodeUniverseManager.getNodeFilePaths(any(), any(), any(), eq(1), eq("f"))) - .thenReturn(fakeLogFilePathList); + Map fakeLogPathSizeMap = new HashMap<>(); + for (String path : fakeLogsList) { + fakeLogPathSizeMap.put(path, 10L); + } + when(mockNodeUniverseManager.getNodeFilePathAndSizes(any(), any(), any(), eq(1), eq("f"))) + .thenReturn(fakeLogPathSizeMap); // Generate a fake shell response containing the output of the "check file exists" script // Mocks the server response as "file existing" when(mockNodeUniverseManager.checkNodeIfFileExists(any(), any(), any())).thenReturn(true); diff --git a/managed/src/test/java/com/yugabyte/yw/common/supportbundle/YbcLogsComponentTest.java b/managed/src/test/java/com/yugabyte/yw/common/supportbundle/YbcLogsComponentTest.java index dff6fcf621da..21e82ed85bc1 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/supportbundle/YbcLogsComponentTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/supportbundle/YbcLogsComponentTest.java @@ -30,9 +30,10 @@ import java.nio.file.Paths; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.stream.Collectors; +import java.util.Map; import org.apache.commons.io.FileUtils; import org.junit.After; import org.junit.Before; @@ -113,10 +114,13 @@ public void setUp() throws Exception { when(mockNodeUniverseManager.checkNodeIfFileExists(any(), any(), any())).thenReturn(true); // Generate a fake shell response containing the entire list of file paths // Mocks the server response - List fakeLogFilePathList = - fakeLogsList.stream().map(Paths::get).collect(Collectors.toList()); - when(mockNodeUniverseManager.getNodeFilePaths(any(), any(), any(), eq(1), eq("f"))) - .thenReturn(fakeLogFilePathList); + Map fakeLogPathSizeMap = new HashMap<>(); + for (String path : fakeLogsList) { + fakeLogPathSizeMap.put(path, 10L); + } + + when(mockNodeUniverseManager.getNodeFilePathAndSizes(any(), any(), any(), eq(1), eq("f"))) + .thenReturn(fakeLogPathSizeMap); when(mockUniverseInfoHandler.downloadNodeFile(any(), any(), any(), any(), any(), any())) .thenAnswer(