From 70c27179b43faaa619a607857e8cdc16b06105de Mon Sep 17 00:00:00 2001 From: hamad45 Date: Tue, 24 Jan 2023 17:22:25 -0800 Subject: [PATCH 1/3] Added support for DXT issues and recommendations --- drishti/main.py | 71 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/drishti/main.py b/drishti/main.py index f55f00b..45dc965 100644 --- a/drishti/main.py +++ b/drishti/main.py @@ -42,6 +42,7 @@ insights_operation = [] insights_metadata = [] +insights_dxt = [] insights_total = dict() @@ -95,6 +96,8 @@ INSIGHTS_MPI_IO_AGGREGATORS_INTRA = 'M08' INSIGHTS_MPI_IO_AGGREGATORS_INTER = 'M09' INSIGHTS_MPI_IO_AGGREGATORS_OK = 'M10' +INSIGHTS_DXT_RANK_ZERO_IMBALANCE = 'D01' +INSIGHTS_DXT_RANK_IMBALANCE = 'D02' # TODO: need to verify the threashold to be between 0 and 1 # TODO: read thresholds from file @@ -179,6 +182,20 @@ help='Export a CSV with the code of all issues that were triggered' ) +parser.add_argument( + '--rank_zero_imbalance', + default=False, + action='store_true', + dest='rank_zero_imbalance', + help=argparse.SUPPRESS) + +parser.add_argument( + '--unbalanced_workload', + default=False, + action='store_true', + dest='unbalanced_workload', + help=argparse.SUPPRESS) + args = parser.parse_args() if args.export_size: @@ -1437,7 +1454,43 @@ def main(): pass except FileNotFoundError: pass + + ######################################################################################################################################################################### + + if args.rank_zero_imbalance: + issue = 'Rank 0 is issuing a lot of I/O requests' + + recommendation = [ + { + 'message': 'Consider using MPI-IO collective' + } + ] + + insights_dxt.append( + message(INSIGHTS_DXT_RANK_ZERO_IMBALANCE, TARGET_DEVELOPER, HIGH, issue, recommendation) + ) + + ######################################################################################################################################################################### + + if args.unbalanced_workload: + issue = 'Detected unbalanced workload between the ranks' + + recommendation = [ + { + 'message': 'Consider better balancing the data transfer between the application ranks' + }, + { + 'message': 'Consider tuning the stripe size and count to better distribute the data' + }, + { + 'message': 'If the application uses netCDF and HDF5, double check the need to set NO_FILL values' + } + ] + insights_dxt.append( + message(INSIGHTS_DXT_RANK_IMBALANCE, TARGET_DEVELOPER, HIGH, issue, recommendation) + ) + ######################################################################################################################################################################### insights_end_time = time.time() @@ -1527,6 +1580,20 @@ def main(): ) ) + if insights_dxt: + console.print( + Panel( + Padding( + Group( + *insights_dxt + ), + (1, 1) + ), + title='DXT', + title_align='left' + ) + ) + console.print( Panel( ' {} | [white]LBNL[/white] | [white]Drishti report generated at {} in[/white] {:.3f} seconds'.format( @@ -1615,7 +1682,9 @@ def main(): INSIGHTS_MPI_IO_BLOCKING_WRITE_USAGE, INSIGHTS_MPI_IO_AGGREGATORS_INTRA, INSIGHTS_MPI_IO_AGGREGATORS_INTER, - INSIGHTS_MPI_IO_AGGREGATORS_OK + INSIGHTS_MPI_IO_AGGREGATORS_OK, + INSIGHTS_DXT_RANK_ZERO_IMBALANCE, + INSIGHTS_DXT_RANK_IMBALANCE ] detected_issues = dict.fromkeys(issues, False) From 75b823a10faba1dc3f2d213d5662345f25a2b81b Mon Sep 17 00:00:00 2001 From: hamad45 Date: Wed, 25 Jan 2023 17:16:09 -0800 Subject: [PATCH 2/3] Added support to display DXT issues using a JSON file --- drishti/main.py | 61 ++++++++++++++++--------------------------------- 1 file changed, 20 insertions(+), 41 deletions(-) diff --git a/drishti/main.py b/drishti/main.py index 45dc965..1eec3b9 100644 --- a/drishti/main.py +++ b/drishti/main.py @@ -5,6 +5,7 @@ import sys import csv import time +import json import shlex import datetime import argparse @@ -183,17 +184,9 @@ ) parser.add_argument( - '--rank_zero_imbalance', + '--json', default=False, - action='store_true', - dest='rank_zero_imbalance', - help=argparse.SUPPRESS) - -parser.add_argument( - '--unbalanced_workload', - default=False, - action='store_true', - dest='unbalanced_workload', + dest='json', help=argparse.SUPPRESS) args = parser.parse_args() @@ -1457,39 +1450,25 @@ def main(): ######################################################################################################################################################################### - if args.rank_zero_imbalance: - issue = 'Rank 0 is issuing a lot of I/O requests' - - recommendation = [ - { - 'message': 'Consider using MPI-IO collective' - } - ] + if args.json: + f = open(args.json) + data = json.load(f) - insights_dxt.append( - message(INSIGHTS_DXT_RANK_ZERO_IMBALANCE, TARGET_DEVELOPER, HIGH, issue, recommendation) - ) - - ######################################################################################################################################################################### - - if args.unbalanced_workload: - issue = 'Detected unbalanced workload between the ranks' - - recommendation = [ - { - 'message': 'Consider better balancing the data transfer between the application ranks' - }, - { - 'message': 'Consider tuning the stripe size and count to better distribute the data' - }, - { - 'message': 'If the application uses netCDF and HDF5, double check the need to set NO_FILL values' - } - ] + for key, value in data.items(): + issue = value['issue'] + recommendation = [] + for rec in value['recommendations']: + new_message = {'message': rec} + recommendation.append(new_message) - insights_dxt.append( - message(INSIGHTS_DXT_RANK_IMBALANCE, TARGET_DEVELOPER, HIGH, issue, recommendation) - ) + if key == "rank_zero_imbalance": + insights_dxt.append( + message(INSIGHTS_DXT_RANK_ZERO_IMBALANCE, TARGET_DEVELOPER, HIGH, issue, recommendation) + ) + elif key == "unbalanced_workload": + insights_dxt.append( + message(INSIGHTS_DXT_RANK_IMBALANCE, TARGET_DEVELOPER, HIGH, issue, recommendation) + ) ######################################################################################################################################################################### From 6a412ef871741a786e5d5cb541d56b358229c800 Mon Sep 17 00:00:00 2001 From: hamad45 Date: Tue, 31 Jan 2023 16:07:08 -0800 Subject: [PATCH 3/3] Updated code to use generic JSON format --- drishti/main.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/drishti/main.py b/drishti/main.py index 1eec3b9..ad3e02f 100644 --- a/drishti/main.py +++ b/drishti/main.py @@ -97,8 +97,6 @@ INSIGHTS_MPI_IO_AGGREGATORS_INTRA = 'M08' INSIGHTS_MPI_IO_AGGREGATORS_INTER = 'M09' INSIGHTS_MPI_IO_AGGREGATORS_OK = 'M10' -INSIGHTS_DXT_RANK_ZERO_IMBALANCE = 'D01' -INSIGHTS_DXT_RANK_IMBALANCE = 'D02' # TODO: need to verify the threashold to be between 0 and 1 # TODO: read thresholds from file @@ -1450,26 +1448,27 @@ def main(): ######################################################################################################################################################################### + codes = [] if args.json: f = open(args.json) data = json.load(f) - for key, value in data.items(): - issue = value['issue'] - recommendation = [] - for rec in value['recommendations']: - new_message = {'message': rec} - recommendation.append(new_message) + for key, values in data.items(): + for value in values: + code = value['code'] + codes.append(code) + + level = value['level'] + issue = value['issue'] + recommendation = [] + for rec in value['recommendations']: + new_message = {'message': rec} + recommendation.append(new_message) - if key == "rank_zero_imbalance": - insights_dxt.append( - message(INSIGHTS_DXT_RANK_ZERO_IMBALANCE, TARGET_DEVELOPER, HIGH, issue, recommendation) - ) - elif key == "unbalanced_workload": insights_dxt.append( - message(INSIGHTS_DXT_RANK_IMBALANCE, TARGET_DEVELOPER, HIGH, issue, recommendation) + message(code, TARGET_DEVELOPER, level, issue, recommendation) ) - + ######################################################################################################################################################################### insights_end_time = time.time() @@ -1661,10 +1660,10 @@ def main(): INSIGHTS_MPI_IO_BLOCKING_WRITE_USAGE, INSIGHTS_MPI_IO_AGGREGATORS_INTRA, INSIGHTS_MPI_IO_AGGREGATORS_INTER, - INSIGHTS_MPI_IO_AGGREGATORS_OK, - INSIGHTS_DXT_RANK_ZERO_IMBALANCE, - INSIGHTS_DXT_RANK_IMBALANCE + INSIGHTS_MPI_IO_AGGREGATORS_OK ] + if codes: + issues.extend(codes) detected_issues = dict.fromkeys(issues, False) detected_issues['JOB'] = job['job']['jobid']