diff --git a/dev-requirements.txt b/dev-requirements.txt index efe1a66..e618593 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -9,7 +9,7 @@ numpy>=1.26.4,<=2.1.1 psutil>=5.9.0,<=6.0.0 packaging>=21.0,<=24.1 -qgate_graph==1.4.16 +qgate_graph==1.4.17 coverage>=7 coverage-badge>=1 diff --git a/main.py b/main.py index 15d6055..e3df458 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,6 @@ from qgate_perf.parallel_executor import ParallelExecutor from qgate_perf.parallel_probe import ParallelProbe from qgate_perf.run_setup import RunSetup - import click import logging import time diff --git a/qgate_perf/parallel_executor.py b/qgate_perf/parallel_executor.py index 1edd0a2..f34b06f 100644 --- a/qgate_perf/parallel_executor.py +++ b/qgate_perf/parallel_executor.py @@ -1,12 +1,10 @@ -import gc -import os.path -from multiprocessing import Process +import concurrent.futures import multiprocessing +import os.path import datetime import time -from concurrent.futures import ThreadPoolExecutor -import concurrent.futures import json +import gc from qgate_perf.file_format import FileFormat from qgate_perf.run_setup import RunSetup from qgate_perf.bundle_helper import BundleHelper @@ -15,8 +13,6 @@ from qgate_perf.run_return import RunReturn from platform import python_version from packaging import version -from qgate_graph.graph_performance import GraphPerformance -from qgate_graph.graph_executor import GraphExecutor from contextlib import suppress @@ -69,7 +65,7 @@ def __init__(self, def _coreThreadClassPool(self, threads, return_key, return_dict, run_setup: RunSetup): try: - with ThreadPoolExecutor(max_workers=threads) as executor: + with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor: features = [] for threadKey in range(threads): run_return=RunReturn(f"{return_key}x{threadKey}", return_dict) @@ -114,7 +110,7 @@ def _coreThreadClassPool(self, threads, return_key, return_dict, run_setup: RunS # t = None # # def _coreThreadPool(func, threads, return_key, return_dict, run_setup): - # with ThreadPoolExecutor(max_workers=threads) as executor: + # with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor: # features = [] # for thread_key in range(threads): # features.append( @@ -237,7 +233,7 @@ def _open_output(self): dirname = os.path.dirname(self._output_file) if dirname: if not os.path.exists(dirname): - os.makedirs(dirname) + os.makedirs(dirname, mode=0o777) return open(self._output_file, 'a') def _executeCore(self, run_setup: RunSetup, return_dict, processes=2, threads=2): @@ -253,18 +249,18 @@ def _executeCore(self, run_setup: RunSetup, return_dict, processes=2, threads=2) if threads == 1: for process_key in range(processes): run_return = RunReturn(process_key, return_dict) - p = Process(target=self._func_wrapper, + p = multiprocessing.Process(target=self._func_wrapper, args=(self._func, run_return, run_setup)) # oldversion args=(process_key, return_dict, run_setup)) proc.append(p) else: for process_key in range(processes): - p = Process(target=self._coreThreadClassPool, + p = multiprocessing.Process(target=self._coreThreadClassPool, args=(threads, process_key, return_dict, run_setup)) - # p = Process(target=self._coreThreadClass, args=(threads, process_key, return_dict, run_setup)) - # p = Process(target=ParallelExecutor._coreThread, args=(self.func, threads, process_key, return_dict, run_setup)) - # p = Process(target=ParallelExecutor._coreThreadPool, args=(self.func, threads, process_key, return_dict, run_setup)) + # p = multiprocessing.target=self._coreThreadClass, args=(threads, process_key, return_dict, run_setup)) + # p = multiprocessing.Process(target=ParallelExecutor._coreThread, args=(self.func, threads, process_key, return_dict, run_setup)) + # p = multiprocessing.Process(target=ParallelExecutor._coreThreadPool, args=(self.func, threads, process_key, return_dict, run_setup)) proc.append(p) # start @@ -479,6 +475,9 @@ def create_graph(self, output_graph_dir="output", picture_dpi=100, suppress_erro :param suppress_error: suppress error (default is False) :return: list of output files """ + from qgate_graph.graph_performance import GraphPerformance + from qgate_graph.graph_executor import GraphExecutor + output_file=[] graph = GraphPerformance(picture_dpi) @@ -499,6 +498,7 @@ def create_graph_perf(self, output_graph_dir="output", picture_dpi=100, suppress :param suppress_error: suppress error (default is False) :return: list of output files """ + from qgate_graph.graph_performance import GraphPerformance graph = GraphPerformance(picture_dpi) return graph.generate_from_file(self._output_file, os.path.join(output_graph_dir,"graph-perf"), suppress_error) @@ -512,6 +512,7 @@ def create_graph_exec(self, output_graph_dir="output", picture_dpi=100, suppress :param suppress_error: suppress error (default is False) :return: list of output files """ + from qgate_graph.graph_executor import GraphExecutor graph = GraphExecutor(picture_dpi) return graph.generate_from_file(self._output_file,os.path.join(output_graph_dir,"graph-exec"), suppress_error) diff --git a/qgate_perf/parallel_probe.py b/qgate_perf/parallel_probe.py index 35ca1ce..c403e72 100644 --- a/qgate_perf/parallel_probe.py +++ b/qgate_perf/parallel_probe.py @@ -1,10 +1,11 @@ import datetime import time -import os, sys, math +import os import json +from qgate_perf.standard_deviation import StandardDeviation from qgate_perf.file_format import FileFormat from qgate_perf.run_setup import RunSetup -from qgate_perf.standard_deviation import StandardDeviation +from math import nan class ParallelProbe: @@ -23,7 +24,7 @@ def __init__(self, run_setup: RunSetup, exception=None): if exception is None: self.total_duration = 0 - self.min_duration = sys.maxsize + self.min_duration = 1000000000 self.max_duration = 0 self.standard_deviation = 0 self.track_init = datetime.datetime.utcnow() @@ -101,7 +102,7 @@ def __str__(self): FileFormat.PRF_TYPE: FileFormat.PRF_DETAIL_TYPE, FileFormat.PRF_DETAIL_PROCESSID: self.pid, # info FileFormat.PRF_DETAIL_CALLS: self.counter, # for perf graph - FileFormat.PRF_DETAIL_AVRG: math.nan if self.counter == 0 else self.total_duration / self.counter, + FileFormat.PRF_DETAIL_AVRG: nan if self.counter == 0 else self.total_duration / self.counter, FileFormat.PRF_DETAIL_MIN: self.min_duration, # info FileFormat.PRF_DETAIL_MAX: self.max_duration, # info FileFormat.PRF_DETAIL_STDEV: self.standard_deviation, # for perf graph diff --git a/qgate_perf/version.py b/qgate_perf/version.py index 96bf01a..01bf168 100644 --- a/qgate_perf/version.py +++ b/qgate_perf/version.py @@ -1,3 +1,3 @@ # Store the version here so: -__version__ = '0.4.18' \ No newline at end of file +__version__ = '0.4.19' \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index d0e1b08..3f5491b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,4 @@ numpy>=1.26.4,<=2.1.1 psutil>=5.9.0,<=6.0.0 packaging>=21.0,<=24.1 -qgate_graph==1.4.16 +qgate_graph==1.4.17