Merge pull request #34 from george0st/changes

Tune imports

dev-requirements.txt

@@ -9,7 +9,7 @@ numpy>=1.26.4,<=2.1.1
 psutil>=5.9.0,<=6.0.0
 packaging>=21.0,<=24.1
 
-qgate_graph==1.4.16
+qgate_graph==1.4.17
 
 coverage>=7
 coverage-badge>=1

main.py

@@ -1,7 +1,6 @@
 from qgate_perf.parallel_executor import ParallelExecutor
 from qgate_perf.parallel_probe import ParallelProbe
 from qgate_perf.run_setup import RunSetup
-
 import click
 import logging
 import time

qgate_perf/parallel_executor.py

@@ -1,12 +1,10 @@
-import gc
-import os.path
-from multiprocessing import Process
+import concurrent.futures
 import multiprocessing
+import os.path
 import datetime
 import time
-from concurrent.futures import ThreadPoolExecutor
-import concurrent.futures
 import json
+import gc
 from qgate_perf.file_format import FileFormat
 from qgate_perf.run_setup import RunSetup
 from qgate_perf.bundle_helper import BundleHelper
@@ -15,8 +13,6 @@
 from qgate_perf.run_return import RunReturn
 from platform import python_version
 from packaging import version
-from qgate_graph.graph_performance import GraphPerformance
-from qgate_graph.graph_executor import GraphExecutor
 from contextlib import suppress
 
 
@@ -69,7 +65,7 @@ def __init__(self,
 
     def _coreThreadClassPool(self, threads, return_key, return_dict, run_setup: RunSetup):
         try:
-            with ThreadPoolExecutor(max_workers=threads) as executor:
+            with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:
                 features = []
                 for threadKey in range(threads):
                     run_return=RunReturn(f"{return_key}x{threadKey}", return_dict)
@@ -114,7 +110,7 @@ def _coreThreadClassPool(self, threads, return_key, return_dict, run_setup: RunS
     #         t = None
     #
     # def _coreThreadPool(func, threads, return_key, return_dict, run_setup):
-    #     with ThreadPoolExecutor(max_workers=threads) as executor:
+    #     with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:
     #         features = []
     #         for thread_key in range(threads):
     #             features.append(
@@ -237,7 +233,7 @@ def _open_output(self):
         dirname = os.path.dirname(self._output_file)
         if dirname:
             if not os.path.exists(dirname):
-                os.makedirs(dirname)
+                os.makedirs(dirname, mode=0o777)
         return open(self._output_file, 'a')
 
     def _executeCore(self, run_setup: RunSetup, return_dict, processes=2, threads=2):
@@ -253,18 +249,18 @@ def _executeCore(self, run_setup: RunSetup, return_dict, processes=2, threads=2)
             if threads == 1:
                 for process_key in range(processes):
                     run_return = RunReturn(process_key, return_dict)
-                    p = Process(target=self._func_wrapper,
+                    p = multiprocessing.Process(target=self._func_wrapper,
                                 args=(self._func, run_return, run_setup))
                     # oldversion                args=(process_key, return_dict, run_setup))
 
                     proc.append(p)
             else:
                 for process_key in range(processes):
-                    p = Process(target=self._coreThreadClassPool,
+                    p = multiprocessing.Process(target=self._coreThreadClassPool,
                                 args=(threads, process_key, return_dict, run_setup))
-                    # p = Process(target=self._coreThreadClass, args=(threads, process_key, return_dict, run_setup))
-                    # p = Process(target=ParallelExecutor._coreThread, args=(self.func, threads, process_key, return_dict, run_setup))
-                    # p = Process(target=ParallelExecutor._coreThreadPool, args=(self.func, threads, process_key, return_dict, run_setup))
+                    # p = multiprocessing.target=self._coreThreadClass, args=(threads, process_key, return_dict, run_setup))
+                    # p = multiprocessing.Process(target=ParallelExecutor._coreThread, args=(self.func, threads, process_key, return_dict, run_setup))
+                    # p = multiprocessing.Process(target=ParallelExecutor._coreThreadPool, args=(self.func, threads, process_key, return_dict, run_setup))
                     proc.append(p)
 
             # start
@@ -479,6 +475,9 @@ def create_graph(self, output_graph_dir="output", picture_dpi=100, suppress_erro
         :param suppress_error:      suppress error (default is False)
         :return:                    list of output files
         """
+        from qgate_graph.graph_performance import GraphPerformance
+        from qgate_graph.graph_executor import GraphExecutor
+
         output_file=[]
 
         graph = GraphPerformance(picture_dpi)
@@ -499,6 +498,7 @@ def create_graph_perf(self, output_graph_dir="output", picture_dpi=100, suppress
         :param suppress_error:      suppress error (default is False)
         :return:                    list of output files
         """
+        from qgate_graph.graph_performance import GraphPerformance
 
         graph = GraphPerformance(picture_dpi)
         return graph.generate_from_file(self._output_file, os.path.join(output_graph_dir,"graph-perf"), suppress_error)
@@ -512,6 +512,7 @@ def create_graph_exec(self, output_graph_dir="output", picture_dpi=100, suppress
         :param suppress_error:      suppress error (default is False)
         :return:                    list of output files
         """
+        from qgate_graph.graph_executor import GraphExecutor
 
         graph = GraphExecutor(picture_dpi)
         return graph.generate_from_file(self._output_file,os.path.join(output_graph_dir,"graph-exec"), suppress_error)

qgate_perf/parallel_probe.py

@@ -1,10 +1,11 @@
 import datetime
 import time
-import os, sys, math
+import os
 import json
+from qgate_perf.standard_deviation import StandardDeviation
 from qgate_perf.file_format import FileFormat
 from qgate_perf.run_setup import RunSetup
-from qgate_perf.standard_deviation import StandardDeviation
+from math import nan
 
 
 class ParallelProbe:
@@ -23,7 +24,7 @@ def __init__(self, run_setup: RunSetup, exception=None):
 
         if exception is None:
             self.total_duration = 0
-            self.min_duration = sys.maxsize
+            self.min_duration = 1000000000
             self.max_duration = 0
             self.standard_deviation = 0
             self.track_init = datetime.datetime.utcnow()
@@ -101,7 +102,7 @@ def __str__(self):
                 FileFormat.PRF_TYPE: FileFormat.PRF_DETAIL_TYPE,
                 FileFormat.PRF_DETAIL_PROCESSID: self.pid,                          # info
                 FileFormat.PRF_DETAIL_CALLS: self.counter,                          # for perf graph
-                FileFormat.PRF_DETAIL_AVRG: math.nan if self.counter == 0 else self.total_duration / self.counter,
+                FileFormat.PRF_DETAIL_AVRG: nan if self.counter == 0 else self.total_duration / self.counter,
                 FileFormat.PRF_DETAIL_MIN: self.min_duration,                       # info
                 FileFormat.PRF_DETAIL_MAX: self.max_duration,                       # info
                 FileFormat.PRF_DETAIL_STDEV: self.standard_deviation,               # for perf graph

qgate_perf/version.py

@@ -1,3 +1,3 @@
 # Store the version here so:
 
-__version__ = '0.4.18'
+__version__ = '0.4.19'

requirements.txt

@@ -6,4 +6,4 @@ numpy>=1.26.4,<=2.1.1
 psutil>=5.9.0,<=6.0.0
 packaging>=21.0,<=24.1
 
-qgate_graph==1.4.16
+qgate_graph==1.4.17