From 097fb6fa642d79a7bda50e0d01f2d679470f2874 Mon Sep 17 00:00:00 2001 From: Tuomas Myllari Date: Mon, 20 May 2024 18:26:39 +0300 Subject: [PATCH] Implement Elmer simulation performance profiling as a post-processing step Minor changes: - Set gmsh_n_threads default already in export - Update waveguides_sim_xsection to use the new profiling post-process script --- .../simulations/export/elmer/elmer_export.py | 13 +- .../post_process/elmer_profiler.py | 128 ++++++++++++++++++ .../simulations/waveguides_sim_xsection.py | 26 ++-- 3 files changed, 143 insertions(+), 24 deletions(-) create mode 100644 klayout_package/python/scripts/simulations/post_process/elmer_profiler.py diff --git a/klayout_package/python/kqcircuits/simulations/export/elmer/elmer_export.py b/klayout_package/python/kqcircuits/simulations/export/elmer/elmer_export.py index 47d4defec..f6d759eea 100644 --- a/klayout_package/python/kqcircuits/simulations/export/elmer/elmer_export.py +++ b/klayout_package/python/kqcircuits/simulations/export/elmer/elmer_export.py @@ -679,16 +679,17 @@ def _update_elmer_workflow(simulations, common_solution, workflow): if requested_cpus > max_cpus: logging.warning(f"Requested more CPUs ({requested_cpus}) than available ({max_cpus})") - workflow["n_workers"] = n_workers - workflow["elmer_n_processes"] = n_processes - workflow["elmer_n_threads"] = n_threads - gmsh_n_threads = workflow.get("gmsh_n_threads", 1) if gmsh_n_threads == -1: if parallelization_level == "full_simulation": - workflow["gmsh_n_threads"] = max(max_cpus // n_workers, 1) + gmsh_n_threads = max(max_cpus // n_workers, 1) else: - workflow["gmsh_n_threads"] = max_cpus + gmsh_n_threads = max_cpus + + workflow["n_workers"] = n_workers + workflow["elmer_n_processes"] = n_processes + workflow["elmer_n_threads"] = n_threads + workflow["gmsh_n_threads"] = gmsh_n_threads parser = argparse.ArgumentParser() parser.add_argument("-q", "--quiet", action="store_true") diff --git a/klayout_package/python/scripts/simulations/post_process/elmer_profiler.py b/klayout_package/python/scripts/simulations/post_process/elmer_profiler.py new file mode 100644 index 000000000..c2dadad3f --- /dev/null +++ b/klayout_package/python/scripts/simulations/post_process/elmer_profiler.py @@ -0,0 +1,128 @@ +# This code is part of KQCircuits +# Copyright (C) 2024 IQM Finland Oy +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with this program. If not, see +# https://www.gnu.org/licenses/gpl-3.0.html. +# +# The software distribution should follow IQM trademark policy for open-source software +# (meetiqm.com/iqm-open-source-trademark-policy). IQM welcomes contributions to the code. +# Please see our contribution agreements for individuals (meetiqm.com/iqm-individual-contributor-license-agreement) +# and organizations (meetiqm.com/iqm-organization-contributor-license-agreement). + + +""" +Produces table of runtimes for gmsh and Elmer and the number of mesh tetrahedron from Elmer results +""" + +import re +import os +import sys +import json +import logging +from pathlib import Path + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "util")) +from post_process_helpers import ( # pylint: disable=wrong-import-position, no-name-in-module + find_varied_parameters, + tabulate_into_csv, +) + + +def _load_elmer_runtimes(path: Path, name: str, elmer_n_processes: int) -> dict: + """Parse Elmer log files in path/log_files for the runtimes and sum the results if multiple files + are found with endings "", "_C", "_L", "_C0". + + Multiplies the CPU time reported by elmer by elmer_n_processes to get realistic CPU runtime + """ + logs_folder = Path(path).joinpath("log_files") + + times = {} + + endings = ["", "_C", "_L", "_C0"] + for end in endings: + log_file = logs_folder.joinpath(name + end + ".Elmer.log") + if log_file.is_file(): + with open(log_file, "r") as f: + for line in reversed(f.readlines()): + if "SOLVER TOTAL TIME(CPU,REAL):" in line: + sp_line = line.rstrip().split() + times["elmer_time_cpu"] = times.get("elmer_time_cpu", 0) + float(sp_line[-2]) + times["elmer_time_real"] = times.get("elmer_time_real", 0) + float(sp_line[-1]) + break + + if not times: + logging.warning(f"No log file found for {name}") + + times["elmer_time_cpu"] = elmer_n_processes * times["elmer_time_cpu"] + return times + + +def _load_elmer_elements(path: Path, name: str) -> dict: + """Parse path/mesh.header for the number of mesh elements used in Elmer""" + log_file = Path(path).joinpath(name).joinpath("mesh.header") + if log_file.is_file(): + with open(log_file, "r") as f: + for line in f: + return {"elmer_elements": line.rstrip().split()[1]} + else: + logging.warning(f"No file found at {log_file}") + return {} + + +def _load_gmsh_data(path: Path, name: str) -> dict: + """Parse Gmsh log file found in path/log_files for the CPU and real runtimes""" + log_file = Path(path).joinpath("log_files").joinpath(name + ".Gmsh.log") + + if log_file.is_file(): + with open(log_file, "r") as f: + lines = f.readlines() + # Sum times used for meshing lines (1D), surfaces (2D) and volumes (3D) which + # are reported separately by Gmsh + search_str = r"\s+(\d+\.\d+)" + times = [[0], [0]] + for line in lines: + if "Info : Done meshing " in line: + for i, s in enumerate(("Wall", "CPU")): + match = re.search(s + search_str, line) + if match: + times[i].append(float(match.group(1))) + return {"gmsh_time_real": sum(times[0]), "gmsh_time_cpu": sum(times[1])} + else: + logging.warning(f"No log file found at {log_file}") + return {} + + +def _load_workflow_data(definition_file: Path) -> dict: + """Load relevant parts of workflow dict""" + with open(definition_file, "r") as f: + json_data = json.load(f) + return {key: json_data["workflow"][key] for key in ("elmer_n_processes", "elmer_n_threads", "gmsh_n_threads")} + + +# Find data files +path = os.path.curdir +names = [f.removesuffix("_project_results.json") for f in os.listdir(path) if f.endswith("_project_results.json")] +if names: + # Find parameters that are swept + definition_files = [f + ".json" for f in names] + parameters, parameter_values = find_varied_parameters(definition_files) + + # Load result data + res = {} + for key, name, definition_file in zip(parameter_values.keys(), names, definition_files): + workflow_data = _load_workflow_data(definition_file) + res[key] = { + **workflow_data, + **_load_gmsh_data(path, name), + **_load_elmer_runtimes(path, name, workflow_data["elmer_n_processes"]), + **_load_elmer_elements(path, name), + } + + tabulate_into_csv(f"{os.path.basename(os.path.abspath(path))}_profile.csv", res, parameters, parameter_values) diff --git a/klayout_package/python/scripts/simulations/waveguides_sim_xsection.py b/klayout_package/python/scripts/simulations/waveguides_sim_xsection.py index 429daaf12..58621705f 100644 --- a/klayout_package/python/scripts/simulations/waveguides_sim_xsection.py +++ b/klayout_package/python/scripts/simulations/waveguides_sim_xsection.py @@ -91,6 +91,7 @@ "python_executable": "python", # use 'kqclib' when using singularity image (you can also put a full path) "elmer_n_processes": -1, # -1 means all the physical cores "elmer_n_threads": 1, # number of omp threads + "gmsh_n_threads": -1, } mesh_size = { @@ -155,27 +156,16 @@ "run_inductance_sim": False, "vtu_output": False, } +post_process = [ + PostProcess("produce_q_factor_table.py", **loss_tangents), + PostProcess("produce_epr_table.py", groups=["ma", "ms", "sa", "substrate", "vacuum"]), + PostProcess("elmer_profiler.py"), +] if do_solution_sweep: solutions = sweep_solution(ElmerCrossSectionSolution, sol_parameters, {"p_element_order": [1, 2, 3]}) - export_elmer( - cross_combine(xsection_simulations, solutions), - path, - workflow=workflow, - post_process=[ - PostProcess("produce_q_factor_table.py", **loss_tangents), - PostProcess("produce_epr_table.py", groups=["ma", "ms", "sa", "substrate", "vacuum"]), - ], - ) + export_elmer(cross_combine(xsection_simulations, solutions), path, workflow=workflow, post_process=post_process) else: export_elmer( - xsection_simulations, - path, - tool="cross-section", - **sol_parameters, - workflow=workflow, - post_process=[ - PostProcess("produce_q_factor_table.py", **loss_tangents), - PostProcess("produce_epr_table.py", groups=["ma", "ms", "sa", "substrate", "vacuum"]), - ], + xsection_simulations, path, tool="cross-section", **sol_parameters, workflow=workflow, post_process=post_process )