From 8f352ac6a27f313f77e2ea0d89cd6e276620ce40 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 21 Sep 2022 08:04:25 +0100 Subject: [PATCH 1/5] renamed to iterate_placements_by_xy_and_type --- .../interface_functions/host_bit_field_router_compressor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spinn_front_end_common/interface/interface_functions/host_bit_field_router_compressor.py b/spinn_front_end_common/interface/interface_functions/host_bit_field_router_compressor.py index 49845f4d6f..cb28cf4eb7 100644 --- a/spinn_front_end_common/interface/interface_functions/host_bit_field_router_compressor.py +++ b/spinn_front_end_common/interface/interface_functions/host_bit_field_router_compressor.py @@ -268,7 +268,7 @@ def get_bit_field_sdram_base_addresses(self, chip_x, chip_y): # pylint: disable=too-many-arguments, unused-argument # locate the bitfields in a chip level scope base_addresses = dict() - for placement in FecDataView.iterate_placements_with_vertex_type( + for placement in FecDataView.iterate_placements_by_xy_and_type( chip_x, chip_y, AbstractSupportsBitFieldRoutingCompression): vertex = placement.vertex base_addresses[placement.p] = vertex.bit_field_base_address( From bbe5a114f8e6761fdf2f7db3bac881b2145dda9c Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 21 Sep 2022 09:27:25 +0100 Subject: [PATCH 2/5] iterate_placements_by_vertex_type --- .../compute_energy_used.py | 4 ++-- .../locate_executable_start_type.py | 23 ++++++++----------- .../report_functions/energy_report.py | 9 ++++---- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/spinn_front_end_common/interface/interface_functions/compute_energy_used.py b/spinn_front_end_common/interface/interface_functions/compute_energy_used.py index 1b1fc0d272..25b3650388 100644 --- a/spinn_front_end_common/interface/interface_functions/compute_energy_used.py +++ b/spinn_front_end_common/interface/interface_functions/compute_energy_used.py @@ -165,8 +165,8 @@ def _compute_energy_consumption( def __find_monitor_placements(): return [placement - for placement in FecDataView.iterate_placemements() - if isinstance(placement.vertex, ChipPowerMonitorMachineVertex)] + for placement in FecDataView.iterate_placements_by_vertex_type( + ChipPowerMonitorMachineVertex)] _COST_PER_TYPE = { diff --git a/spinn_front_end_common/interface/interface_functions/locate_executable_start_type.py b/spinn_front_end_common/interface/interface_functions/locate_executable_start_type.py index 781957525b..0643cfa67d 100644 --- a/spinn_front_end_common/interface/interface_functions/locate_executable_start_type.py +++ b/spinn_front_end_common/interface/interface_functions/locate_executable_start_type.py @@ -27,19 +27,16 @@ def locate_executable_start_type(): """ binary_start_types = dict() - progress = ProgressBar( - FecDataView.get_n_placements(), "Finding executable start types") - for placement in progress.over(FecDataView.iterate_placemements()): - - if isinstance(placement.vertex, AbstractHasAssociatedBinary): - bin_type = placement.vertex.get_binary_start_type() - # update core subset with location of the vertex on the - # machine - if bin_type not in binary_start_types: - binary_start_types[bin_type] = CoreSubsets() - - __add_vertex_to_subset( - placement, binary_start_types[bin_type]) + for placement in FecDataView.iterate_placements_by_vertex_type( + AbstractHasAssociatedBinary): + bin_type = placement.vertex.get_binary_start_type() + # update core subset with location of the vertex on the + # machine + if bin_type not in binary_start_types: + binary_start_types[bin_type] = CoreSubsets() + + __add_vertex_to_subset( + placement, binary_start_types[bin_type]) # only got apps with no binary, such as external devices. # return no app diff --git a/spinn_front_end_common/utilities/report_functions/energy_report.py b/spinn_front_end_common/utilities/report_functions/energy_report.py index 6b272c541c..ccc915a0cf 100644 --- a/spinn_front_end_common/utilities/report_functions/energy_report.py +++ b/spinn_front_end_common/utilities/report_functions/energy_report.py @@ -173,11 +173,10 @@ def _write_detailed_report(self, power_used, f): # sort what to report by chip active_chips = defaultdict(dict) - for placement in FecDataView.iterate_placemements(): - vertex = placement.vertex - if not isinstance(vertex, ChipPowerMonitorMachineVertex): - labels = active_chips[placement.x, placement.y] - labels[placement.p] = vertex.label + for placement in FecDataView.iterate_placements_by_vertex_type( + ChipPowerMonitorMachineVertex): + labels = active_chips[placement.x, placement.y] + labels[placement.p] = placement.vertex.label for xy in active_chips: self._write_chips_active_cost( xy, active_chips[xy], runtime_total_ms, power_used, f) From af1ccb2a1a221df424d28c3cad12f718a9d8db21 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 21 Sep 2022 10:56:17 +0100 Subject: [PATCH 3/5] merge buffer_extractor into buffer_manager.py and fix it --- .../interface/abstract_spinnaker_base.py | 5 +- .../buffer_management/buffer_manager.py | 49 +++++++++------ .../interface/interface_functions/__init__.py | 4 +- .../interface_functions/buffer_extractor.py | 59 ------------------- 4 files changed, 35 insertions(+), 82 deletions(-) delete mode 100644 spinn_front_end_common/interface/interface_functions/buffer_extractor.py diff --git a/spinn_front_end_common/interface/abstract_spinnaker_base.py b/spinn_front_end_common/interface/abstract_spinnaker_base.py index af21aca2be..7fed5cb647 100644 --- a/spinn_front_end_common/interface/abstract_spinnaker_base.py +++ b/spinn_front_end_common/interface/abstract_spinnaker_base.py @@ -71,7 +71,7 @@ from spinn_front_end_common.data import FecTimer from spinn_front_end_common.interface.config_handler import ConfigHandler from spinn_front_end_common.interface.interface_functions import ( - application_finisher, application_runner, buffer_extractor, + application_finisher, application_runner, buffer_manager_creator, chip_io_buf_clearer, chip_io_buf_extractor, chip_provenance_updater, chip_runtime_updater, compute_energy_used, create_notification_protocol, database_interface, @@ -2111,7 +2111,8 @@ def _execute_buffer_extractor(self): with FecTimer(RUN_LOOP, "Buffer extractor") as timer: if timer.skip_if_virtual_board(): return - buffer_extractor() + bm = self._data_writer.get_buffer_manager() + bm.get_placement_data() def _do_extract_from_machine(self): """ diff --git a/spinn_front_end_common/interface/buffer_management/buffer_manager.py b/spinn_front_end_common/interface/buffer_management/buffer_manager.py index 4c34c9b9c7..efb598199a 100644 --- a/spinn_front_end_common/interface/buffer_management/buffer_manager.py +++ b/spinn_front_end_common/interface/buffer_management/buffer_manager.py @@ -529,33 +529,47 @@ def stop(self): with self._thread_lock_buffer_out: self._finished = True - def get_data_for_placements(self, recording_placements, progress=None): + def __count_regions(self): """ - :param ~pacman.model.placements.Placements recording_placements: - Where to get the data from. May not be all placements - :param progress: How to measure/display the progress. - :type progress: ~spinn_utilities.progress_bar.ProgressBar or None + :rtype: tuple(int, list(~.Placement)) """ - if self._java_caller is not None: - self._java_caller.set_placements(recording_placements) + # Count the regions to be read + n_regions_to_read = 0 + recording_placements = list() + for placement in FecDataView.iterate_placements_by_vertex_type( + AbstractReceiveBuffersToHost): + vertex = placement.vertex + n_regions_to_read += len(vertex.get_recorded_region_ids()) + recording_placements.append(placement) + return n_regions_to_read, recording_placements + def get_placement_data(self): timer = Timer() with timer: with self._thread_lock_buffer_out: if self._java_caller is not None: - self._java_caller.get_all_data() - if progress: - progress.end() - elif get_config_bool( - "Machine", "enable_advanced_monitor_support"): - self.__old_get_data_for_placements_with_monitors( - recording_placements, progress) + self.__get_data_for_placements_using_java() else: - self.__old_get_data_for_placements( - recording_placements, progress) + n_regions, recording_placements = self.__count_regions() + progress = ProgressBar( + n_regions, "Extracting buffers from the last run") + if get_config_bool( + "Machine", "enable_advanced_monitor_support"): + self.__old_get_data_for_placements_with_monitors( + recording_placements, progress) + else: + self.__old_get_data_for_placements( + recording_placements, progress) with ProvenanceWriter() as db: db.insert_category_timing(BUFFER, timer.measured_interval, None) + def __get_data_for_placements_using_java(self): + logger.info("Starting buffer extraction using Java") + self._java_caller.set_placements( + FecDataView.iterate_placements_by_vertex_type( + AbstractReceiveBuffersToHost)) + self._java_caller.get_all_data() + def __old_get_data_for_placements_with_monitors( self, recording_placements, progress): """ @@ -589,8 +603,7 @@ def __old_get_data_for_placements(self, recording_placements, progress): vertex = placement.vertex for recording_region_id in vertex.get_recorded_region_ids(): self._retreive_by_placement(placement, recording_region_id) - if progress is not None: - progress.update() + progress.update() def get_data_by_placement(self, placement, recording_region_id): """ Get the data container for all the data retrieved\ diff --git a/spinn_front_end_common/interface/interface_functions/__init__.py b/spinn_front_end_common/interface/interface_functions/__init__.py index e0ad269f81..1c6d1d0e94 100644 --- a/spinn_front_end_common/interface/interface_functions/__init__.py +++ b/spinn_front_end_common/interface/interface_functions/__init__.py @@ -15,7 +15,6 @@ from .application_finisher import application_finisher from .application_runner import application_runner -from .buffer_extractor import buffer_extractor from .chip_iobuf_clearer import chip_io_buf_clearer from .chip_iobuf_extractor import chip_io_buf_extractor from .buffer_manager_creator import buffer_manager_creator @@ -63,8 +62,7 @@ __all__ = [ - "application_finisher", - "application_runner", "buffer_extractor", + "application_finisher", "application_runner", "buffer_manager_creator", "chip_io_buf_clearer", "chip_io_buf_extractor", "chip_provenance_updater", "chip_runtime_updater", "create_notification_protocol", diff --git a/spinn_front_end_common/interface/interface_functions/buffer_extractor.py b/spinn_front_end_common/interface/interface_functions/buffer_extractor.py deleted file mode 100644 index cca6ff4701..0000000000 --- a/spinn_front_end_common/interface/interface_functions/buffer_extractor.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (c) 2017-2019 The University of Manchester -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import logging -from spinn_utilities.log import FormatAdapter -from spinn_utilities.progress_bar import ProgressBar -from spinn_front_end_common.data import FecDataView -from spinn_front_end_common.interface.buffer_management.buffer_models import ( - AbstractReceiveBuffersToHost) - -logger = FormatAdapter(logging.getLogger(__name__)) - - -def buffer_extractor(): - """ Extracts data in between runs. - - """ - # Count the regions to be read - n_regions_to_read, recording_placements = _count_regions() - if not n_regions_to_read: - logger.info("No recorded data to extract") - return - - # Read back the regions - progress = ProgressBar( - n_regions_to_read, "Extracting buffers from the last run") - buffer_manager = FecDataView.get_buffer_manager() - try: - buffer_manager.get_data_for_placements(recording_placements, progress) - finally: - progress.end() - - -def _count_regions(): - """ - :rtype: tuple(int, list(~.Placement)) - """ - # Count the regions to be read - n_regions_to_read = 0 - recording_placements = list() - for app_vertex in FecDataView.iterate_vertices(): - for vertex in app_vertex.machine_vertices: - if isinstance(vertex, AbstractReceiveBuffersToHost): - n_regions_to_read += len(vertex.get_recorded_region_ids()) - placement = FecDataView.get_placement_of_vertex(vertex) - recording_placements.append(placement) - return n_regions_to_read, recording_placements From 6aed41f94ed2ae1e2d7e42617c8fb898c385c737 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 21 Sep 2022 11:05:24 +0100 Subject: [PATCH 4/5] flake8 --- .../interface/interface_functions/compute_energy_used.py | 2 +- .../interface_functions/locate_executable_start_type.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/spinn_front_end_common/interface/interface_functions/compute_energy_used.py b/spinn_front_end_common/interface/interface_functions/compute_energy_used.py index 25b3650388..c7fb45bb31 100644 --- a/spinn_front_end_common/interface/interface_functions/compute_energy_used.py +++ b/spinn_front_end_common/interface/interface_functions/compute_energy_used.py @@ -166,7 +166,7 @@ def _compute_energy_consumption( def __find_monitor_placements(): return [placement for placement in FecDataView.iterate_placements_by_vertex_type( - ChipPowerMonitorMachineVertex)] + ChipPowerMonitorMachineVertex)] _COST_PER_TYPE = { diff --git a/spinn_front_end_common/interface/interface_functions/locate_executable_start_type.py b/spinn_front_end_common/interface/interface_functions/locate_executable_start_type.py index 0643cfa67d..ea0f8a2177 100644 --- a/spinn_front_end_common/interface/interface_functions/locate_executable_start_type.py +++ b/spinn_front_end_common/interface/interface_functions/locate_executable_start_type.py @@ -13,7 +13,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from spinn_utilities.progress_bar import ProgressBar from spinn_machine import CoreSubsets from spinn_front_end_common.abstract_models import AbstractHasAssociatedBinary from spinn_front_end_common.data import FecDataView From 39f6446455edf6389d7253f2682c7cca56f73f83 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 21 Sep 2022 11:17:51 +0100 Subject: [PATCH 5/5] pylint --- .../interface/buffer_management/buffer_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spinn_front_end_common/interface/buffer_management/buffer_manager.py b/spinn_front_end_common/interface/buffer_management/buffer_manager.py index efb598199a..8b57f67b88 100644 --- a/spinn_front_end_common/interface/buffer_management/buffer_manager.py +++ b/spinn_front_end_common/interface/buffer_management/buffer_manager.py @@ -554,7 +554,7 @@ def get_placement_data(self): progress = ProgressBar( n_regions, "Extracting buffers from the last run") if get_config_bool( - "Machine", "enable_advanced_monitor_support"): + "Machine", "enable_advanced_monitor_support"): self.__old_get_data_for_placements_with_monitors( recording_placements, progress) else: @@ -569,7 +569,7 @@ def __get_data_for_placements_using_java(self): FecDataView.iterate_placements_by_vertex_type( AbstractReceiveBuffersToHost)) self._java_caller.get_all_data() - + def __old_get_data_for_placements_with_monitors( self, recording_placements, progress): """