From 45ee2f9ba7f529bdde63f3baf9a0f8cbf840be48 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Tue, 5 Oct 2021 13:08:20 -0600 Subject: [PATCH 01/37] make copy of residential_hpxml workflow generator to get started on hes workflow --- .../residential_hpxml_hes.py | 208 ++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 buildstockbatch/workflow_generator/residential_hpxml_hes.py diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py new file mode 100644 index 00000000..ff34cfa4 --- /dev/null +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -0,0 +1,208 @@ +# -*- coding: utf-8 -*- + +""" +buildstockbatch.workflow_generator.residential_hpxml +~~~~~~~~~~~~~~~ +This object contains the residential classes for generating OSW files from individual samples + +:author: Joe Robertson +:copyright: (c) 2021 by The Alliance for Sustainable Energy +:license: BSD-3 +""" + +import datetime as dt +import json +import logging +import re +import yamale + +from .base import WorkflowGeneratorBase + +logger = logging.getLogger(__name__) + + +class ResidentialHpxmlWorkflowGenerator(WorkflowGeneratorBase): + + @classmethod + def validate(cls, cfg): + """Validate arguments + + :param cfg: project configuration + :type cfg: dict + """ + schema_yml = """ + measures_to_ignore: list(str(), required=False) + build_existing_model: map(required=False) + reporting_measures: list(include('measure-spec'), required=False) + simulation_output_report: map(required=False) + server_directory_cleanup: map(required=False) + --- + measure-spec: + measure_dir_name: str(required=True) + arguments: map(required=False) + """ + workflow_generator_args = cfg['workflow_generator']['args'] + schema_yml = re.sub(r'^ {8}', '', schema_yml, flags=re.MULTILINE) + schema = yamale.make_schema(content=schema_yml, parser='ruamel') + data = yamale.make_data(content=json.dumps(workflow_generator_args), parser='ruamel') + yamale.validate(schema, data, strict=True) + return True + + def reporting_measures(self): + """Return a list of reporting measures to include in outputs""" + workflow_args = self.cfg['workflow_generator'].get('args', {}) + return [x['measure_dir_name'] for x in workflow_args.get('reporting_measures', [])] + + def create_osw(self, sim_id, building_id, upgrade_idx): + """ + Generate and return the osw as a python dict + + :param sim_id: simulation id, looks like 'bldg0000001up01' + :param building_id: integer building id to use from the sampled buildstock.csv + :param upgrade_idx: integer index of the upgrade scenario to apply, None if baseline + """ + # Default argument values + workflow_args = { + 'build_existing_model': {}, + 'measures': [], + 'simulation_output_report': {}, + 'server_directory_cleanup': {} + } + workflow_args.update(self.cfg['workflow_generator'].get('args', {})) + + logger.debug('Generating OSW, sim_id={}'.format(sim_id)) + + sim_ctl_args = { + 'simulation_control_timestep': 60, + 'simulation_control_run_period_begin_month': 1, + 'simulation_control_run_period_begin_day_of_month': 1, + 'simulation_control_run_period_end_month': 12, + 'simulation_control_run_period_end_day_of_month': 31, + 'simulation_control_run_period_calendar_year': 2007, + 'debug': False, + 'add_component_loads': False + } + + bld_exist_model_args = { + 'building_id': building_id, + 'sample_weight': self.n_datapoints / self.cfg['baseline']['n_buildings_represented'] + } + if 'measures_to_ignore' in workflow_args: + bld_exist_model_args['measures_to_ignore'] = '|'.join(workflow_args['measures_to_ignore']) + bld_exist_model_args.update(sim_ctl_args) + bld_exist_model_args.update(workflow_args['build_existing_model']) + + sim_out_rep_args = { + 'timeseries_frequency': 'none', + 'include_timeseries_fuel_consumptions': False, + 'include_timeseries_end_use_consumptions': False, + 'include_timeseries_hot_water_uses': False, + 'include_timeseries_total_loads': False, + 'include_timeseries_component_loads': False, + 'include_timeseries_zone_temperatures': False, + 'include_timeseries_airflows': False, + 'include_timeseries_weather': False, + } + sim_out_rep_args.update(workflow_args['simulation_output_report']) + + osw = { + 'id': sim_id, + 'steps': [ + { + 'measure_dir_name': 'BuildExistingModel', + 'arguments': bld_exist_model_args + } + ], + 'created_at': dt.datetime.now().isoformat(), + 'measure_paths': [ + 'measures', + 'resources/hpxml-measures' + ], + 'run_options': { + 'skip_zip_results': True + } + } + + osw['steps'].extend(workflow_args['measures']) + + server_dir_cleanup_args = { + 'retain_in_osm': False, + 'retain_in_idf': True, + 'retain_pre_process_idf': False, + 'retain_eplusout_audit': False, + 'retain_eplusout_bnd': False, + 'retain_eplusout_eio': False, + 'retain_eplusout_end': False, + 'retain_eplusout_err': False, + 'retain_eplusout_eso': False, + 'retain_eplusout_mdd': False, + 'retain_eplusout_mtd': False, + 'retain_eplusout_rdd': False, + 'retain_eplusout_shd': False, + 'retain_eplusout_sql': False, + 'retain_eplustbl_htm': False, + 'retain_sqlite_err': False, + 'retain_stdout_energyplus': False, + 'retain_stdout_expandobject': False, + 'retain_schedules_csv': True + } + server_dir_cleanup_args.update(workflow_args['server_directory_cleanup']) + + osw['steps'].extend([ + { + 'measure_dir_name': 'ReportSimulationOutput', + 'arguments': sim_out_rep_args + }, + { + 'measure_dir_name': 'ReportHPXMLOutput', + 'arguments': {} + }, + { + 'measure_dir_name': 'UpgradeCosts', + 'arguments': {} + }, + { + 'measure_dir_name': 'ServerDirectoryCleanup', + 'arguments': server_dir_cleanup_args + } + ]) + + if upgrade_idx is not None: + measure_d = self.cfg['upgrades'][upgrade_idx] + apply_upgrade_measure = { + 'measure_dir_name': 'ApplyUpgrade', + 'arguments': { + 'run_measure': 1 + } + } + if 'upgrade_name' in measure_d: + apply_upgrade_measure['arguments']['upgrade_name'] = measure_d['upgrade_name'] + for opt_num, option in enumerate(measure_d['options'], 1): + apply_upgrade_measure['arguments']['option_{}'.format(opt_num)] = option['option'] + if 'lifetime' in option: + apply_upgrade_measure['arguments']['option_{}_lifetime'.format(opt_num)] = option['lifetime'] + if 'apply_logic' in option: + apply_upgrade_measure['arguments']['option_{}_apply_logic'.format(opt_num)] = \ + self.make_apply_logic_arg(option['apply_logic']) + for cost_num, cost in enumerate(option.get('costs', []), 1): + for arg in ('value', 'multiplier'): + if arg not in cost: + continue + apply_upgrade_measure['arguments']['option_{}_cost_{}_{}'.format(opt_num, cost_num, arg)] = \ + cost[arg] + if 'package_apply_logic' in measure_d: + apply_upgrade_measure['arguments']['package_apply_logic'] = \ + self.make_apply_logic_arg(measure_d['package_apply_logic']) + + build_existing_model_idx = \ + [x['measure_dir_name'] == 'BuildExistingModel' for x in osw['steps']].index(True) + osw['steps'].insert(build_existing_model_idx + 1, apply_upgrade_measure) + + if 'reporting_measures' in workflow_args: + for reporting_measure in workflow_args['reporting_measures']: + if 'arguments' not in reporting_measure: + reporting_measure['arguments'] = {} + reporting_measure['measure_type'] = 'ReportingMeasure' + osw['steps'].insert(-1, reporting_measure) # right before ServerDirectoryCleanup + + return osw From a1efb5a31567a5f5987776ea3f8571f03819fbd7 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Wed, 27 Oct 2021 12:26:43 -0600 Subject: [PATCH 02/37] first pass at connecting OS-HEScore using filepath yml argument --- buildstockbatch/base.py | 6 ++- buildstockbatch/localdocker.py | 14 +++++-- buildstockbatch/utils.py | 4 ++ .../workflow_generator/__init__.py | 3 +- .../residential_hpxml_hes.py | 40 +++++++++++++++++-- .../test_workflow_generator.py | 1 + 6 files changed, 59 insertions(+), 9 deletions(-) diff --git a/buildstockbatch/base.py b/buildstockbatch/base.py index fb421ca2..6cbd0973 100644 --- a/buildstockbatch/base.py +++ b/buildstockbatch/base.py @@ -63,7 +63,11 @@ def __init__(self, project_filename): self.project_dir = os.path.join(self.buildstock_dir, self.cfg['project_directory']) if not os.path.isdir(self.project_dir): raise FileNotFoundError(f'project_directory = {self.project_dir} is not a directory.') - + if 'hescore_hpxml' in self.cfg['workflow_generator']['args']: + self.os_hescore_dir = self.cfg['workflow_generator']['args']['hescore_hpxml']['os_hescore_directory'] + else: + self.os_hescore_dir = None + # Load in OS_VERSION and OS_SHA arguments if they exist in the YAML, # otherwise use defaults specified here. self.os_version = self.cfg.get('os_version', self.DEFAULT_OS_VERSION) diff --git a/buildstockbatch/localdocker.py b/buildstockbatch/localdocker.py index 9addc1b4..9f47f99b 100644 --- a/buildstockbatch/localdocker.py +++ b/buildstockbatch/localdocker.py @@ -58,9 +58,9 @@ def validate_project(project_file): @property def docker_image(self): + # return 'nrel/hescore-hpxml-openstudio:latest' return 'nrel/openstudio:{}'.format(self.os_version) - class LocalDockerBatch(DockerBatchBase): def __init__(self, project_filename): @@ -87,7 +87,7 @@ def weather_dir(self): return self._weather_dir.name @classmethod - def run_building(cls, project_dir, buildstock_dir, weather_dir, docker_image, results_dir, measures_only, + def run_building(cls, project_dir, buildstock_dir, weather_dir, os_hescore_dir, docker_image, results_dir, measures_only, n_datapoints, cfg, i, upgrade_idx=None): upgrade_id = 0 if upgrade_idx is None else upgrade_idx + 1 @@ -96,7 +96,6 @@ def run_building(cls, project_dir, buildstock_dir, weather_dir, docker_image, re sim_id, sim_dir = cls.make_sim_dir(i, upgrade_idx, os.path.join(results_dir, 'simulation_output')) except SimulationExists: return - bind_mounts = [ (sim_dir, '', 'rw'), (os.path.join(buildstock_dir, 'measures'), 'measures', 'ro'), @@ -104,9 +103,17 @@ def run_building(cls, project_dir, buildstock_dir, weather_dir, docker_image, re (os.path.join(project_dir, 'housing_characteristics'), 'lib/housing_characteristics', 'ro'), (weather_dir, 'weather', 'ro') ] + + # ResStock-hpxml measure directory if os.path.exists(os.path.join(buildstock_dir, 'resources', 'hpxml-measures')): bind_mounts += [(os.path.join(buildstock_dir, 'resources', 'hpxml-measures'), 'resources/hpxml-measures', 'ro')] + + # OS-HEScore measure directories + if os_hescore_dir and os.path.exists(os_hescore_dir): + bind_mounts += [(os.path.join(os_hescore_dir, 'rulesets'), 'OpenStudio-HEScore/rulesets', 'ro'), + (os.path.join(os_hescore_dir, 'hpxml-measures'), 'OpenStudio-HEScore/hpxml-measures', 'ro')] + docker_volume_mounts = dict([(key, {'bind': f'/var/simdata/openstudio/{bind}', 'mode': mode}) for key, bind, mode in bind_mounts]) # noqa E501 for bind in bind_mounts: dir_to_make = os.path.join(sim_dir, *bind[1].split('/')) @@ -172,6 +179,7 @@ def run_batch(self, n_jobs=None, measures_only=False, sampling_only=False): self.project_dir, self.buildstock_dir, self.weather_dir, + self.os_hescore_dir, self.docker_image, self.results_dir, measures_only, diff --git a/buildstockbatch/utils.py b/buildstockbatch/utils.py index af6592a7..502db352 100644 --- a/buildstockbatch/utils.py +++ b/buildstockbatch/utils.py @@ -30,6 +30,10 @@ def get_project_configuration(project_file): # Set absolute paths cfg['buildstock_directory'] = path_rel_to_file(project_file, cfg['buildstock_directory']) + if 'hescore_hpxml' in cfg['workflow_generator']['args']: + cfg['workflow_generator']['args']['hescore_hpxml']['os_hescore_directory'] = \ + path_rel_to_file('', cfg['workflow_generator']['args']['hescore_hpxml']['os_hescore_directory']) + # if 'precomputed_sample' in cfg.get('baseline', {}): # cfg['baseline']['precomputed_sample'] = \ # path_rel_to_file(project_file, cfg['baseline']['precomputed_sample']) diff --git a/buildstockbatch/workflow_generator/__init__.py b/buildstockbatch/workflow_generator/__init__.py index d97c74ec..299dca33 100644 --- a/buildstockbatch/workflow_generator/__init__.py +++ b/buildstockbatch/workflow_generator/__init__.py @@ -2,4 +2,5 @@ from .residential import ResidentialDefaultWorkflowGenerator # noqa F041 from .commercial import CommercialDefaultWorkflowGenerator # noqa F041 -from .residential_hpxml import ResidentialHpxmlWorkflowGenerator # noqa F041 \ No newline at end of file +from .residential_hpxml import ResidentialHpxmlWorkflowGenerator # noqa F041 +from .residential_hpxml_hes import ResidentialHpxmlHesWorkflowGenerator # noqa F041 \ No newline at end of file diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index ff34cfa4..6a0d4765 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -buildstockbatch.workflow_generator.residential_hpxml +buildstockbatch.workflow_generator.residential_hpxml_hes ~~~~~~~~~~~~~~~ This object contains the residential classes for generating OSW files from individual samples @@ -15,13 +15,14 @@ import logging import re import yamale +import os from .base import WorkflowGeneratorBase logger = logging.getLogger(__name__) -class ResidentialHpxmlWorkflowGenerator(WorkflowGeneratorBase): +class ResidentialHpxmlHesWorkflowGenerator(WorkflowGeneratorBase): @classmethod def validate(cls, cfg): @@ -33,6 +34,7 @@ def validate(cls, cfg): schema_yml = """ measures_to_ignore: list(str(), required=False) build_existing_model: map(required=False) + hescore_hpxml: map(required=True) reporting_measures: list(include('measure-spec'), required=False) simulation_output_report: map(required=False) server_directory_cleanup: map(required=False) @@ -64,6 +66,7 @@ def create_osw(self, sim_id, building_id, upgrade_idx): # Default argument values workflow_args = { 'build_existing_model': {}, + 'hescore_hpxml': {}, 'measures': [], 'simulation_output_report': {}, 'server_directory_cleanup': {} @@ -92,6 +95,21 @@ def create_osw(self, sim_id, building_id, upgrade_idx): bld_exist_model_args.update(sim_ctl_args) bld_exist_model_args.update(workflow_args['build_existing_model']) + os_to_hpxml_args = { + 'hpxml_path': 'fixme', + 'output_dir': 'fixme', + 'debug': 'fixme', + 'add_component_loads': 'fixme', + 'skip_validation': 'fixme', + 'building_id': 'fixme' + } + + hes_ruleset_args = { + 'json_path': 'fixme', + 'hpxml_output_path': 'fixme' + } + + ## FIXME: update for OS-HEScore sim_out_rep_args = { 'timeseries_frequency': 'none', 'include_timeseries_fuel_consumptions': False, @@ -109,12 +127,26 @@ def create_osw(self, sim_id, building_id, upgrade_idx): 'id': sim_id, 'steps': [ { - 'measure_dir_name': 'BuildExistingModel', + 'measure_dir_name': 'BuildExistingModel', #(resstock) 'arguments': bld_exist_model_args + }, + { + 'measure_dir_name': 'HEScoreHPXML', #(resstock) + 'arguments': workflow_args['hescore_hpxml'] + }, + { + 'measure_dir_name': 'HEScoreRuleset', #(OS-HEScore) + 'arguments': hes_ruleset_args + }, + { + 'measure_dir_name': 'HPXMLtoOpenStudio', #(OS-HEScore) + 'arguments': os_to_hpxml_args } ], 'created_at': dt.datetime.now().isoformat(), 'measure_paths': [ + 'OpenStudio-HEScore/rulesets', + 'OpenStudio-HEScore/hpxml-measures', 'measures', 'resources/hpxml-measures' ], @@ -150,7 +182,7 @@ def create_osw(self, sim_id, building_id, upgrade_idx): osw['steps'].extend([ { - 'measure_dir_name': 'ReportSimulationOutput', + 'measure_dir_name': 'SimulationOutputReport', #(OS-HEScore) 'arguments': sim_out_rep_args }, { diff --git a/buildstockbatch/workflow_generator/test_workflow_generator.py b/buildstockbatch/workflow_generator/test_workflow_generator.py index 8d44dda7..e6f75821 100644 --- a/buildstockbatch/workflow_generator/test_workflow_generator.py +++ b/buildstockbatch/workflow_generator/test_workflow_generator.py @@ -1,6 +1,7 @@ from buildstockbatch.workflow_generator.base import WorkflowGeneratorBase from buildstockbatch.workflow_generator.residential import ResidentialDefaultWorkflowGenerator from buildstockbatch.workflow_generator.residential_hpxml import ResidentialHpxmlWorkflowGenerator +from buildstockbatch.workflow_generator.residential_hpxml_hes import ResidentialHpxmlHesWorkflowGenerator from buildstockbatch.workflow_generator.commercial import CommercialDefaultWorkflowGenerator import pytest From a270ff684409db5185e979dde19595057e3c726e Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Thu, 11 Nov 2021 08:51:59 -0700 Subject: [PATCH 03/37] try to pull local Docker image first --- buildstockbatch/localdocker.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/buildstockbatch/localdocker.py b/buildstockbatch/localdocker.py index 9f47f99b..644075a8 100644 --- a/buildstockbatch/localdocker.py +++ b/buildstockbatch/localdocker.py @@ -31,6 +31,7 @@ from buildstockbatch.base import BuildStockBatchBase, SimulationExists from buildstockbatch import postprocessing from .utils import log_error_details, ContainerRuntime +from docker.errors import ImageNotFound logger = logging.getLogger(__name__) @@ -58,7 +59,7 @@ def validate_project(project_file): @property def docker_image(self): - # return 'nrel/hescore-hpxml-openstudio:latest' + # return 'nrel/hescore-hpxml-openstudio' return 'nrel/openstudio:{}'.format(self.os_version) class LocalDockerBatch(DockerBatchBase): @@ -66,7 +67,11 @@ class LocalDockerBatch(DockerBatchBase): def __init__(self, project_filename): super().__init__(project_filename) logger.debug(f'Pulling docker image: {self.docker_image}') - self.docker_client.images.pull(self.docker_image) + + try: + self.docker_client.images.get(self.docker_image) + except ImageNotFound: + self.docker_client.images.pull(self.docker_image) # Create simulation_output dir sim_out_ts_dir = os.path.join(self.results_dir, 'simulation_output', 'timeseries') From d8cd3069452668738eaa67c891cb481562a15486 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Thu, 11 Nov 2021 08:53:14 -0700 Subject: [PATCH 04/37] update yml field name; some updates to HES workflow generator --- buildstockbatch/base.py | 6 ++--- buildstockbatch/utils.py | 6 ++--- .../residential_hpxml_hes.py | 24 ++++++++++--------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/buildstockbatch/base.py b/buildstockbatch/base.py index 6cbd0973..b9193c39 100644 --- a/buildstockbatch/base.py +++ b/buildstockbatch/base.py @@ -63,11 +63,11 @@ def __init__(self, project_filename): self.project_dir = os.path.join(self.buildstock_dir, self.cfg['project_directory']) if not os.path.isdir(self.project_dir): raise FileNotFoundError(f'project_directory = {self.project_dir} is not a directory.') - if 'hescore_hpxml' in self.cfg['workflow_generator']['args']: - self.os_hescore_dir = self.cfg['workflow_generator']['args']['hescore_hpxml']['os_hescore_directory'] + if 'openstudio_hescore' in self.cfg['workflow_generator']['args']: + self.os_hescore_dir = self.cfg['workflow_generator']['args']['openstudio_hescore']['os_hescore_directory'] else: self.os_hescore_dir = None - + # Load in OS_VERSION and OS_SHA arguments if they exist in the YAML, # otherwise use defaults specified here. self.os_version = self.cfg.get('os_version', self.DEFAULT_OS_VERSION) diff --git a/buildstockbatch/utils.py b/buildstockbatch/utils.py index 502db352..e6f071a7 100644 --- a/buildstockbatch/utils.py +++ b/buildstockbatch/utils.py @@ -30,9 +30,9 @@ def get_project_configuration(project_file): # Set absolute paths cfg['buildstock_directory'] = path_rel_to_file(project_file, cfg['buildstock_directory']) - if 'hescore_hpxml' in cfg['workflow_generator']['args']: - cfg['workflow_generator']['args']['hescore_hpxml']['os_hescore_directory'] = \ - path_rel_to_file('', cfg['workflow_generator']['args']['hescore_hpxml']['os_hescore_directory']) + if 'openstudio_hescore' in cfg['workflow_generator']['args']: + cfg['workflow_generator']['args']['openstudio_hescore']['os_hescore_directory'] = \ + path_rel_to_file('', cfg['workflow_generator']['args']['openstudio_hescore']['os_hescore_directory']) # if 'precomputed_sample' in cfg.get('baseline', {}): # cfg['baseline']['precomputed_sample'] = \ diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index 6a0d4765..3751b3ec 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -34,7 +34,7 @@ def validate(cls, cfg): schema_yml = """ measures_to_ignore: list(str(), required=False) build_existing_model: map(required=False) - hescore_hpxml: map(required=True) + openstudio_hescore: map(required=True) reporting_measures: list(include('measure-spec'), required=False) simulation_output_report: map(required=False) server_directory_cleanup: map(required=False) @@ -66,7 +66,7 @@ def create_osw(self, sim_id, building_id, upgrade_idx): # Default argument values workflow_args = { 'build_existing_model': {}, - 'hescore_hpxml': {}, + 'openstudio_hescore': {}, 'measures': [], 'simulation_output_report': {}, 'server_directory_cleanup': {} @@ -95,8 +95,16 @@ def create_osw(self, sim_id, building_id, upgrade_idx): bld_exist_model_args.update(sim_ctl_args) bld_exist_model_args.update(workflow_args['build_existing_model']) - os_to_hpxml_args = { - 'hpxml_path': 'fixme', + hes_ruleset_args = { + 'json_path': '/var/simdata/openstudio/run/testingout.json', + 'hpxml_output_path': 'fixme' + } + + if not os.path.exists('testing'): + os.makedirs(os.path.join('testing')) + + hpxml_to_os_args = { + 'hpxml_path': 'in.xml', 'output_dir': 'fixme', 'debug': 'fixme', 'add_component_loads': 'fixme', @@ -104,11 +112,6 @@ def create_osw(self, sim_id, building_id, upgrade_idx): 'building_id': 'fixme' } - hes_ruleset_args = { - 'json_path': 'fixme', - 'hpxml_output_path': 'fixme' - } - ## FIXME: update for OS-HEScore sim_out_rep_args = { 'timeseries_frequency': 'none', @@ -132,7 +135,6 @@ def create_osw(self, sim_id, building_id, upgrade_idx): }, { 'measure_dir_name': 'HEScoreHPXML', #(resstock) - 'arguments': workflow_args['hescore_hpxml'] }, { 'measure_dir_name': 'HEScoreRuleset', #(OS-HEScore) @@ -140,7 +142,7 @@ def create_osw(self, sim_id, building_id, upgrade_idx): }, { 'measure_dir_name': 'HPXMLtoOpenStudio', #(OS-HEScore) - 'arguments': os_to_hpxml_args + 'arguments': hpxml_to_os_args } ], 'created_at': dt.datetime.now().isoformat(), From 7483a12a788ac230bf8922cd363fa78274fa923c Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Fri, 19 Nov 2021 12:13:54 -0700 Subject: [PATCH 05/37] fix a couple of osw arguments; temporary changes for testing --- buildstockbatch/localdocker.py | 19 ++++++++++--------- .../residential_hpxml_hes.py | 6 +++--- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/buildstockbatch/localdocker.py b/buildstockbatch/localdocker.py index 644075a8..0aa1c157 100644 --- a/buildstockbatch/localdocker.py +++ b/buildstockbatch/localdocker.py @@ -59,8 +59,8 @@ def validate_project(project_file): @property def docker_image(self): - # return 'nrel/hescore-hpxml-openstudio' - return 'nrel/openstudio:{}'.format(self.os_version) + #FIXME temporary docker image for testing + return 'nrel/hescore-hpxml-openstudio' class LocalDockerBatch(DockerBatchBase): @@ -211,13 +211,14 @@ def run_batch(self, n_jobs=None, measures_only=False, sampling_only=False): json.dump(dpouts, f) del dpouts - sim_out_tarfile_name = os.path.join(sim_out_dir, 'simulations_job0.tar.gz') - logger.debug(f'Compressing simulation outputs to {sim_out_tarfile_name}') - with tarfile.open(sim_out_tarfile_name, 'w:gz') as tarf: - for dirname in os.listdir(sim_out_dir): - if re.match(r'up\d+', dirname) and os.path.isdir(os.path.join(sim_out_dir, dirname)): - tarf.add(os.path.join(sim_out_dir, dirname), arcname=dirname) - shutil.rmtree(os.path.join(sim_out_dir, dirname)) + #FIXME temporarily comment out for testing + # sim_out_tarfile_name = os.path.join(sim_out_dir, 'simulations_job0.tar.gz') + # logger.debug(f'Compressing simulation outputs to {sim_out_tarfile_name}') + # with tarfile.open(sim_out_tarfile_name, 'w:gz') as tarf: + # for dirname in os.listdir(sim_out_dir): + # if re.match(r'up\d+', dirname) and os.path.isdir(os.path.join(sim_out_dir, dirname)): + # tarf.add(os.path.join(sim_out_dir, dirname), arcname=dirname) + # shutil.rmtree(os.path.join(sim_out_dir, dirname)) @property def output_dir(self): diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index 3751b3ec..c3124606 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -97,14 +97,14 @@ def create_osw(self, sim_id, building_id, upgrade_idx): hes_ruleset_args = { 'json_path': '/var/simdata/openstudio/run/testingout.json', - 'hpxml_output_path': 'fixme' + 'hpxml_output_path': '/var/simdata/openstudio/run/hes.xml' } if not os.path.exists('testing'): os.makedirs(os.path.join('testing')) hpxml_to_os_args = { - 'hpxml_path': 'in.xml', + 'hpxml_path': '/var/simdata/openstudio/run/hes.xml', 'output_dir': 'fixme', 'debug': 'fixme', 'add_component_loads': 'fixme', @@ -184,7 +184,7 @@ def create_osw(self, sim_id, building_id, upgrade_idx): osw['steps'].extend([ { - 'measure_dir_name': 'SimulationOutputReport', #(OS-HEScore) + 'measure_dir_name': 'ReportSimulationOutput', #(OS-HEScore) 'arguments': sim_out_rep_args }, { From a748763d52847ebc6a81d438eb4a5ed00f89fedb Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Tue, 30 Nov 2021 15:11:53 -0700 Subject: [PATCH 06/37] mount OS-HEScore weather directory; update hardcoded workflow generator args --- buildstockbatch/localdocker.py | 3 ++- .../workflow_generator/residential_hpxml_hes.py | 9 ++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/buildstockbatch/localdocker.py b/buildstockbatch/localdocker.py index 0aa1c157..e5202626 100644 --- a/buildstockbatch/localdocker.py +++ b/buildstockbatch/localdocker.py @@ -117,7 +117,8 @@ def run_building(cls, project_dir, buildstock_dir, weather_dir, os_hescore_dir, # OS-HEScore measure directories if os_hescore_dir and os.path.exists(os_hescore_dir): bind_mounts += [(os.path.join(os_hescore_dir, 'rulesets'), 'OpenStudio-HEScore/rulesets', 'ro'), - (os.path.join(os_hescore_dir, 'hpxml-measures'), 'OpenStudio-HEScore/hpxml-measures', 'ro')] + (os.path.join(os_hescore_dir, 'hpxml-measures'), 'OpenStudio-HEScore/hpxml-measures', 'ro'), + (os.path.join(os_hescore_dir, 'weather'), 'OpenStudio-HEScore/weather', 'ro')] docker_volume_mounts = dict([(key, {'bind': f'/var/simdata/openstudio/{bind}', 'mode': mode}) for key, bind, mode in bind_mounts]) # noqa E501 for bind in bind_mounts: diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index c3124606..de8e77fb 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -105,11 +105,10 @@ def create_osw(self, sim_id, building_id, upgrade_idx): hpxml_to_os_args = { 'hpxml_path': '/var/simdata/openstudio/run/hes.xml', - 'output_dir': 'fixme', - 'debug': 'fixme', - 'add_component_loads': 'fixme', - 'skip_validation': 'fixme', - 'building_id': 'fixme' + 'output_dir': '/var/simdata/openstudio/run', + 'debug': 'false', + 'add_component_loads': 'false', + 'skip_validation': 'false' } ## FIXME: update for OS-HEScore From 570d5cd95702516f474725c3db8b96e17b05b16e Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Wed, 8 Dec 2021 15:31:29 -0700 Subject: [PATCH 07/37] new argument passed to BuildExistingModel --- buildstockbatch/workflow_generator/residential_hpxml_hes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index de8e77fb..c0e5cfe5 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -88,7 +88,8 @@ def create_osw(self, sim_id, building_id, upgrade_idx): bld_exist_model_args = { 'building_id': building_id, - 'sample_weight': self.n_datapoints / self.cfg['baseline']['n_buildings_represented'] + 'sample_weight': self.n_datapoints / self.cfg['baseline']['n_buildings_represented'], + 'build_hpxml_only': True } if 'measures_to_ignore' in workflow_args: bld_exist_model_args['measures_to_ignore'] = '|'.join(workflow_args['measures_to_ignore']) @@ -96,7 +97,7 @@ def create_osw(self, sim_id, building_id, upgrade_idx): bld_exist_model_args.update(workflow_args['build_existing_model']) hes_ruleset_args = { - 'json_path': '/var/simdata/openstudio/run/testingout.json', + 'json_path': '/var/simdata/openstudio/run/hes.json', 'hpxml_output_path': '/var/simdata/openstudio/run/hes.xml' } @@ -111,7 +112,6 @@ def create_osw(self, sim_id, building_id, upgrade_idx): 'skip_validation': 'false' } - ## FIXME: update for OS-HEScore sim_out_rep_args = { 'timeseries_frequency': 'none', 'include_timeseries_fuel_consumptions': False, From 633ae54e7c1c4ae7a5fa1eadf2b247ae429bf412 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Fri, 10 Dec 2021 13:07:23 -0700 Subject: [PATCH 08/37] style fixes --- buildstockbatch/aws/awsbase.py | 6 +- buildstockbatch/base.py | 2 +- buildstockbatch/eagle.py | 62 +++++++-------- buildstockbatch/localdocker.py | 15 ++-- buildstockbatch/sampler/sobol_lib.py | 76 +++++++++---------- buildstockbatch/test/conftest.py | 2 +- buildstockbatch/test/test_base.py | 30 ++++---- buildstockbatch/test/test_utils.py | 6 +- .../workflow_generator/commercial.py | 4 +- .../workflow_generator/residential.py | 42 +++++----- .../workflow_generator/residential_hpxml.py | 38 +++++----- .../residential_hpxml_hes.py | 50 ++++++------ 12 files changed, 167 insertions(+), 166 deletions(-) diff --git a/buildstockbatch/aws/awsbase.py b/buildstockbatch/aws/awsbase.py index 8884e761..6c21d1fe 100644 --- a/buildstockbatch/aws/awsbase.py +++ b/buildstockbatch/aws/awsbase.py @@ -104,9 +104,9 @@ def delete_role(self, role_name): for policy in response['AttachedPolicies']: self.iam.detach_role_policy( - RoleName=role_name, - PolicyArn=policy['PolicyArn'] - ) + RoleName=role_name, + PolicyArn=policy['PolicyArn'] + ) logger.info(f'Policies detached from role {role_name}.') diff --git a/buildstockbatch/base.py b/buildstockbatch/base.py index b9193c39..30e2efa4 100644 --- a/buildstockbatch/base.py +++ b/buildstockbatch/base.py @@ -67,7 +67,7 @@ def __init__(self, project_filename): self.os_hescore_dir = self.cfg['workflow_generator']['args']['openstudio_hescore']['os_hescore_directory'] else: self.os_hescore_dir = None - + # Load in OS_VERSION and OS_SHA arguments if they exist in the YAML, # otherwise use defaults specified here. self.os_version = self.cfg.get('os_version', self.DEFAULT_OS_VERSION) diff --git a/buildstockbatch/eagle.py b/buildstockbatch/eagle.py index 02b36b72..ff2d3aaf 100644 --- a/buildstockbatch/eagle.py +++ b/buildstockbatch/eagle.py @@ -97,9 +97,9 @@ def clear_and_copy_dir(src, dst): @property def singularity_image_url(self): return 'https://s3.amazonaws.com/openstudio-builds/{ver}/OpenStudio-{ver}.{sha}-Singularity.simg'.format( - ver=self.os_version, - sha=self.os_sha - ) + ver=self.os_version, + sha=self.os_sha + ) @property def singularity_image(self): @@ -549,35 +549,35 @@ def get_dask_client(self): logging_config = { - 'version': 1, - 'disable_existing_loggers': True, - 'formatters': { - 'defaultfmt': { - 'format': '%(levelname)s:%(asctime)s:%(name)s:%(message)s', - 'datefmt': '%Y-%m-%d %H:%M:%S' - } - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'formatter': 'defaultfmt', - 'level': 'DEBUG', - 'stream': 'ext://sys.stdout', - } - }, - 'loggers': { - '__main__': { - 'level': 'DEBUG', - 'propagate': True, - 'handlers': ['console'] - }, - 'buildstockbatch': { - 'level': 'DEBUG', - 'propagate': True, - 'handlers': ['console'] - } + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'defaultfmt': { + 'format': '%(levelname)s:%(asctime)s:%(name)s:%(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S' + } + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'defaultfmt', + 'level': 'DEBUG', + 'stream': 'ext://sys.stdout', + } + }, + 'loggers': { + '__main__': { + 'level': 'DEBUG', + 'propagate': True, + 'handlers': ['console'] }, - } + 'buildstockbatch': { + 'level': 'DEBUG', + 'propagate': True, + 'handlers': ['console'] + } + }, +} def user_cli(argv=sys.argv[1:]): diff --git a/buildstockbatch/localdocker.py b/buildstockbatch/localdocker.py index e5202626..3f4a4aa1 100644 --- a/buildstockbatch/localdocker.py +++ b/buildstockbatch/localdocker.py @@ -59,15 +59,16 @@ def validate_project(project_file): @property def docker_image(self): - #FIXME temporary docker image for testing + # FIXME temporary docker image for testing return 'nrel/hescore-hpxml-openstudio' + class LocalDockerBatch(DockerBatchBase): def __init__(self, project_filename): super().__init__(project_filename) logger.debug(f'Pulling docker image: {self.docker_image}') - + try: self.docker_client.images.get(self.docker_image) except ImageNotFound: @@ -92,8 +93,8 @@ def weather_dir(self): return self._weather_dir.name @classmethod - def run_building(cls, project_dir, buildstock_dir, weather_dir, os_hescore_dir, docker_image, results_dir, measures_only, - n_datapoints, cfg, i, upgrade_idx=None): + def run_building(cls, project_dir, buildstock_dir, weather_dir, os_hescore_dir, docker_image, results_dir, + measures_only, n_datapoints, cfg, i, upgrade_idx=None): upgrade_id = 0 if upgrade_idx is None else upgrade_idx + 1 @@ -112,8 +113,8 @@ def run_building(cls, project_dir, buildstock_dir, weather_dir, os_hescore_dir, # ResStock-hpxml measure directory if os.path.exists(os.path.join(buildstock_dir, 'resources', 'hpxml-measures')): bind_mounts += [(os.path.join(buildstock_dir, 'resources', 'hpxml-measures'), - 'resources/hpxml-measures', 'ro')] - + 'resources/hpxml-measures', 'ro')] + # OS-HEScore measure directories if os_hescore_dir and os.path.exists(os_hescore_dir): bind_mounts += [(os.path.join(os_hescore_dir, 'rulesets'), 'OpenStudio-HEScore/rulesets', 'ro'), @@ -212,7 +213,7 @@ def run_batch(self, n_jobs=None, measures_only=False, sampling_only=False): json.dump(dpouts, f) del dpouts - #FIXME temporarily comment out for testing + # FIXME temporarily comment out for testing # sim_out_tarfile_name = os.path.join(sim_out_dir, 'simulations_job0.tar.gz') # logger.debug(f'Compressing simulation outputs to {sim_out_tarfile_name}') # with tarfile.open(sim_out_tarfile_name, 'w:gz') as tarf: diff --git a/buildstockbatch/sampler/sobol_lib.py b/buildstockbatch/sampler/sobol_lib.py index 8a015c3b..9e6d7041 100644 --- a/buildstockbatch/sampler/sobol_lib.py +++ b/buildstockbatch/sampler/sobol_lib.py @@ -8,7 +8,7 @@ def i4_bit_hi1(n): # *****************************************************************************80 # - ## I4_BIT_HI1 returns the position of the high 1 bit base 2 in an integer. + # I4_BIT_HI1 returns the position of the high 1 bit base 2 in an integer. # # Example: # @@ -69,7 +69,7 @@ def i4_bit_hi1(n): def i4_bit_lo0(n): # *****************************************************************************80 # - ## I4_BIT_LO0 returns the position of the low 0 bit base 2 in an integer. + # I4_BIT_LO0 returns the position of the low 0 bit base 2 in an integer. # # Example: # @@ -132,7 +132,7 @@ def i4_bit_lo0(n): def i4_sobol_generate(m, n, skip): # *****************************************************************************80 # - ## I4_SOBOL_GENERATE generates a Sobol dataset. + # I4_SOBOL_GENERATE generates a Sobol dataset. # # Licensing: # @@ -167,7 +167,7 @@ def i4_sobol_generate(m, n, skip): def i4_sobol(dim_num, seed): # *****************************************************************************80 # - ## I4_SOBOL generates a new quasirandom Sobol vector with each call. + # I4_SOBOL generates a new quasirandom Sobol vector with each call. # # Discussion: # @@ -256,55 +256,55 @@ def i4_sobol(dim_num, seed): # Initialize (part of) V. # v = zeros((dim_max, log_max)) - v[0:40, 0] = transpose([ \ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, \ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, \ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, \ + v[0:40, 0] = transpose([ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]) - v[2:40, 1] = transpose([ \ - 1, 3, 1, 3, 1, 3, 3, 1, \ - 3, 1, 3, 1, 3, 1, 1, 3, 1, 3, \ - 1, 3, 1, 3, 3, 1, 3, 1, 3, 1, \ + v[2:40, 1] = transpose([ + 1, 3, 1, 3, 1, 3, 3, 1, + 3, 1, 3, 1, 3, 1, 1, 3, 1, 3, + 1, 3, 1, 3, 3, 1, 3, 1, 3, 1, 3, 1, 1, 3, 1, 3, 1, 3, 1, 3]) - v[3:40, 2] = transpose([ \ - 7, 5, 1, 3, 3, 7, 5, \ - 5, 7, 7, 1, 3, 3, 7, 5, 1, 1, \ - 5, 3, 3, 1, 7, 5, 1, 3, 3, 7, \ + v[3:40, 2] = transpose([ + 7, 5, 1, 3, 3, 7, 5, + 5, 7, 7, 1, 3, 3, 7, 5, 1, 1, + 5, 3, 3, 1, 7, 5, 1, 3, 3, 7, 5, 1, 1, 5, 7, 7, 5, 1, 3, 3]) - v[5:40, 3] = transpose([ \ - 1, 7, 9, 13, 11, \ - 1, 3, 7, 9, 5, 13, 13, 11, 3, 15, \ - 5, 3, 15, 7, 9, 13, 9, 1, 11, 7, \ + v[5:40, 3] = transpose([ + 1, 7, 9, 13, 11, + 1, 3, 7, 9, 5, 13, 13, 11, 3, 15, + 5, 3, 15, 7, 9, 13, 9, 1, 11, 7, 5, 15, 1, 15, 11, 5, 3, 1, 7, 9]) - v[7:40, 4] = transpose([ \ - 9, 3, 27, \ - 15, 29, 21, 23, 19, 11, 25, 7, 13, 17, \ - 1, 25, 29, 3, 31, 11, 5, 23, 27, 19, \ + v[7:40, 4] = transpose([ + 9, 3, 27, + 15, 29, 21, 23, 19, 11, 25, 7, 13, 17, + 1, 25, 29, 3, 31, 11, 5, 23, 27, 19, 21, 5, 1, 17, 13, 7, 15, 9, 31, 9]) - v[13:40, 5] = transpose([ \ - 37, 33, 7, 5, 11, 39, 63, \ - 27, 17, 15, 23, 29, 3, 21, 13, 31, 25, \ + v[13:40, 5] = transpose([ + 37, 33, 7, 5, 11, 39, 63, + 27, 17, 15, 23, 29, 3, 21, 13, 31, 25, 9, 49, 33, 19, 29, 11, 19, 27, 15, 25]) - v[19:40, 6] = transpose([ \ - 13, \ - 33, 115, 41, 79, 17, 29, 119, 75, 73, 105, \ + v[19:40, 6] = transpose([ + 13, + 33, 115, 41, 79, 17, 29, 119, 75, 73, 105, 7, 59, 65, 21, 3, 113, 61, 89, 45, 107]) - v[37:40, 7] = transpose([ \ + v[37:40, 7] = transpose([ 7, 23, 39]) # # Set POLY. # - poly = [ \ - 1, 3, 7, 11, 13, 19, 25, 37, 59, 47, \ - 61, 55, 41, 67, 97, 91, 109, 103, 115, 131, \ - 193, 137, 145, 143, 241, 157, 185, 167, 229, 171, \ + poly = [ + 1, 3, 7, 11, 13, 19, 25, 37, 59, 47, + 61, 55, 41, 67, 97, 91, 109, 103, 115, 131, + 193, 137, 145, 143, 241, 157, 185, 167, 229, 171, 213, 191, 253, 203, 211, 239, 247, 285, 369, 299] atmost = 2 ** log_max - 1 @@ -444,7 +444,7 @@ def i4_sobol(dim_num, seed): def i4_uniform(a, b, seed): # *****************************************************************************80 # - ## I4_UNIFORM returns a scaled pseudorandom I4. + # I4_UNIFORM returns a scaled pseudorandom I4. # # Discussion: # @@ -539,7 +539,7 @@ def i4_uniform(a, b, seed): def prime_ge(n): # *****************************************************************************80 # - ## PRIME_GE returns the smallest prime greater than or equal to N. + # PRIME_GE returns the smallest prime greater than or equal to N. # # Example: # @@ -587,7 +587,7 @@ def prime_ge(n): def isprime(n): # *****************************************************************************80 # - ## IS_PRIME returns True if N is a prime number, False otherwise + # IS_PRIME returns True if N is a prime number, False otherwise # # Licensing: # diff --git a/buildstockbatch/test/conftest.py b/buildstockbatch/test/conftest.py index db805fcc..7cbd3233 100644 --- a/buildstockbatch/test/conftest.py +++ b/buildstockbatch/test/conftest.py @@ -57,7 +57,7 @@ def _basic_residential_project_file(update_args={}, raw=False): 'options': [ {'option': 'Infiltration|11.25 ACH50'} ] - }], + }], 'eagle': { 'sampling': { 'time': 20 diff --git a/buildstockbatch/test/test_base.py b/buildstockbatch/test/test_base.py index e8e38095..e50e5d11 100644 --- a/buildstockbatch/test/test_base.py +++ b/buildstockbatch/test/test_base.py @@ -238,21 +238,21 @@ def test_upload_files(mocked_boto3, basic_residential_project_file): region = 'test_region' upload_config = { - 'postprocessing': { - 'aws': { - 'region_name': region, - 's3': { - 'bucket': s3_bucket, - 'prefix': s3_prefix, - }, - 'athena': { - 'glue_service_role': role, - 'database_name': db_name, - 'max_crawling_time': 250 - } - } - } - } + 'postprocessing': { + 'aws': { + 'region_name': region, + 's3': { + 'bucket': s3_bucket, + 'prefix': s3_prefix, + }, + 'athena': { + 'glue_service_role': role, + 'database_name': db_name, + 'max_crawling_time': 250 + } + } + } + } mocked_glueclient = MagicMock() mocked_glueclient.get_crawler = MagicMock(return_value={'Crawler': {'State': 'READY'}}) mocked_boto3.client = MagicMock(return_value=mocked_glueclient) diff --git a/buildstockbatch/test/test_utils.py b/buildstockbatch/test/test_utils.py index 62c5d215..1f98f3dc 100644 --- a/buildstockbatch/test/test_utils.py +++ b/buildstockbatch/test/test_utils.py @@ -6,9 +6,9 @@ def test_str_repr(): test_obj = [{1, 2, 3, 4, 5, 6}, {"List1": ["Item1", ('a', 'b', 'c', 'd'), "item3"], - "long_name_list": ["long_name_one_two_three", "long_name"], - "dict": {"key1": ["List_item1", "List_item2", "List_item3"], "Key2": "value2", - "key3": "value3", "key4": "val4"}}] + "long_name_list": ["long_name_one_two_three", "long_name"], + "dict": {"key1": ["List_item1", "List_item2", "List_item3"], "Key2": "value2", + "key3": "value3", "key4": "val4"}}] gen_repr = _str_repr(test_obj, list_max=2, dict_max=3, string_max=10) true_repr = "[{'1','2','3' ...6},{'List1': ['Item1',('a','b' ...4) ...3],'long_...14..._list': ['long_...23..."\ diff --git a/buildstockbatch/workflow_generator/commercial.py b/buildstockbatch/workflow_generator/commercial.py index 5d0ab739..ce7ecb65 100644 --- a/buildstockbatch/workflow_generator/commercial.py +++ b/buildstockbatch/workflow_generator/commercial.py @@ -187,11 +187,11 @@ def get_cfg_path(cfg_path): if actual_argument_value not in expected_argument_type: error_msgs += f"* Found unexpected argument value {actual_argument_value} for "\ f"{measure_names[measure_name]} in yaml file. Valid values are " \ - f"{expected_argument_type}.\n" + f"{expected_argument_type}.\n" for arg, default in required_args_no_default.items(): error_msgs += f"* Required argument {arg} for measure {measure_name} wasn't supplied. " \ - f"There is no default for this argument.\n" + f"There is no default for this argument.\n" for arg, default in required_args_with_default.items(): warning_msgs += f"* Required argument {arg} for measure {measure_name} wasn't supplied. " \ diff --git a/buildstockbatch/workflow_generator/residential.py b/buildstockbatch/workflow_generator/residential.py index c768f54a..847d3a13 100644 --- a/buildstockbatch/workflow_generator/residential.py +++ b/buildstockbatch/workflow_generator/residential.py @@ -180,11 +180,11 @@ def get_cfg_path(cfg_path): if actual_argument_value not in expected_argument_type: error_msgs += f"* Found unexpected argument value {actual_argument_value} for "\ f"{measure_names[measure_name]} in yaml file. Valid values are " \ - f"{expected_argument_type}.\n" + f"{expected_argument_type}.\n" for arg, default in required_args_no_default.items(): error_msgs += f"* Required argument {arg} for measure {measure_name} wasn't supplied. " \ - f"There is no default for this argument.\n" + f"There is no default for this argument.\n" for arg, default in required_args_with_default.items(): warning_msgs += f"* Required argument {arg} for measure {measure_name} wasn't supplied. " \ @@ -284,25 +284,25 @@ def create_osw(self, sim_id, building_id, upgrade_idx): osw['steps'].extend(workflow_args['measures']) server_dir_cleanup_args = { - 'retain_in_osm': False, - 'retain_in_idf': True, - 'retain_pre_process_idf': False, - 'retain_eplusout_audit': False, - 'retain_eplusout_bnd': False, - 'retain_eplusout_eio': False, - 'retain_eplusout_end': False, - 'retain_eplusout_err': False, - 'retain_eplusout_eso': False, - 'retain_eplusout_mdd': False, - 'retain_eplusout_mtd': False, - 'retain_eplusout_rdd': False, - 'retain_eplusout_shd': False, - 'retain_eplusout_sql': False, - 'retain_eplustbl_htm': False, - 'retain_sqlite_err': False, - 'retain_stdout_energyplus': False, - 'retain_stdout_expandobject': False, - 'retain_schedules_csv': True + 'retain_in_osm': False, + 'retain_in_idf': True, + 'retain_pre_process_idf': False, + 'retain_eplusout_audit': False, + 'retain_eplusout_bnd': False, + 'retain_eplusout_eio': False, + 'retain_eplusout_end': False, + 'retain_eplusout_err': False, + 'retain_eplusout_eso': False, + 'retain_eplusout_mdd': False, + 'retain_eplusout_mtd': False, + 'retain_eplusout_rdd': False, + 'retain_eplusout_shd': False, + 'retain_eplusout_sql': False, + 'retain_eplustbl_htm': False, + 'retain_sqlite_err': False, + 'retain_stdout_energyplus': False, + 'retain_stdout_expandobject': False, + 'retain_schedules_csv': True } server_dir_cleanup_args.update(workflow_args['server_directory_cleanup']) diff --git a/buildstockbatch/workflow_generator/residential_hpxml.py b/buildstockbatch/workflow_generator/residential_hpxml.py index ff34cfa4..c6a2146e 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml.py +++ b/buildstockbatch/workflow_generator/residential_hpxml.py @@ -126,25 +126,25 @@ def create_osw(self, sim_id, building_id, upgrade_idx): osw['steps'].extend(workflow_args['measures']) server_dir_cleanup_args = { - 'retain_in_osm': False, - 'retain_in_idf': True, - 'retain_pre_process_idf': False, - 'retain_eplusout_audit': False, - 'retain_eplusout_bnd': False, - 'retain_eplusout_eio': False, - 'retain_eplusout_end': False, - 'retain_eplusout_err': False, - 'retain_eplusout_eso': False, - 'retain_eplusout_mdd': False, - 'retain_eplusout_mtd': False, - 'retain_eplusout_rdd': False, - 'retain_eplusout_shd': False, - 'retain_eplusout_sql': False, - 'retain_eplustbl_htm': False, - 'retain_sqlite_err': False, - 'retain_stdout_energyplus': False, - 'retain_stdout_expandobject': False, - 'retain_schedules_csv': True + 'retain_in_osm': False, + 'retain_in_idf': True, + 'retain_pre_process_idf': False, + 'retain_eplusout_audit': False, + 'retain_eplusout_bnd': False, + 'retain_eplusout_eio': False, + 'retain_eplusout_end': False, + 'retain_eplusout_err': False, + 'retain_eplusout_eso': False, + 'retain_eplusout_mdd': False, + 'retain_eplusout_mtd': False, + 'retain_eplusout_rdd': False, + 'retain_eplusout_shd': False, + 'retain_eplusout_sql': False, + 'retain_eplustbl_htm': False, + 'retain_sqlite_err': False, + 'retain_stdout_energyplus': False, + 'retain_stdout_expandobject': False, + 'retain_schedules_csv': True } server_dir_cleanup_args.update(workflow_args['server_directory_cleanup']) diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index c0e5cfe5..7dc4a2d8 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -100,7 +100,7 @@ def create_osw(self, sim_id, building_id, upgrade_idx): 'json_path': '/var/simdata/openstudio/run/hes.json', 'hpxml_output_path': '/var/simdata/openstudio/run/hes.xml' } - + if not os.path.exists('testing'): os.makedirs(os.path.join('testing')) @@ -129,18 +129,18 @@ def create_osw(self, sim_id, building_id, upgrade_idx): 'id': sim_id, 'steps': [ { - 'measure_dir_name': 'BuildExistingModel', #(resstock) + 'measure_dir_name': 'BuildExistingModel', # (resstock) 'arguments': bld_exist_model_args }, { - 'measure_dir_name': 'HEScoreHPXML', #(resstock) + 'measure_dir_name': 'HEScoreHPXML', # (resstock) }, { - 'measure_dir_name': 'HEScoreRuleset', #(OS-HEScore) + 'measure_dir_name': 'HEScoreRuleset', # (OS-HEScore) 'arguments': hes_ruleset_args }, { - 'measure_dir_name': 'HPXMLtoOpenStudio', #(OS-HEScore) + 'measure_dir_name': 'HPXMLtoOpenStudio', # (OS-HEScore) 'arguments': hpxml_to_os_args } ], @@ -159,31 +159,31 @@ def create_osw(self, sim_id, building_id, upgrade_idx): osw['steps'].extend(workflow_args['measures']) server_dir_cleanup_args = { - 'retain_in_osm': False, - 'retain_in_idf': True, - 'retain_pre_process_idf': False, - 'retain_eplusout_audit': False, - 'retain_eplusout_bnd': False, - 'retain_eplusout_eio': False, - 'retain_eplusout_end': False, - 'retain_eplusout_err': False, - 'retain_eplusout_eso': False, - 'retain_eplusout_mdd': False, - 'retain_eplusout_mtd': False, - 'retain_eplusout_rdd': False, - 'retain_eplusout_shd': False, - 'retain_eplusout_sql': False, - 'retain_eplustbl_htm': False, - 'retain_sqlite_err': False, - 'retain_stdout_energyplus': False, - 'retain_stdout_expandobject': False, - 'retain_schedules_csv': True + 'retain_in_osm': False, + 'retain_in_idf': True, + 'retain_pre_process_idf': False, + 'retain_eplusout_audit': False, + 'retain_eplusout_bnd': False, + 'retain_eplusout_eio': False, + 'retain_eplusout_end': False, + 'retain_eplusout_err': False, + 'retain_eplusout_eso': False, + 'retain_eplusout_mdd': False, + 'retain_eplusout_mtd': False, + 'retain_eplusout_rdd': False, + 'retain_eplusout_shd': False, + 'retain_eplusout_sql': False, + 'retain_eplustbl_htm': False, + 'retain_sqlite_err': False, + 'retain_stdout_energyplus': False, + 'retain_stdout_expandobject': False, + 'retain_schedules_csv': True } server_dir_cleanup_args.update(workflow_args['server_directory_cleanup']) osw['steps'].extend([ { - 'measure_dir_name': 'ReportSimulationOutput', #(OS-HEScore) + 'measure_dir_name': 'ReportSimulationOutput', # (OS-HEScore) 'arguments': sim_out_rep_args }, { From bd2ba325be5c2c83cac0c20e4f92559dc30febb0 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Fri, 10 Dec 2021 15:32:57 -0700 Subject: [PATCH 09/37] fix check for OS_HEScore directory path --- buildstockbatch/base.py | 4 ++-- buildstockbatch/utils.py | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/buildstockbatch/base.py b/buildstockbatch/base.py index 30e2efa4..7da03fcd 100644 --- a/buildstockbatch/base.py +++ b/buildstockbatch/base.py @@ -63,8 +63,8 @@ def __init__(self, project_filename): self.project_dir = os.path.join(self.buildstock_dir, self.cfg['project_directory']) if not os.path.isdir(self.project_dir): raise FileNotFoundError(f'project_directory = {self.project_dir} is not a directory.') - if 'openstudio_hescore' in self.cfg['workflow_generator']['args']: - self.os_hescore_dir = self.cfg['workflow_generator']['args']['openstudio_hescore']['os_hescore_directory'] + if self.cfg['workflow_generator']['type'] == 'residential_hpxml_hes': + self.os_hescore_dir = path_rel_to_file('',self.cfg['workflow_generator']['args']['openstudio_hescore']['os_hescore_directory']) else: self.os_hescore_dir = None diff --git a/buildstockbatch/utils.py b/buildstockbatch/utils.py index e6f071a7..af6592a7 100644 --- a/buildstockbatch/utils.py +++ b/buildstockbatch/utils.py @@ -30,10 +30,6 @@ def get_project_configuration(project_file): # Set absolute paths cfg['buildstock_directory'] = path_rel_to_file(project_file, cfg['buildstock_directory']) - if 'openstudio_hescore' in cfg['workflow_generator']['args']: - cfg['workflow_generator']['args']['openstudio_hescore']['os_hescore_directory'] = \ - path_rel_to_file('', cfg['workflow_generator']['args']['openstudio_hescore']['os_hescore_directory']) - # if 'precomputed_sample' in cfg.get('baseline', {}): # cfg['baseline']['precomputed_sample'] = \ # path_rel_to_file(project_file, cfg['baseline']['precomputed_sample']) From ced0abc76d498a29751fabcad4b231e552db8715 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Fri, 10 Dec 2021 16:20:08 -0700 Subject: [PATCH 10/37] output timeseries for hes workflow generator --- buildstockbatch/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildstockbatch/base.py b/buildstockbatch/base.py index 1659e8d2..07a07ce2 100644 --- a/buildstockbatch/base.py +++ b/buildstockbatch/base.py @@ -562,7 +562,7 @@ def get_dask_client(self): def process_results(self, skip_combine=False, force_upload=False): self.get_dask_client() # noqa: F841 - if self.cfg['workflow_generator']['type'] == 'residential_hpxml': + if 'residential_hpxml' in self.cfg['workflow_generator']['type']: if 'simulation_output_report' in self.cfg['workflow_generator']['args'].keys(): if 'timeseries_frequency' in self.cfg['workflow_generator']['args']['simulation_output_report'].keys(): do_timeseries = \ From 0b5324e576ebe7366e6674ce065f3c6cabeeb25b Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Fri, 10 Dec 2021 16:20:23 -0700 Subject: [PATCH 11/37] change opestudio docker image --- buildstockbatch/localdocker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildstockbatch/localdocker.py b/buildstockbatch/localdocker.py index 3f4a4aa1..1363e48d 100644 --- a/buildstockbatch/localdocker.py +++ b/buildstockbatch/localdocker.py @@ -60,7 +60,8 @@ def validate_project(project_file): @property def docker_image(self): # FIXME temporary docker image for testing - return 'nrel/hescore-hpxml-openstudio' + # return 'nrel/hescore-hpxml-openstudio' + return 'nrel/openstudio:{}'.format(self.os_version) class LocalDockerBatch(DockerBatchBase): From 553d5feccb5333e60187a7aa24258d2b5be982e7 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Wed, 15 Dec 2021 12:24:51 -0700 Subject: [PATCH 12/37] apply defaults in BuildExistingModel --- buildstockbatch/localdocker.py | 4 ++-- buildstockbatch/workflow_generator/residential_hpxml_hes.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/buildstockbatch/localdocker.py b/buildstockbatch/localdocker.py index 1363e48d..5e55b498 100644 --- a/buildstockbatch/localdocker.py +++ b/buildstockbatch/localdocker.py @@ -60,8 +60,8 @@ def validate_project(project_file): @property def docker_image(self): # FIXME temporary docker image for testing - # return 'nrel/hescore-hpxml-openstudio' - return 'nrel/openstudio:{}'.format(self.os_version) + return 'nrel/hescore-hpxml-openstudio' + # return 'nrel/openstudio:{}'.format(self.os_version) class LocalDockerBatch(DockerBatchBase): diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index 7dc4a2d8..09d33ae9 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -95,6 +95,7 @@ def create_osw(self, sim_id, building_id, upgrade_idx): bld_exist_model_args['measures_to_ignore'] = '|'.join(workflow_args['measures_to_ignore']) bld_exist_model_args.update(sim_ctl_args) bld_exist_model_args.update(workflow_args['build_existing_model']) + bld_exist_model_args.update({'apply_defaults':True}) hes_ruleset_args = { 'json_path': '/var/simdata/openstudio/run/hes.json', From 20317eac4340af38b411078a09c0e89d5b5c750d Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Tue, 11 Jan 2022 10:26:50 -0700 Subject: [PATCH 13/37] updated HEScoreHPXML arguments --- .../workflow_generator/residential_hpxml_hes.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index 09d33ae9..1c27376f 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -95,11 +95,16 @@ def create_osw(self, sim_id, building_id, upgrade_idx): bld_exist_model_args['measures_to_ignore'] = '|'.join(workflow_args['measures_to_ignore']) bld_exist_model_args.update(sim_ctl_args) bld_exist_model_args.update(workflow_args['build_existing_model']) - bld_exist_model_args.update({'apply_defaults':True}) + bld_exist_model_args.update({'apply_hpxml_defaults':True}) + + hes_to_hpxml_args = { + 'hpxml_path': '../existing.xml', + 'output_path': '../hes.json' + } hes_ruleset_args = { - 'json_path': '/var/simdata/openstudio/run/hes.json', - 'hpxml_output_path': '/var/simdata/openstudio/run/hes.xml' + 'json_path': '/var/simdata/openstudio/run/hes.json', ## FIXME - generic path + 'hpxml_output_path': '/var/simdata/openstudio/run/hes.xml' ## FIXME - generic path } if not os.path.exists('testing'): @@ -135,6 +140,7 @@ def create_osw(self, sim_id, building_id, upgrade_idx): }, { 'measure_dir_name': 'HEScoreHPXML', # (resstock) + 'arguments': hes_to_hpxml_args }, { 'measure_dir_name': 'HEScoreRuleset', # (OS-HEScore) From ed499b4218204439c816ceffdb5d0322c7f58552 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Tue, 11 Jan 2022 16:02:18 -0700 Subject: [PATCH 14/37] unit test for hes workflow generator --- .../test_workflow_generator.py | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/buildstockbatch/workflow_generator/test_workflow_generator.py b/buildstockbatch/workflow_generator/test_workflow_generator.py index e6f75821..06959567 100644 --- a/buildstockbatch/workflow_generator/test_workflow_generator.py +++ b/buildstockbatch/workflow_generator/test_workflow_generator.py @@ -389,6 +389,102 @@ def test_residential_hpxml(mocker): assert(server_dir_cleanup_step['measure_dir_name'] == 'ServerDirectoryCleanup') +def test_residential_hpxml_hes(mocker): + sim_id = 'bldb1up1' + building_id = 1 + upgrade_idx = 0 + cfg = { + 'baseline': { + 'n_buildings_represented': 100 + }, + 'workflow_generator': { + 'type': 'residential_hpxml', + 'args': { + 'build_existing_model': { + 'simulation_control_run_period_begin_month': 2, + 'simulation_control_run_period_begin_day_of_month': 1, + 'simulation_control_run_period_end_month': 2, + 'simulation_control_run_period_end_day_of_month': 28, + 'simulation_control_run_period_calendar_year': 2010, + 'apply_hpxml_defaults': True, + 'build_hpxml_only': True, + }, + + 'openstudio_hescore': { + 'os_hescore_directory': '../OpenStudio-HEScore' + }, + + 'simulation_output_report': { + 'timeseries_frequency': 'hourly', + 'include_timeseries_end_use_consumptions': True, + 'include_timeseries_total_loads': True, + 'include_timeseries_zone_temperatures': False, + } + } + }, + 'upgrades': [ + { + 'options': [ + { + 'option': 'Parameter|Option', + } + ], + } + ] + } + n_datapoints = 10 + osw_gen = ResidentialHpxmlHesWorkflowGenerator(cfg, n_datapoints) + osw = osw_gen.create_osw(sim_id, building_id, upgrade_idx) + + steps = osw['steps'] + assert(len(steps) == 9) + + build_existing_model_step = steps[0] + assert(build_existing_model_step['measure_dir_name'] == 'BuildExistingModel') + assert(build_existing_model_step['arguments']['simulation_control_run_period_begin_month'] == 2) + assert(build_existing_model_step['arguments']['simulation_control_run_period_begin_day_of_month'] == 1) + assert(build_existing_model_step['arguments']['simulation_control_run_period_end_month'] == 2) + assert(build_existing_model_step['arguments']['simulation_control_run_period_end_day_of_month'] == 28) + assert(build_existing_model_step['arguments']['simulation_control_run_period_calendar_year'] == 2010) + + apply_upgrade_step = steps[1] + assert(apply_upgrade_step['measure_dir_name'] == 'ApplyUpgrade') + + hescore_hpxml_step = steps[2] + assert(hescore_hpxml_step['measure_dir_name'] == 'HEScoreHPXML') + assert(hescore_hpxml_step['arguments']['hpxml_path'] == '../existing.xml') + assert(hescore_hpxml_step['arguments']['output_path'] == '../hes.json') + + hescore_ruleset_step = steps[3] + assert(hescore_ruleset_step['measure_dir_name'] == 'HEScoreRuleset') + assert(hescore_ruleset_step['arguments']['json_path'] == '/var/simdata/openstudio/run/hes.json') #FIXME temporary filepath + assert(hescore_ruleset_step['arguments']['hpxml_output_path'] == '/var/simdata/openstudio/run/hes.xml') #FIXME temporary filepath + + hpxml_to_os_step = steps[4] + assert(hpxml_to_os_step['measure_dir_name'] == 'HPXMLtoOpenStudio') + assert(hpxml_to_os_step['arguments']['hpxml_path'] == '/var/simdata/openstudio/run/hes.xml') #FIXME temporary filepath + assert(hpxml_to_os_step['arguments']['output_dir'] == '/var/simdata/openstudio/run') #FIXME temporary filepath + assert(hpxml_to_os_step['arguments']['debug'] is False) + assert(hpxml_to_os_step['arguments']['add_component_loads'] is False) + assert(hpxml_to_os_step['arguments']['skip_validation'] is False) + + simulation_output_step = steps[5] + assert(simulation_output_step['measure_dir_name'] == 'ReportSimulationOutput') + assert(simulation_output_step['arguments']['timeseries_frequency'] == 'hourly') + assert(simulation_output_step['arguments']['include_timeseries_end_use_consumptions'] is True) + assert(simulation_output_step['arguments']['include_timeseries_total_loads'] is True) + assert(simulation_output_step['arguments']['include_timeseries_zone_temperatures'] is False) + + hpxml_output_step = steps[6] + assert(hpxml_output_step['measure_dir_name'] == 'ReportHPXMLOutput') + + upgrade_costs_step = steps[7] + assert(upgrade_costs_step['measure_dir_name'] == 'UpgradeCosts') + + server_dir_cleanup_step = steps[8] + assert(server_dir_cleanup_step['measure_dir_name'] == 'ServerDirectoryCleanup') + + def test_com_default_workflow_generator(mocker): mocker.patch.object(CommercialDefaultWorkflowGenerator, 'validate_measures_and_arguments', return_value=True) sim_id = 'bldb1up1' From 110742e4dde70073e01f67439294ecf05554d4e2 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Tue, 11 Jan 2022 16:03:09 -0700 Subject: [PATCH 15/37] use bool types in argument dict --- buildstockbatch/workflow_generator/residential_hpxml_hes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index 1c27376f..53c9f6d2 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -113,9 +113,9 @@ def create_osw(self, sim_id, building_id, upgrade_idx): hpxml_to_os_args = { 'hpxml_path': '/var/simdata/openstudio/run/hes.xml', 'output_dir': '/var/simdata/openstudio/run', - 'debug': 'false', - 'add_component_loads': 'false', - 'skip_validation': 'false' + 'debug': False, + 'add_component_loads': False, + 'skip_validation': False } sim_out_rep_args = { From 708b8ca8ae78091998f76f0e7f799a2707ad3d13 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Tue, 25 Jan 2022 15:43:36 -0700 Subject: [PATCH 16/37] inherit residential_hpxml in residential_hpxml_hes workflow generator --- buildstockbatch/base.py | 4 +- .../residential_hpxml_hes.py | 240 +----------------- 2 files changed, 9 insertions(+), 235 deletions(-) diff --git a/buildstockbatch/base.py b/buildstockbatch/base.py index 07a07ce2..4fdde4c0 100644 --- a/buildstockbatch/base.py +++ b/buildstockbatch/base.py @@ -63,8 +63,8 @@ def __init__(self, project_filename): self.project_dir = os.path.join(self.buildstock_dir, self.cfg['project_directory']) if not os.path.isdir(self.project_dir): raise FileNotFoundError(f'project_directory = {self.project_dir} is not a directory.') - if self.cfg['workflow_generator']['type'] == 'residential_hpxml_hes': - self.os_hescore_dir = path_rel_to_file('',self.cfg['workflow_generator']['args']['openstudio_hescore']['os_hescore_directory']) + if 'os_hescore_directory' in self.cfg['workflow_generator']['args']['build_existing_model']: + self.os_hescore_dir = path_rel_to_file('',self.cfg['workflow_generator']['args']['build_existing_model']['os_hescore_directory']) else: self.os_hescore_dir = None diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index 53c9f6d2..a747408f 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -10,239 +10,13 @@ :license: BSD-3 """ -import datetime as dt -import json -import logging -import re -import yamale -import os - -from .base import WorkflowGeneratorBase - -logger = logging.getLogger(__name__) - - -class ResidentialHpxmlHesWorkflowGenerator(WorkflowGeneratorBase): - - @classmethod - def validate(cls, cfg): - """Validate arguments - - :param cfg: project configuration - :type cfg: dict - """ - schema_yml = """ - measures_to_ignore: list(str(), required=False) - build_existing_model: map(required=False) - openstudio_hescore: map(required=True) - reporting_measures: list(include('measure-spec'), required=False) - simulation_output_report: map(required=False) - server_directory_cleanup: map(required=False) - --- - measure-spec: - measure_dir_name: str(required=True) - arguments: map(required=False) - """ - workflow_generator_args = cfg['workflow_generator']['args'] - schema_yml = re.sub(r'^ {8}', '', schema_yml, flags=re.MULTILINE) - schema = yamale.make_schema(content=schema_yml, parser='ruamel') - data = yamale.make_data(content=json.dumps(workflow_generator_args), parser='ruamel') - yamale.validate(schema, data, strict=True) - return True - - def reporting_measures(self): - """Return a list of reporting measures to include in outputs""" - workflow_args = self.cfg['workflow_generator'].get('args', {}) - return [x['measure_dir_name'] for x in workflow_args.get('reporting_measures', [])] +from .residential_hpxml import ResidentialHpxmlWorkflowGenerator # noqa F041 +class ResidentialHpxmlHesWorkflowGenerator(ResidentialHpxmlWorkflowGenerator): + def create_osw(self, sim_id, building_id, upgrade_idx): - """ - Generate and return the osw as a python dict - - :param sim_id: simulation id, looks like 'bldg0000001up01' - :param building_id: integer building id to use from the sampled buildstock.csv - :param upgrade_idx: integer index of the upgrade scenario to apply, None if baseline - """ - # Default argument values - workflow_args = { - 'build_existing_model': {}, - 'openstudio_hescore': {}, - 'measures': [], - 'simulation_output_report': {}, - 'server_directory_cleanup': {} - } - workflow_args.update(self.cfg['workflow_generator'].get('args', {})) - - logger.debug('Generating OSW, sim_id={}'.format(sim_id)) - - sim_ctl_args = { - 'simulation_control_timestep': 60, - 'simulation_control_run_period_begin_month': 1, - 'simulation_control_run_period_begin_day_of_month': 1, - 'simulation_control_run_period_end_month': 12, - 'simulation_control_run_period_end_day_of_month': 31, - 'simulation_control_run_period_calendar_year': 2007, - 'debug': False, - 'add_component_loads': False - } - - bld_exist_model_args = { - 'building_id': building_id, - 'sample_weight': self.n_datapoints / self.cfg['baseline']['n_buildings_represented'], - 'build_hpxml_only': True - } - if 'measures_to_ignore' in workflow_args: - bld_exist_model_args['measures_to_ignore'] = '|'.join(workflow_args['measures_to_ignore']) - bld_exist_model_args.update(sim_ctl_args) - bld_exist_model_args.update(workflow_args['build_existing_model']) - bld_exist_model_args.update({'apply_hpxml_defaults':True}) - - hes_to_hpxml_args = { - 'hpxml_path': '../existing.xml', - 'output_path': '../hes.json' - } - - hes_ruleset_args = { - 'json_path': '/var/simdata/openstudio/run/hes.json', ## FIXME - generic path - 'hpxml_output_path': '/var/simdata/openstudio/run/hes.xml' ## FIXME - generic path - } - - if not os.path.exists('testing'): - os.makedirs(os.path.join('testing')) - - hpxml_to_os_args = { - 'hpxml_path': '/var/simdata/openstudio/run/hes.xml', - 'output_dir': '/var/simdata/openstudio/run', - 'debug': False, - 'add_component_loads': False, - 'skip_validation': False - } - - sim_out_rep_args = { - 'timeseries_frequency': 'none', - 'include_timeseries_fuel_consumptions': False, - 'include_timeseries_end_use_consumptions': False, - 'include_timeseries_hot_water_uses': False, - 'include_timeseries_total_loads': False, - 'include_timeseries_component_loads': False, - 'include_timeseries_zone_temperatures': False, - 'include_timeseries_airflows': False, - 'include_timeseries_weather': False, - } - sim_out_rep_args.update(workflow_args['simulation_output_report']) - - osw = { - 'id': sim_id, - 'steps': [ - { - 'measure_dir_name': 'BuildExistingModel', # (resstock) - 'arguments': bld_exist_model_args - }, - { - 'measure_dir_name': 'HEScoreHPXML', # (resstock) - 'arguments': hes_to_hpxml_args - }, - { - 'measure_dir_name': 'HEScoreRuleset', # (OS-HEScore) - 'arguments': hes_ruleset_args - }, - { - 'measure_dir_name': 'HPXMLtoOpenStudio', # (OS-HEScore) - 'arguments': hpxml_to_os_args - } - ], - 'created_at': dt.datetime.now().isoformat(), - 'measure_paths': [ - 'OpenStudio-HEScore/rulesets', - 'OpenStudio-HEScore/hpxml-measures', - 'measures', - 'resources/hpxml-measures' - ], - 'run_options': { - 'skip_zip_results': True - } - } - - osw['steps'].extend(workflow_args['measures']) - - server_dir_cleanup_args = { - 'retain_in_osm': False, - 'retain_in_idf': True, - 'retain_pre_process_idf': False, - 'retain_eplusout_audit': False, - 'retain_eplusout_bnd': False, - 'retain_eplusout_eio': False, - 'retain_eplusout_end': False, - 'retain_eplusout_err': False, - 'retain_eplusout_eso': False, - 'retain_eplusout_mdd': False, - 'retain_eplusout_mtd': False, - 'retain_eplusout_rdd': False, - 'retain_eplusout_shd': False, - 'retain_eplusout_sql': False, - 'retain_eplustbl_htm': False, - 'retain_sqlite_err': False, - 'retain_stdout_energyplus': False, - 'retain_stdout_expandobject': False, - 'retain_schedules_csv': True - } - server_dir_cleanup_args.update(workflow_args['server_directory_cleanup']) - - osw['steps'].extend([ - { - 'measure_dir_name': 'ReportSimulationOutput', # (OS-HEScore) - 'arguments': sim_out_rep_args - }, - { - 'measure_dir_name': 'ReportHPXMLOutput', - 'arguments': {} - }, - { - 'measure_dir_name': 'UpgradeCosts', - 'arguments': {} - }, - { - 'measure_dir_name': 'ServerDirectoryCleanup', - 'arguments': server_dir_cleanup_args - } - ]) - - if upgrade_idx is not None: - measure_d = self.cfg['upgrades'][upgrade_idx] - apply_upgrade_measure = { - 'measure_dir_name': 'ApplyUpgrade', - 'arguments': { - 'run_measure': 1 - } - } - if 'upgrade_name' in measure_d: - apply_upgrade_measure['arguments']['upgrade_name'] = measure_d['upgrade_name'] - for opt_num, option in enumerate(measure_d['options'], 1): - apply_upgrade_measure['arguments']['option_{}'.format(opt_num)] = option['option'] - if 'lifetime' in option: - apply_upgrade_measure['arguments']['option_{}_lifetime'.format(opt_num)] = option['lifetime'] - if 'apply_logic' in option: - apply_upgrade_measure['arguments']['option_{}_apply_logic'.format(opt_num)] = \ - self.make_apply_logic_arg(option['apply_logic']) - for cost_num, cost in enumerate(option.get('costs', []), 1): - for arg in ('value', 'multiplier'): - if arg not in cost: - continue - apply_upgrade_measure['arguments']['option_{}_cost_{}_{}'.format(opt_num, cost_num, arg)] = \ - cost[arg] - if 'package_apply_logic' in measure_d: - apply_upgrade_measure['arguments']['package_apply_logic'] = \ - self.make_apply_logic_arg(measure_d['package_apply_logic']) - - build_existing_model_idx = \ - [x['measure_dir_name'] == 'BuildExistingModel' for x in osw['steps']].index(True) - osw['steps'].insert(build_existing_model_idx + 1, apply_upgrade_measure) - - if 'reporting_measures' in workflow_args: - for reporting_measure in workflow_args['reporting_measures']: - if 'arguments' not in reporting_measure: - reporting_measure['arguments'] = {} - reporting_measure['measure_type'] = 'ReportingMeasure' - osw['steps'].insert(-1, reporting_measure) # right before ServerDirectoryCleanup + osw = super().create_osw(sim_id, building_id, upgrade_idx) + if 'os_hescore_directory' in osw['steps'][0]['arguments']: + osw['steps'][0]['arguments']['os_hescore_directory'] = '../../OpenStudio-HEScore' - return osw + return osw \ No newline at end of file From f81e138fa4e3b1db477e2d4d731f2fbd4a183e0f Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Tue, 25 Jan 2022 18:01:29 -0700 Subject: [PATCH 17/37] specify OS-HEScore measure path to use the correct reporting measure --- buildstockbatch/workflow_generator/residential_hpxml_hes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index a747408f..f65fbf25 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -19,4 +19,6 @@ def create_osw(self, sim_id, building_id, upgrade_idx): if 'os_hescore_directory' in osw['steps'][0]['arguments']: osw['steps'][0]['arguments']['os_hescore_directory'] = '../../OpenStudio-HEScore' + # Add measure path for reporting measure + osw['measure_paths'] = ['OpenStudio-HEScore/hpxml-measures'] + osw['measure_paths'] return osw \ No newline at end of file From a209fe6e4e4f86992e6e0ebe91e6896e5636d57c Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Wed, 26 Jan 2022 10:57:31 -0700 Subject: [PATCH 18/37] update test for hpxml hes workflow generator --- .../test_workflow_generator.py | 34 ++++--------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/buildstockbatch/workflow_generator/test_workflow_generator.py b/buildstockbatch/workflow_generator/test_workflow_generator.py index 06959567..aa60e4b9 100644 --- a/buildstockbatch/workflow_generator/test_workflow_generator.py +++ b/buildstockbatch/workflow_generator/test_workflow_generator.py @@ -406,11 +406,6 @@ def test_residential_hpxml_hes(mocker): 'simulation_control_run_period_end_month': 2, 'simulation_control_run_period_end_day_of_month': 28, 'simulation_control_run_period_calendar_year': 2010, - 'apply_hpxml_defaults': True, - 'build_hpxml_only': True, - }, - - 'openstudio_hescore': { 'os_hescore_directory': '../OpenStudio-HEScore' }, @@ -437,7 +432,7 @@ def test_residential_hpxml_hes(mocker): osw = osw_gen.create_osw(sim_id, building_id, upgrade_idx) steps = osw['steps'] - assert(len(steps) == 9) + assert(len(steps) == 6) build_existing_model_step = steps[0] assert(build_existing_model_step['measure_dir_name'] == 'BuildExistingModel') @@ -446,42 +441,25 @@ def test_residential_hpxml_hes(mocker): assert(build_existing_model_step['arguments']['simulation_control_run_period_end_month'] == 2) assert(build_existing_model_step['arguments']['simulation_control_run_period_end_day_of_month'] == 28) assert(build_existing_model_step['arguments']['simulation_control_run_period_calendar_year'] == 2010) + assert(build_existing_model_step['arguments']['os_hescore_directory'] == "../../OpenStudio-HEScore") apply_upgrade_step = steps[1] assert(apply_upgrade_step['measure_dir_name'] == 'ApplyUpgrade') - hescore_hpxml_step = steps[2] - assert(hescore_hpxml_step['measure_dir_name'] == 'HEScoreHPXML') - assert(hescore_hpxml_step['arguments']['hpxml_path'] == '../existing.xml') - assert(hescore_hpxml_step['arguments']['output_path'] == '../hes.json') - - hescore_ruleset_step = steps[3] - assert(hescore_ruleset_step['measure_dir_name'] == 'HEScoreRuleset') - assert(hescore_ruleset_step['arguments']['json_path'] == '/var/simdata/openstudio/run/hes.json') #FIXME temporary filepath - assert(hescore_ruleset_step['arguments']['hpxml_output_path'] == '/var/simdata/openstudio/run/hes.xml') #FIXME temporary filepath - - hpxml_to_os_step = steps[4] - assert(hpxml_to_os_step['measure_dir_name'] == 'HPXMLtoOpenStudio') - assert(hpxml_to_os_step['arguments']['hpxml_path'] == '/var/simdata/openstudio/run/hes.xml') #FIXME temporary filepath - assert(hpxml_to_os_step['arguments']['output_dir'] == '/var/simdata/openstudio/run') #FIXME temporary filepath - assert(hpxml_to_os_step['arguments']['debug'] is False) - assert(hpxml_to_os_step['arguments']['add_component_loads'] is False) - assert(hpxml_to_os_step['arguments']['skip_validation'] is False) - - simulation_output_step = steps[5] + simulation_output_step = steps[2] assert(simulation_output_step['measure_dir_name'] == 'ReportSimulationOutput') assert(simulation_output_step['arguments']['timeseries_frequency'] == 'hourly') assert(simulation_output_step['arguments']['include_timeseries_end_use_consumptions'] is True) assert(simulation_output_step['arguments']['include_timeseries_total_loads'] is True) assert(simulation_output_step['arguments']['include_timeseries_zone_temperatures'] is False) - hpxml_output_step = steps[6] + hpxml_output_step = steps[3] assert(hpxml_output_step['measure_dir_name'] == 'ReportHPXMLOutput') - upgrade_costs_step = steps[7] + upgrade_costs_step = steps[4] assert(upgrade_costs_step['measure_dir_name'] == 'UpgradeCosts') - server_dir_cleanup_step = steps[8] + server_dir_cleanup_step = steps[5] assert(server_dir_cleanup_step['measure_dir_name'] == 'ServerDirectoryCleanup') From 54f7f221876dc86a2f7948600dcd7397c97b6b0e Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Wed, 26 Jan 2022 11:07:45 -0700 Subject: [PATCH 19/37] style fixes --- buildstockbatch/base.py | 3 ++- buildstockbatch/workflow_generator/residential_hpxml_hes.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/buildstockbatch/base.py b/buildstockbatch/base.py index 4fdde4c0..15a61508 100644 --- a/buildstockbatch/base.py +++ b/buildstockbatch/base.py @@ -64,7 +64,8 @@ def __init__(self, project_filename): if not os.path.isdir(self.project_dir): raise FileNotFoundError(f'project_directory = {self.project_dir} is not a directory.') if 'os_hescore_directory' in self.cfg['workflow_generator']['args']['build_existing_model']: - self.os_hescore_dir = path_rel_to_file('',self.cfg['workflow_generator']['args']['build_existing_model']['os_hescore_directory']) + self.os_hescore_dir = path_rel_to_file( + '', self.cfg['workflow_generator']['args']['build_existing_model']['os_hescore_directory']) else: self.os_hescore_dir = None diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index f65fbf25..26161792 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -12,8 +12,9 @@ from .residential_hpxml import ResidentialHpxmlWorkflowGenerator # noqa F041 + class ResidentialHpxmlHesWorkflowGenerator(ResidentialHpxmlWorkflowGenerator): - + def create_osw(self, sim_id, building_id, upgrade_idx): osw = super().create_osw(sim_id, building_id, upgrade_idx) if 'os_hescore_directory' in osw['steps'][0]['arguments']: @@ -21,4 +22,5 @@ def create_osw(self, sim_id, building_id, upgrade_idx): # Add measure path for reporting measure osw['measure_paths'] = ['OpenStudio-HEScore/hpxml-measures'] + osw['measure_paths'] - return osw \ No newline at end of file + + return osw From bccc0c10d6d0e3b451616367c0cd7de702adb428 Mon Sep 17 00:00:00 2001 From: Joe Robertson Date: Wed, 16 Feb 2022 14:43:24 -0700 Subject: [PATCH 20/37] Fix syntax. --- buildstockbatch/localdocker.py | 4 ++-- buildstockbatch/workflow_generator/residential_hpxml_hes.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/buildstockbatch/localdocker.py b/buildstockbatch/localdocker.py index 5e55b498..5add427f 100644 --- a/buildstockbatch/localdocker.py +++ b/buildstockbatch/localdocker.py @@ -22,10 +22,10 @@ import logging import os import pandas as pd -import re +# import re import shutil import sys -import tarfile +# import tarfile import tempfile from buildstockbatch.base import BuildStockBatchBase, SimulationExists diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index 26161792..34c7a6b5 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -22,5 +22,5 @@ def create_osw(self, sim_id, building_id, upgrade_idx): # Add measure path for reporting measure osw['measure_paths'] = ['OpenStudio-HEScore/hpxml-measures'] + osw['measure_paths'] - + return osw From 837098edf2ef81bb3e550b8e92c1f40c3a64a321 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Wed, 16 Feb 2022 16:34:16 -0700 Subject: [PATCH 21/37] bind OS-HEScore/hescore-hpxml to local docker image --- buildstockbatch/localdocker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/buildstockbatch/localdocker.py b/buildstockbatch/localdocker.py index 5e55b498..54d7565b 100644 --- a/buildstockbatch/localdocker.py +++ b/buildstockbatch/localdocker.py @@ -120,6 +120,7 @@ def run_building(cls, project_dir, buildstock_dir, weather_dir, os_hescore_dir, if os_hescore_dir and os.path.exists(os_hescore_dir): bind_mounts += [(os.path.join(os_hescore_dir, 'rulesets'), 'OpenStudio-HEScore/rulesets', 'ro'), (os.path.join(os_hescore_dir, 'hpxml-measures'), 'OpenStudio-HEScore/hpxml-measures', 'ro'), + (os.path.join(os_hescore_dir, 'hescore-hpxml'), 'OpenStudio-HEScore/hescore-hpxml', 'ro'), (os.path.join(os_hescore_dir, 'weather'), 'OpenStudio-HEScore/weather', 'ro')] docker_volume_mounts = dict([(key, {'bind': f'/var/simdata/openstudio/{bind}', 'mode': mode}) for key, bind, mode in bind_mounts]) # noqa E501 From 947df50b02f992c5de3f088f67e799f46c85c129 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Wed, 16 Feb 2022 17:18:47 -0700 Subject: [PATCH 22/37] address pytest errors --- buildstockbatch/base.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/buildstockbatch/base.py b/buildstockbatch/base.py index 18e1dc4c..8588cf12 100644 --- a/buildstockbatch/base.py +++ b/buildstockbatch/base.py @@ -63,11 +63,12 @@ def __init__(self, project_filename): self.project_dir = os.path.join(self.buildstock_dir, self.cfg['project_directory']) if not os.path.isdir(self.project_dir): raise FileNotFoundError(f'project_directory = {self.project_dir} is not a directory.') - if 'os_hescore_directory' in self.cfg['workflow_generator']['args']['build_existing_model']: - self.os_hescore_dir = path_rel_to_file( - '', self.cfg['workflow_generator']['args']['build_existing_model']['os_hescore_directory']) - else: - self.os_hescore_dir = None + if 'build_existing_model' in self.cfg['workflow_generator']['args']: + if 'os_hescore_directory' in self.cfg['workflow_generator']['args']['build_existing_model']: + self.os_hescore_dir = path_rel_to_file( + '', self.cfg['workflow_generator']['args']['build_existing_model']['os_hescore_directory']) + else: + self.os_hescore_dir = None # Load in OS_VERSION and OS_SHA arguments if they exist in the YAML, # otherwise use defaults specified here. From 8bc4b4f5f095d6a09b0f4ff74e11c7310eab93d5 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Wed, 16 Feb 2022 17:42:08 -0700 Subject: [PATCH 23/37] temporary yml for hes workflow testing --- project_resstock_national_hes.yml | 47 +++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 project_resstock_national_hes.yml diff --git a/project_resstock_national_hes.yml b/project_resstock_national_hes.yml new file mode 100644 index 00000000..b6249b98 --- /dev/null +++ b/project_resstock_national_hes.yml @@ -0,0 +1,47 @@ +###### FIXME: Temporary yml for testing, delete before merge + +schema_version: '0.3' +buildstock_directory: ../resstock # Relative to this file or absolute +project_directory: project_national # Relative to buildstock_directory +output_directory: national_baseline_hes +# weather_files_url: https://data.nrel.gov/system/files/156/BuildStock_TMY3_FIPS.zip +weather_files_path: c:/OpenStudio/BuildStock_TMY3_FIPS.zip + +sampler: + type: residential_quota_downselect + args: + n_datapoints: 10 + logic: Geometry Building Type RECS|Single-Family Detached + resample: false + +workflow_generator: + type: residential_hpxml_hes + args: + build_existing_model: + simulation_control_timestep: 60 + simulation_control_run_period_begin_month: 1 + simulation_control_run_period_begin_day_of_month: 1 + simulation_control_run_period_end_month: 12 + simulation_control_run_period_end_day_of_month: 31 + simulation_control_run_period_calendar_year: 2007 + os_hescore_directory: ../OpenStudio-HEScore + + simulation_output_report: + timeseries_frequency: none + + server_directory_cleanup: + retain_in_idf: false + retain_schedules_csv: false + +baseline: + n_buildings_represented: 110000000 + +eagle: + n_jobs: 2 + minutes_per_sim: 1 + account: + sampling: + time: 10 + postprocessing: + time: 10 + n_workers: 1 From a22e02c1dd44e43f143996c36c6349ccceed8cb7 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Wed, 16 Feb 2022 17:42:27 -0700 Subject: [PATCH 24/37] fix relative path for OS-HEScore directory --- buildstockbatch/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildstockbatch/base.py b/buildstockbatch/base.py index 8588cf12..08ad648d 100644 --- a/buildstockbatch/base.py +++ b/buildstockbatch/base.py @@ -66,7 +66,7 @@ def __init__(self, project_filename): if 'build_existing_model' in self.cfg['workflow_generator']['args']: if 'os_hescore_directory' in self.cfg['workflow_generator']['args']['build_existing_model']: self.os_hescore_dir = path_rel_to_file( - '', self.cfg['workflow_generator']['args']['build_existing_model']['os_hescore_directory']) + project_filename, self.cfg['workflow_generator']['args']['build_existing_model']['os_hescore_directory']) else: self.os_hescore_dir = None From 5bdb4e5bd2b2dbdc56395a0a708ea66cfe18252e Mon Sep 17 00:00:00 2001 From: Noel Merket Date: Tue, 1 Mar 2022 13:32:04 -0700 Subject: [PATCH 25/37] mounting into /opt/hescore-hpxml --- buildstockbatch/localdocker.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/buildstockbatch/localdocker.py b/buildstockbatch/localdocker.py index e919a4ca..263845c2 100644 --- a/buildstockbatch/localdocker.py +++ b/buildstockbatch/localdocker.py @@ -113,18 +113,28 @@ def run_building(cls, project_dir, buildstock_dir, weather_dir, os_hescore_dir, # ResStock-hpxml measure directory if os.path.exists(os.path.join(buildstock_dir, 'resources', 'hpxml-measures')): - bind_mounts += [(os.path.join(buildstock_dir, 'resources', 'hpxml-measures'), - 'resources/hpxml-measures', 'ro')] + bind_mounts.append( + (os.path.join(buildstock_dir, 'resources', 'hpxml-measures'), 'resources/hpxml-measures', 'ro') + ) # OS-HEScore measure directories if os_hescore_dir and os.path.exists(os_hescore_dir): - bind_mounts += [(os.path.join(os_hescore_dir, 'rulesets'), 'OpenStudio-HEScore/rulesets', 'ro'), - (os.path.join(os_hescore_dir, 'hpxml-measures'), 'OpenStudio-HEScore/hpxml-measures', 'ro'), - (os.path.join(os_hescore_dir, 'hescore-hpxml'), 'OpenStudio-HEScore/hescore-hpxml', 'ro'), - (os.path.join(os_hescore_dir, 'weather'), 'OpenStudio-HEScore/weather', 'ro')] + bind_mounts.extend([ + (os.path.join(os_hescore_dir), 'OpenStudio-HEScore', 'ro'), + (os.path.join(os_hescore_dir, 'hescore-hpxml'), '/opt/hescore-hpxml', 'ro') + ]) + + docker_volume_mounts = { + key: { + 'bind': bind if bind.startswith('/') else f'/var/simdata/openstudio/{bind}', + 'mode': mode + } + for key, bind, mode in bind_mounts + } - docker_volume_mounts = dict([(key, {'bind': f'/var/simdata/openstudio/{bind}', 'mode': mode}) for key, bind, mode in bind_mounts]) # noqa E501 for bind in bind_mounts: + if bind[1].startswith('/'): + continue dir_to_make = os.path.join(sim_dir, *bind[1].split('/')) if not os.path.exists(dir_to_make): os.makedirs(dir_to_make) From 955f610ebc4f36b9efedf9c1ed1b7d6527870e2f Mon Sep 17 00:00:00 2001 From: Noel Merket Date: Wed, 2 Mar 2022 11:39:14 -0700 Subject: [PATCH 26/37] keeping the whole batch from failing on a single simulation failure --- buildstockbatch/localdocker.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/buildstockbatch/localdocker.py b/buildstockbatch/localdocker.py index 263845c2..0b4f61c9 100644 --- a/buildstockbatch/localdocker.py +++ b/buildstockbatch/localdocker.py @@ -22,10 +22,10 @@ import logging import os import pandas as pd -# import re +import re import shutil import sys -# import tarfile +import tarfile import tempfile from buildstockbatch.base import BuildStockBatchBase, SimulationExists @@ -155,16 +155,19 @@ def run_building(cls, project_dir, buildstock_dir, weather_dir, os_hescore_dir, extra_kws = {} if sys.platform.startswith('linux'): extra_kws['user'] = f'{os.getuid()}:{os.getgid()}' - container_output = docker_client.containers.run( + + container = docker_client.containers.run( docker_image, args, remove=True, + detach=True, volumes=docker_volume_mounts, name=sim_id, **extra_kws ) with open(os.path.join(sim_dir, 'docker_output.log'), 'wb') as f_out: - f_out.write(container_output) + for x in container.logs(stream=True): + f_out.write(x) # Clean up directories created with the docker mounts for dirname in ('lib', 'measures', 'weather'): @@ -226,13 +229,13 @@ def run_batch(self, n_jobs=None, measures_only=False, sampling_only=False): del dpouts # FIXME temporarily comment out for testing - # sim_out_tarfile_name = os.path.join(sim_out_dir, 'simulations_job0.tar.gz') - # logger.debug(f'Compressing simulation outputs to {sim_out_tarfile_name}') - # with tarfile.open(sim_out_tarfile_name, 'w:gz') as tarf: - # for dirname in os.listdir(sim_out_dir): - # if re.match(r'up\d+', dirname) and os.path.isdir(os.path.join(sim_out_dir, dirname)): - # tarf.add(os.path.join(sim_out_dir, dirname), arcname=dirname) - # shutil.rmtree(os.path.join(sim_out_dir, dirname)) + sim_out_tarfile_name = os.path.join(sim_out_dir, 'simulations_job0.tar.gz') + logger.debug(f'Compressing simulation outputs to {sim_out_tarfile_name}') + with tarfile.open(sim_out_tarfile_name, 'w:gz') as tarf: + for dirname in os.listdir(sim_out_dir): + if re.match(r'up\d+', dirname) and os.path.isdir(os.path.join(sim_out_dir, dirname)): + tarf.add(os.path.join(sim_out_dir, dirname), arcname=dirname) + shutil.rmtree(os.path.join(sim_out_dir, dirname)) @property def output_dir(self): From 5a141beff5bff678e2133bc46ef2291e5fca7137 Mon Sep 17 00:00:00 2001 From: Noel Merket Date: Fri, 4 Mar 2022 09:14:50 -0700 Subject: [PATCH 27/37] minor formatting change --- buildstockbatch/workflow_generator/residential_hpxml_hes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index 34c7a6b5..876e195c 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -21,6 +21,6 @@ def create_osw(self, sim_id, building_id, upgrade_idx): osw['steps'][0]['arguments']['os_hescore_directory'] = '../../OpenStudio-HEScore' # Add measure path for reporting measure - osw['measure_paths'] = ['OpenStudio-HEScore/hpxml-measures'] + osw['measure_paths'] + osw['measure_paths'].insert(0, 'OpenStudio-HEScore/hpxml-measures') return osw From 2a0023298980cbc402c4c84727366fa01eede30c Mon Sep 17 00:00:00 2001 From: Noel Merket Date: Tue, 8 Mar 2022 15:15:07 -0700 Subject: [PATCH 28/37] building hescore image on the fly --- buildstockbatch/localdocker.py | 35 ++++++++++++------- .../residential_hpxml_hes.py | 4 +-- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/buildstockbatch/localdocker.py b/buildstockbatch/localdocker.py index 0b4f61c9..36026701 100644 --- a/buildstockbatch/localdocker.py +++ b/buildstockbatch/localdocker.py @@ -52,6 +52,22 @@ def __init__(self, project_filename): self._weather_dir = None + # FIXME: something better and more general than os_hescore_dir + if self.os_hescore_dir: + logger.debug("Building HEScore docker image") + self.docker_client.images.build( + path=self.os_hescore_dir, + tag=self.docker_image, + dockerfile="resstock/Dockerfile", + rm=True + ) + logger.debug("Docker image built") + else: + try: + self.docker_client.images.get(self.docker_image) + except ImageNotFound: + self.docker_client.images.pull(self.docker_image) + @staticmethod def validate_project(project_file): super(DockerBatchBase, DockerBatchBase).validate_project(project_file) @@ -59,9 +75,10 @@ def validate_project(project_file): @property def docker_image(self): - # FIXME temporary docker image for testing - return 'nrel/hescore-hpxml-openstudio' - # return 'nrel/openstudio:{}'.format(self.os_version) + if self.os_hescore_dir: + return 'nrel/hescore-hpxml-openstudio' + else: + return 'nrel/openstudio:{}'.format(self.os_version) class LocalDockerBatch(DockerBatchBase): @@ -70,11 +87,6 @@ def __init__(self, project_filename): super().__init__(project_filename) logger.debug(f'Pulling docker image: {self.docker_image}') - try: - self.docker_client.images.get(self.docker_image) - except ImageNotFound: - self.docker_client.images.pull(self.docker_image) - # Create simulation_output dir sim_out_ts_dir = os.path.join(self.results_dir, 'simulation_output', 'timeseries') os.makedirs(sim_out_ts_dir, exist_ok=True) @@ -117,12 +129,9 @@ def run_building(cls, project_dir, buildstock_dir, weather_dir, os_hescore_dir, (os.path.join(buildstock_dir, 'resources', 'hpxml-measures'), 'resources/hpxml-measures', 'ro') ) - # OS-HEScore measure directories + # OS-HEScore weather directory if os_hescore_dir and os.path.exists(os_hescore_dir): - bind_mounts.extend([ - (os.path.join(os_hescore_dir), 'OpenStudio-HEScore', 'ro'), - (os.path.join(os_hescore_dir, 'hescore-hpxml'), '/opt/hescore-hpxml', 'ro') - ]) + bind_mounts.append((os.path.join(os_hescore_dir, 'weather'), '/opt/OpenStudio-HEScore/weather', 'ro')) docker_volume_mounts = { key: { diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index 876e195c..eecf18a4 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -18,9 +18,9 @@ class ResidentialHpxmlHesWorkflowGenerator(ResidentialHpxmlWorkflowGenerator): def create_osw(self, sim_id, building_id, upgrade_idx): osw = super().create_osw(sim_id, building_id, upgrade_idx) if 'os_hescore_directory' in osw['steps'][0]['arguments']: - osw['steps'][0]['arguments']['os_hescore_directory'] = '../../OpenStudio-HEScore' + osw['steps'][0]['arguments']['os_hescore_directory'] = '/opt/OpenStudio-HEScore' # Add measure path for reporting measure - osw['measure_paths'].insert(0, 'OpenStudio-HEScore/hpxml-measures') + osw['measure_paths'].insert(0, '/opt/OpenStudio-HEScore/hpxml-measures') return osw From 0ac675a50761cfe451ec8e18040299656402178f Mon Sep 17 00:00:00 2001 From: Noel Merket Date: Wed, 9 Mar 2022 16:51:34 -0700 Subject: [PATCH 29/37] adding resstock hescore reporting measure --- .../workflow_generator/residential_hpxml_hes.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index eecf18a4..96152aa2 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -22,5 +22,17 @@ def create_osw(self, sim_id, building_id, upgrade_idx): # Add measure path for reporting measure osw['measure_paths'].insert(0, '/opt/OpenStudio-HEScore/hpxml-measures') + osw['measure_paths'].insert(1, '/opt/OpenStudio-HEScore/rulesets') + + # Add HEScore reporting measure + osw['steps'].insert(-1, { + 'measure_dir_name': 'ReportResStockHEScore', + 'arguments': {} + }) return osw + + def reporting_measures(self): + reporting_measures = super().reporting_measures() + reporting_measures.append('ReportResStockHEScore') + return reporting_measures From 82f4c7f2f38f85151313777be732353b215f671d Mon Sep 17 00:00:00 2001 From: Scott Horowitz Date: Thu, 7 Apr 2022 17:15:35 -0600 Subject: [PATCH 30/37] Revert "adding resstock hescore reporting measure" This reverts commit 0ac675a50761cfe451ec8e18040299656402178f. --- .../workflow_generator/residential_hpxml_hes.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index 96152aa2..eecf18a4 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -22,17 +22,5 @@ def create_osw(self, sim_id, building_id, upgrade_idx): # Add measure path for reporting measure osw['measure_paths'].insert(0, '/opt/OpenStudio-HEScore/hpxml-measures') - osw['measure_paths'].insert(1, '/opt/OpenStudio-HEScore/rulesets') - - # Add HEScore reporting measure - osw['steps'].insert(-1, { - 'measure_dir_name': 'ReportResStockHEScore', - 'arguments': {} - }) return osw - - def reporting_measures(self): - reporting_measures = super().reporting_measures() - reporting_measures.append('ReportResStockHEScore') - return reporting_measures From 35ba146ef2977d68cec70aec785d3e5c4ac8c37c Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Thu, 10 Nov 2022 16:35:07 -0700 Subject: [PATCH 31/37] add buildarg to hes docker image, minor updates to hes workflow yml --- buildstockbatch/localdocker.py | 3 ++- project_resstock_national_hes.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/buildstockbatch/localdocker.py b/buildstockbatch/localdocker.py index c912269a..031ea2f3 100644 --- a/buildstockbatch/localdocker.py +++ b/buildstockbatch/localdocker.py @@ -60,7 +60,8 @@ def __init__(self, project_filename): path=self.os_hescore_dir, tag=self.docker_image, dockerfile="resstock/Dockerfile", - rm=True + rm=True, + buildargs={'os_version': self.os_version} ) logger.debug("Docker image built") else: diff --git a/project_resstock_national_hes.yml b/project_resstock_national_hes.yml index b6249b98..f723c2fb 100644 --- a/project_resstock_national_hes.yml +++ b/project_resstock_national_hes.yml @@ -1,6 +1,7 @@ ###### FIXME: Temporary yml for testing, delete before merge - schema_version: '0.3' +os_version: 3.4.0 +os_sha: 4bd816f785 buildstock_directory: ../resstock # Relative to this file or absolute project_directory: project_national # Relative to buildstock_directory output_directory: national_baseline_hes From 7cd801ca285b4a600129485a2ca14225e8a8e6c6 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Thu, 10 Nov 2022 16:37:17 -0700 Subject: [PATCH 32/37] break up yml validation method in order to edit schema in child class --- .../workflow_generator/residential_hpxml.py | 14 ++++++++++---- .../workflow_generator/residential_hpxml_hes.py | 13 +++++++++++-- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/buildstockbatch/workflow_generator/residential_hpxml.py b/buildstockbatch/workflow_generator/residential_hpxml.py index 12be8a95..8bf18206 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml.py +++ b/buildstockbatch/workflow_generator/residential_hpxml.py @@ -34,7 +34,7 @@ def get_measure_xml(xml_path): class ResidentialHpxmlWorkflowGenerator(WorkflowGeneratorBase): @classmethod - def validate(cls, cfg): + def get_yml_schema(cls): """Validate arguments :param cfg: project configuration @@ -127,9 +127,15 @@ def validate(cls, cfg): retain_schedules_csv: bool(required=False) debug: bool(required=False) """ - workflow_generator_args = cfg['workflow_generator']['args'] schema_yml = re.sub(r'^ {8}', '', schema_yml, flags=re.MULTILINE) schema = yamale.make_schema(content=schema_yml, parser='ruamel') + return(schema) + + @classmethod + def validate(cls, cfg): + schema = cls.get_yml_schema() + + workflow_generator_args = cfg['workflow_generator']['args'] data = yamale.make_data(content=json.dumps(workflow_generator_args), parser='ruamel') yamale.validate(schema, data, strict=True) return cls.validate_measures_and_arguments(cfg) @@ -374,8 +380,8 @@ def create_osw(self, sim_id, building_id, upgrade_idx): { 'measure_dir_name': 'HPXMLtoOpenStudio', 'arguments': { - 'hpxml_path': '../../run/home.xml', - 'output_dir': '../../run', + 'hpxml_path': '/var/simdata/openstudio/run/home.xml', + 'output_dir': '/var/simdata/openstudio/run', 'debug': debug, 'add_component_loads': add_component_loads } diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index eecf18a4..49bb4f43 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -11,7 +11,7 @@ """ from .residential_hpxml import ResidentialHpxmlWorkflowGenerator # noqa F041 - +import yamale class ResidentialHpxmlHesWorkflowGenerator(ResidentialHpxmlWorkflowGenerator): @@ -22,5 +22,14 @@ def create_osw(self, sim_id, building_id, upgrade_idx): # Add measure path for reporting measure osw['measure_paths'].insert(0, '/opt/OpenStudio-HEScore/hpxml-measures') - return osw + + @classmethod + def get_yml_schema(cls): + schema = super().get_yml_schema() + + # Require os_hescore_directory argument + string_validator = yamale.validators.String(required=True) + schema.includes['build-existing-model-spec'].dict['os_hescore_directory'] = string_validator + print(schema.includes['build-existing-model-spec'].dict) + return(schema) From 653f6d28dd282164a9d0320b6a8410f8629f82be Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Thu, 10 Nov 2022 16:57:13 -0700 Subject: [PATCH 33/37] style fixes --- buildstockbatch/eagle.py | 8 ++-- .../workflow_generator/__init__.py | 2 +- .../workflow_generator/residential_hpxml.py | 38 +++++++++---------- .../residential_hpxml_hes.py | 1 + docs/conf.py | 20 +++++----- 5 files changed, 35 insertions(+), 34 deletions(-) diff --git a/buildstockbatch/eagle.py b/buildstockbatch/eagle.py index 00c36b98..349beb40 100644 --- a/buildstockbatch/eagle.py +++ b/buildstockbatch/eagle.py @@ -103,10 +103,10 @@ def singularity_image_url(self): else: prefix_ver = self.os_version return 'https://s3.amazonaws.com/openstudio-builds/{prefix_ver}/OpenStudio-{ver}.{sha}-Singularity.simg'.format( - prefix_ver=prefix_ver, - ver=self.os_version, - sha=self.os_sha - ) + prefix_ver=prefix_ver, + ver=self.os_version, + sha=self.os_sha + ) @property def singularity_image(self): diff --git a/buildstockbatch/workflow_generator/__init__.py b/buildstockbatch/workflow_generator/__init__.py index 299dca33..f8091152 100644 --- a/buildstockbatch/workflow_generator/__init__.py +++ b/buildstockbatch/workflow_generator/__init__.py @@ -3,4 +3,4 @@ from .residential import ResidentialDefaultWorkflowGenerator # noqa F041 from .commercial import CommercialDefaultWorkflowGenerator # noqa F041 from .residential_hpxml import ResidentialHpxmlWorkflowGenerator # noqa F041 -from .residential_hpxml_hes import ResidentialHpxmlHesWorkflowGenerator # noqa F041 \ No newline at end of file +from .residential_hpxml_hes import ResidentialHpxmlHesWorkflowGenerator # noqa F041 diff --git a/buildstockbatch/workflow_generator/residential_hpxml.py b/buildstockbatch/workflow_generator/residential_hpxml.py index 2a88661e..19185e9d 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml.py +++ b/buildstockbatch/workflow_generator/residential_hpxml.py @@ -356,25 +356,25 @@ def create_osw(self, sim_id, building_id, upgrade_idx): debug = workflow_args['debug'] server_dir_cleanup_args = { - 'retain_in_osm': False, - 'retain_in_idf': True, - 'retain_pre_process_idf': False, - 'retain_eplusout_audit': False, - 'retain_eplusout_bnd': False, - 'retain_eplusout_eio': False, - 'retain_eplusout_end': False, - 'retain_eplusout_err': False, - 'retain_eplusout_eso': False, - 'retain_eplusout_mdd': False, - 'retain_eplusout_mtd': False, - 'retain_eplusout_rdd': False, - 'retain_eplusout_shd': False, - 'retain_eplusout_msgpack': False, - 'retain_eplustbl_htm': False, - 'retain_stdout_energyplus': False, - 'retain_stdout_expandobject': False, - 'retain_schedules_csv': True, - 'debug': debug + 'retain_in_osm': False, + 'retain_in_idf': True, + 'retain_pre_process_idf': False, + 'retain_eplusout_audit': False, + 'retain_eplusout_bnd': False, + 'retain_eplusout_eio': False, + 'retain_eplusout_end': False, + 'retain_eplusout_err': False, + 'retain_eplusout_eso': False, + 'retain_eplusout_mdd': False, + 'retain_eplusout_mtd': False, + 'retain_eplusout_rdd': False, + 'retain_eplusout_shd': False, + 'retain_eplusout_msgpack': False, + 'retain_eplustbl_htm': False, + 'retain_stdout_energyplus': False, + 'retain_stdout_expandobject': False, + 'retain_schedules_csv': True, + 'debug': debug } server_dir_cleanup_args.update(workflow_args['server_directory_cleanup']) diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index 49bb4f43..19b22337 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -13,6 +13,7 @@ from .residential_hpxml import ResidentialHpxmlWorkflowGenerator # noqa F041 import yamale + class ResidentialHpxmlHesWorkflowGenerator(ResidentialHpxmlWorkflowGenerator): def create_osw(self, sim_id, building_id, upgrade_idx): diff --git a/docs/conf.py b/docs/conf.py index ceb6b474..ee285577 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -227,14 +227,14 @@ # Ignore reference targets not found nitpick_ignore = [ - ('py:func', 'BuildStockBatchBase.validate_precomputed_sample'), - ('py:func', 'BuildStockBatchBase.validate_xor_nor_schema_keys'), - ('py:func', 'EagleBatch.run_building'), - ('py:class', 'sampler.CommercialSobolSingularitySampler'), - ('py:class', 'sampler.CommercialSobolDockerSampler'), - ('py:class', 'workflow_generator.CommercialDefaultWorkflowGenerator'), - ('py:class', 'sampler.PrecomputedSampler'), - ('py:class', 'sampler.BuildStockSampler'), - ('py:class', 'BuildStockBatchBase'), - ('py:func', 'BuildStockBatchBase.run_sampling') + ('py:func', 'BuildStockBatchBase.validate_precomputed_sample'), + ('py:func', 'BuildStockBatchBase.validate_xor_nor_schema_keys'), + ('py:func', 'EagleBatch.run_building'), + ('py:class', 'sampler.CommercialSobolSingularitySampler'), + ('py:class', 'sampler.CommercialSobolDockerSampler'), + ('py:class', 'workflow_generator.CommercialDefaultWorkflowGenerator'), + ('py:class', 'sampler.PrecomputedSampler'), + ('py:class', 'sampler.BuildStockSampler'), + ('py:class', 'BuildStockBatchBase'), + ('py:func', 'BuildStockBatchBase.run_sampling') ] From 7bc8030c9bf0ed59dbfc32121e2b4020b3d86bd5 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Thu, 10 Nov 2022 17:09:46 -0700 Subject: [PATCH 34/37] more style fixes --- .../workflow_generator/residential_hpxml.py | 2 +- .../residential_hpxml_hes.py | 2 +- .../test_workflow_generator.py | 34 +++++++++---------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/buildstockbatch/workflow_generator/residential_hpxml.py b/buildstockbatch/workflow_generator/residential_hpxml.py index 19185e9d..455608be 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml.py +++ b/buildstockbatch/workflow_generator/residential_hpxml.py @@ -130,7 +130,7 @@ def get_yml_schema(cls): """ schema_yml = re.sub(r'^ {8}', '', schema_yml, flags=re.MULTILINE) schema = yamale.make_schema(content=schema_yml, parser='ruamel') - return(schema) + return schema @classmethod def validate(cls, cfg): diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index 19b22337..eb2b9157 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -33,4 +33,4 @@ def get_yml_schema(cls): string_validator = yamale.validators.String(required=True) schema.includes['build-existing-model-spec'].dict['os_hescore_directory'] = string_validator print(schema.includes['build-existing-model-spec'].dict) - return(schema) + return schema diff --git a/buildstockbatch/workflow_generator/test_workflow_generator.py b/buildstockbatch/workflow_generator/test_workflow_generator.py index 1c988ee5..7cbaa067 100644 --- a/buildstockbatch/workflow_generator/test_workflow_generator.py +++ b/buildstockbatch/workflow_generator/test_workflow_generator.py @@ -451,35 +451,35 @@ def test_residential_hpxml_hes(mocker): osw = osw_gen.create_osw(sim_id, building_id, upgrade_idx) steps = osw['steps'] - assert(len(steps) == 6) + assert len(steps) == 6 build_existing_model_step = steps[0] - assert(build_existing_model_step['measure_dir_name'] == 'BuildExistingModel') - assert(build_existing_model_step['arguments']['simulation_control_run_period_begin_month'] == 2) - assert(build_existing_model_step['arguments']['simulation_control_run_period_begin_day_of_month'] == 1) - assert(build_existing_model_step['arguments']['simulation_control_run_period_end_month'] == 2) - assert(build_existing_model_step['arguments']['simulation_control_run_period_end_day_of_month'] == 28) - assert(build_existing_model_step['arguments']['simulation_control_run_period_calendar_year'] == 2010) - assert(build_existing_model_step['arguments']['os_hescore_directory'] == "../../OpenStudio-HEScore") + assert build_existing_model_step['measure_dir_name'] == 'BuildExistingModel' + assert build_existing_model_step['arguments']['simulation_control_run_period_begin_month'] == 2 + assert build_existing_model_step['arguments']['simulation_control_run_period_begin_day_of_month'] == 1 + assert build_existing_model_step['arguments']['simulation_control_run_period_end_month'] == 2 + assert build_existing_model_step['arguments']['simulation_control_run_period_end_day_of_month'] == 28 + assert build_existing_model_step['arguments']['simulation_control_run_period_calendar_year'] == 2010 + assert build_existing_model_step['arguments']['os_hescore_directory'] == "../../OpenStudio-HEScore" apply_upgrade_step = steps[1] - assert(apply_upgrade_step['measure_dir_name'] == 'ApplyUpgrade') + assert apply_upgrade_step['measure_dir_name'] == 'ApplyUpgrade' simulation_output_step = steps[2] - assert(simulation_output_step['measure_dir_name'] == 'ReportSimulationOutput') - assert(simulation_output_step['arguments']['timeseries_frequency'] == 'hourly') - assert(simulation_output_step['arguments']['include_timeseries_end_use_consumptions'] is True) - assert(simulation_output_step['arguments']['include_timeseries_total_loads'] is True) - assert(simulation_output_step['arguments']['include_timeseries_zone_temperatures'] is False) + assert simulation_output_step['measure_dir_name'] == 'ReportSimulationOutput' + assert simulation_output_step['arguments']['timeseries_frequency'] == 'hourly' + assert simulation_output_step['arguments']['include_timeseries_end_use_consumptions'] is True + assert simulation_output_step['arguments']['include_timeseries_total_loads'] is True + assert simulation_output_step['arguments']['include_timeseries_zone_temperatures'] is False hpxml_output_step = steps[3] - assert(hpxml_output_step['measure_dir_name'] == 'ReportHPXMLOutput') + assert hpxml_output_step['measure_dir_name'] == 'ReportHPXMLOutput' upgrade_costs_step = steps[4] - assert(upgrade_costs_step['measure_dir_name'] == 'UpgradeCosts') + assert upgrade_costs_step['measure_dir_name'] == 'UpgradeCosts' server_dir_cleanup_step = steps[5] - assert(server_dir_cleanup_step['measure_dir_name'] == 'ServerDirectoryCleanup') + assert server_dir_cleanup_step['measure_dir_name'] == 'ServerDirectoryCleanup' def test_com_default_workflow_generator_basic(mocker): From 50125e37b774e55145e96e55314a6b8ce5e6b3fa Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Thu, 10 Nov 2022 18:37:41 -0700 Subject: [PATCH 35/37] update hes workflow generator test --- .../test_workflow_generator.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/buildstockbatch/workflow_generator/test_workflow_generator.py b/buildstockbatch/workflow_generator/test_workflow_generator.py index 7cbaa067..3022404f 100644 --- a/buildstockbatch/workflow_generator/test_workflow_generator.py +++ b/buildstockbatch/workflow_generator/test_workflow_generator.py @@ -451,7 +451,7 @@ def test_residential_hpxml_hes(mocker): osw = osw_gen.create_osw(sim_id, building_id, upgrade_idx) steps = osw['steps'] - assert len(steps) == 6 + assert len(steps) == 8 build_existing_model_step = steps[0] assert build_existing_model_step['measure_dir_name'] == 'BuildExistingModel' @@ -460,25 +460,35 @@ def test_residential_hpxml_hes(mocker): assert build_existing_model_step['arguments']['simulation_control_run_period_end_month'] == 2 assert build_existing_model_step['arguments']['simulation_control_run_period_end_day_of_month'] == 28 assert build_existing_model_step['arguments']['simulation_control_run_period_calendar_year'] == 2010 - assert build_existing_model_step['arguments']['os_hescore_directory'] == "../../OpenStudio-HEScore" + assert build_existing_model_step['arguments']['os_hescore_directory'] == "/opt/OpenStudio-HEScore" apply_upgrade_step = steps[1] assert apply_upgrade_step['measure_dir_name'] == 'ApplyUpgrade' - simulation_output_step = steps[2] + apply_upgrade_step = steps[2] + assert apply_upgrade_step['measure_dir_name'] == 'HPXMLtoOpenStudio' + assert apply_upgrade_step['arguments']['hpxml_path'] == '/var/simdata/openstudio/run/home.xml' + assert apply_upgrade_step['arguments']['output_dir'] == '/var/simdata/openstudio/run' + assert apply_upgrade_step['arguments']['debug'] is False + assert apply_upgrade_step['arguments']['add_component_loads'] is False + + simulation_output_step = steps[3] assert simulation_output_step['measure_dir_name'] == 'ReportSimulationOutput' assert simulation_output_step['arguments']['timeseries_frequency'] == 'hourly' assert simulation_output_step['arguments']['include_timeseries_end_use_consumptions'] is True assert simulation_output_step['arguments']['include_timeseries_total_loads'] is True assert simulation_output_step['arguments']['include_timeseries_zone_temperatures'] is False - hpxml_output_step = steps[3] + hpxml_output_step = steps[4] assert hpxml_output_step['measure_dir_name'] == 'ReportHPXMLOutput' - upgrade_costs_step = steps[4] + hpxml_output_step = steps[5] + assert hpxml_output_step['measure_dir_name'] == 'ReportUtilityBills' + + upgrade_costs_step = steps[6] assert upgrade_costs_step['measure_dir_name'] == 'UpgradeCosts' - server_dir_cleanup_step = steps[5] + server_dir_cleanup_step = steps[7] assert server_dir_cleanup_step['measure_dir_name'] == 'ServerDirectoryCleanup' From 0af1325a1380a652baaa693c3e78b56340ace311 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Fri, 11 Nov 2022 09:32:46 -0700 Subject: [PATCH 36/37] Fix missing os_hes_dir variable in some cases --- buildstockbatch/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildstockbatch/base.py b/buildstockbatch/base.py index 9a5b06ed..088b0933 100644 --- a/buildstockbatch/base.py +++ b/buildstockbatch/base.py @@ -65,12 +65,12 @@ def __init__(self, project_filename): self.project_dir = os.path.join(self.buildstock_dir, self.cfg['project_directory']) if not os.path.isdir(self.project_dir): raise FileNotFoundError(f'project_directory = {self.project_dir} is not a directory.') + + self.os_hescore_dir = None if 'build_existing_model' in self.cfg['workflow_generator']['args']: if 'os_hescore_directory' in self.cfg['workflow_generator']['args']['build_existing_model']: self.os_hescore_dir = path_rel_to_file( project_filename, self.cfg['workflow_generator']['args']['build_existing_model']['os_hescore_directory']) - else: - self.os_hescore_dir = None # Load in OS_VERSION and OS_SHA arguments if they exist in the YAML, # otherwise use defaults specified here. From d8fa15f5cdc531270009970d50f02fe3e7c0c394 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Mon, 28 Nov 2022 19:17:58 -0700 Subject: [PATCH 37/37] remove debug statement --- buildstockbatch/workflow_generator/residential_hpxml_hes.py | 1 - project_resstock_national_hes.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/buildstockbatch/workflow_generator/residential_hpxml_hes.py b/buildstockbatch/workflow_generator/residential_hpxml_hes.py index eb2b9157..1806beb2 100644 --- a/buildstockbatch/workflow_generator/residential_hpxml_hes.py +++ b/buildstockbatch/workflow_generator/residential_hpxml_hes.py @@ -32,5 +32,4 @@ def get_yml_schema(cls): # Require os_hescore_directory argument string_validator = yamale.validators.String(required=True) schema.includes['build-existing-model-spec'].dict['os_hescore_directory'] = string_validator - print(schema.includes['build-existing-model-spec'].dict) return schema diff --git a/project_resstock_national_hes.yml b/project_resstock_national_hes.yml index f723c2fb..ceb4f4dc 100644 --- a/project_resstock_national_hes.yml +++ b/project_resstock_national_hes.yml @@ -1,6 +1,6 @@ ###### FIXME: Temporary yml for testing, delete before merge schema_version: '0.3' -os_version: 3.4.0 +os_version: 3.5.0 os_sha: 4bd816f785 buildstock_directory: ../resstock # Relative to this file or absolute project_directory: project_national # Relative to buildstock_directory