ResStock-HPXML: HES Workflow Generator

buildstockbatch/aws/awsbase.py

@@ -104,9 +104,9 @@ def delete_role(self, role_name):
 
             for policy in response['AttachedPolicies']:
                 self.iam.detach_role_policy(
-                        RoleName=role_name,
-                        PolicyArn=policy['PolicyArn']
-                    )
+                    RoleName=role_name,
+                    PolicyArn=policy['PolicyArn']
+                )
 
             logger.info(f'Policies detached from role {role_name}.')
 

buildstockbatch/base.py

@@ -66,6 +66,12 @@ def __init__(self, project_filename):
         if not os.path.isdir(self.project_dir):
             raise FileNotFoundError(f'project_directory = {self.project_dir} is not a directory.')
 
+        self.os_hescore_dir = None
+        if 'build_existing_model' in self.cfg['workflow_generator']['args']:
+            if 'os_hescore_directory' in self.cfg['workflow_generator']['args']['build_existing_model']:
+                self.os_hescore_dir = path_rel_to_file(
+                    project_filename, self.cfg['workflow_generator']['args']['build_existing_model']['os_hescore_directory'])
+
         # Load in OS_VERSION and OS_SHA arguments if they exist in the YAML,
         # otherwise use defaults specified here.
         self.os_version = self.cfg.get('os_version', self.DEFAULT_OS_VERSION)
@@ -737,7 +743,7 @@ def get_dask_client(self):
     def process_results(self, skip_combine=False, force_upload=False):
         self.get_dask_client()  # noqa: F841
 
-        if self.cfg['workflow_generator']['type'] == 'residential_hpxml':
+        if 'residential_hpxml' in self.cfg['workflow_generator']['type']:
             if 'simulation_output_report' in self.cfg['workflow_generator']['args'].keys():
                 if 'timeseries_frequency' in self.cfg['workflow_generator']['args']['simulation_output_report'].keys():
                     do_timeseries = \

buildstockbatch/eagle.py

@@ -103,10 +103,10 @@ def singularity_image_url(self):
         else:
             prefix_ver = self.os_version
         return 'https://s3.amazonaws.com/openstudio-builds/{prefix_ver}/OpenStudio-{ver}.{sha}-Singularity.simg'.format(
-                    prefix_ver=prefix_ver,
-                    ver=self.os_version,
-                    sha=self.os_sha
-                )
+            prefix_ver=prefix_ver,
+            ver=self.os_version,
+            sha=self.os_sha
+        )
 
     @property
     def singularity_image(self):
@@ -653,35 +653,35 @@ def rerun_failed_jobs(self, hipri=False):
 
 
 logging_config = {
-        'version': 1,
-        'disable_existing_loggers': True,
-        'formatters': {
-            'defaultfmt': {
-                'format': '%(levelname)s:%(asctime)s:%(name)s:%(message)s',
-                'datefmt': '%Y-%m-%d %H:%M:%S'
-            }
-        },
-        'handlers': {
-            'console': {
-                'class': 'logging.StreamHandler',
-                'formatter': 'defaultfmt',
-                'level': 'DEBUG',
-                'stream': 'ext://sys.stdout',
-            }
-        },
-        'loggers': {
-            '__main__': {
-                'level': 'DEBUG',
-                'propagate': True,
-                'handlers': ['console']
-            },
-            'buildstockbatch': {
-                'level': 'DEBUG',
-                'propagate': True,
-                'handlers': ['console']
-            }
+    'version': 1,
+    'disable_existing_loggers': True,
+    'formatters': {
+        'defaultfmt': {
+            'format': '%(levelname)s:%(asctime)s:%(name)s:%(message)s',
+            'datefmt': '%Y-%m-%d %H:%M:%S'
+        }
+    },
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+            'formatter': 'defaultfmt',
+            'level': 'DEBUG',
+            'stream': 'ext://sys.stdout',
+        }
+    },
+    'loggers': {
+        '__main__': {
+            'level': 'DEBUG',
+            'propagate': True,
+            'handlers': ['console']
         },
-    }
+        'buildstockbatch': {
+            'level': 'DEBUG',
+            'propagate': True,
+            'handlers': ['console']
+        }
+    },
+}
 
 
 def user_cli(argv=sys.argv[1:]):

buildstockbatch/localdocker.py

@@ -31,6 +31,7 @@
 from buildstockbatch.base import BuildStockBatchBase, SimulationExists
 from buildstockbatch import postprocessing
 from .utils import log_error_details, ContainerRuntime
+from docker.errors import ImageNotFound
 from buildstockbatch.__version__ import __version__ as bsb_version
 
 logger = logging.getLogger(__name__)
@@ -52,22 +53,41 @@ def __init__(self, project_filename):
 
         self._weather_dir = None
 
+        # FIXME: something better and more general than os_hescore_dir
+        if self.os_hescore_dir:
+            logger.debug("Building HEScore docker image")
+            self.docker_client.images.build(
+                path=self.os_hescore_dir,
+                tag=self.docker_image,
+                dockerfile="resstock/Dockerfile",
+                rm=True,
+                buildargs={'os_version': self.os_version}
+            )
+            logger.debug("Docker image built")
+        else:
+            try:
+                self.docker_client.images.get(self.docker_image)
+            except ImageNotFound:
+                self.docker_client.images.pull(self.docker_image)
+
     @staticmethod
     def validate_project(project_file):
         super(DockerBatchBase, DockerBatchBase).validate_project(project_file)
         # LocalDocker specific code goes here
 
     @property
     def docker_image(self):
-        return 'nrel/openstudio:{}'.format(self.os_version)
+        if self.os_hescore_dir:
+            return 'nrel/hescore-hpxml-openstudio'
+        else:
+            return 'nrel/openstudio:{}'.format(self.os_version)
 
 
 class LocalDockerBatch(DockerBatchBase):
 
     def __init__(self, project_filename):
         super().__init__(project_filename)
         logger.debug(f'Pulling docker image: {self.docker_image}')
-        self.docker_client.images.pull(self.docker_image)
 
         # Create simulation_output dir
         sim_out_ts_dir = os.path.join(self.results_dir, 'simulation_output', 'timeseries')
@@ -149,28 +169,44 @@ def weather_dir(self):
         return self._weather_dir.name
 
     @classmethod
-    def run_building(cls, project_dir, buildstock_dir, weather_dir, docker_image, results_dir, measures_only,
-                     n_datapoints, cfg, i, upgrade_idx=None):
+    def run_building(cls, project_dir, buildstock_dir, weather_dir, os_hescore_dir, docker_image, results_dir,
+                     measures_only, n_datapoints, cfg, i, upgrade_idx=None):
 
         upgrade_id = 0 if upgrade_idx is None else upgrade_idx + 1
 
         try:
             sim_id, sim_dir = cls.make_sim_dir(i, upgrade_idx, os.path.join(results_dir, 'simulation_output'))
         except SimulationExists:
             return
-
         bind_mounts = [
             (sim_dir, '', 'rw'),
             (os.path.join(buildstock_dir, 'measures'), 'measures', 'ro'),
             (os.path.join(buildstock_dir, 'resources'), 'lib/resources', 'ro'),
             (os.path.join(project_dir, 'housing_characteristics'), 'lib/housing_characteristics', 'ro'),
             (weather_dir, 'weather', 'ro')
         ]
+
+        # ResStock-hpxml measure directory
         if os.path.exists(os.path.join(buildstock_dir, 'resources', 'hpxml-measures')):
-            bind_mounts += [(os.path.join(buildstock_dir, 'resources', 'hpxml-measures'),
-                            'resources/hpxml-measures', 'ro')]
-        docker_volume_mounts = dict([(key, {'bind': f'/var/simdata/openstudio/{bind}', 'mode': mode}) for key, bind, mode in bind_mounts])  # noqa E501
+            bind_mounts.append(
+                (os.path.join(buildstock_dir, 'resources', 'hpxml-measures'), 'resources/hpxml-measures', 'ro')
+            )
+
+        # OS-HEScore weather directory
+        if os_hescore_dir and os.path.exists(os_hescore_dir):
+            bind_mounts.append((os.path.join(os_hescore_dir, 'weather'), '/opt/OpenStudio-HEScore/weather', 'ro'))
+
+        docker_volume_mounts = {
+            key: {
+                'bind': bind if bind.startswith('/') else f'/var/simdata/openstudio/{bind}',
+                'mode': mode
+            }
+            for key, bind, mode in bind_mounts
+        }
+
         for bind in bind_mounts:
+            if bind[1].startswith('/'):
+                continue
             dir_to_make = os.path.join(sim_dir, *bind[1].split('/'))
             if not os.path.exists(dir_to_make):
                 os.makedirs(dir_to_make)
@@ -211,17 +247,20 @@ def run_building(cls, project_dir, buildstock_dir, weather_dir, docker_image, re
         extra_kws = {}
         if sys.platform.startswith('linux'):
             extra_kws['user'] = f'{os.getuid()}:{os.getgid()}'
-        container_output = docker_client.containers.run(
+
+        container = docker_client.containers.run(
             docker_image,
             run_cmd,
             remove=True,
+            detach=True,
             volumes=docker_volume_mounts,
             name=sim_id,
             environment=env_vars,
             **extra_kws
         )
         with open(os.path.join(sim_dir, 'docker_output.log'), 'wb') as f_out:
-            f_out.write(container_output)
+            for x in container.logs(stream=True):
+                f_out.write(x)
 
         # Clean up directories created with the docker mounts
         for dirname in ('lib', 'measures', 'weather'):
@@ -255,6 +294,7 @@ def run_batch(self, n_jobs=None, measures_only=False, sampling_only=False):
             self.project_dir,
             self.buildstock_dir,
             self.weather_dir,
+            self.os_hescore_dir,
             self.docker_image,
             self.results_dir,
             measures_only,
@@ -281,6 +321,7 @@ def run_batch(self, n_jobs=None, measures_only=False, sampling_only=False):
             json.dump(dpouts, f)
         del dpouts
 
+        # FIXME temporarily comment out for testing
         sim_out_tarfile_name = os.path.join(sim_out_dir, 'simulations_job0.tar.gz')
         logger.debug(f'Compressing simulation outputs to {sim_out_tarfile_name}')
         with tarfile.open(sim_out_tarfile_name, 'w:gz') as tarf: