From 9c74e1a923829ae43b942ad293384444db430360 Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Tue, 5 Nov 2024 16:36:21 -0600 Subject: [PATCH 1/7] Remove eagle related code and reference since eagle is retired. --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/pull_request_template.md | 2 +- buildstockbatch/eagle.sh | 16 ------- buildstockbatch/eagle_postprocessing.sh | 33 --------------- buildstockbatch/gcp/gcp.py | 2 +- buildstockbatch/hpc.py | 55 +++---------------------- buildstockbatch/postprocessing.py | 2 +- buildstockbatch/schemas/v0.4.yaml | 3 +- buildstockbatch/test/conftest.py | 2 +- buildstockbatch/test/test_hpc.py | 42 +++++++------------ buildstockbatch/test/test_validation.py | 34 +++------------ docs/conf.py | 4 +- docs/installation.rst | 53 ------------------------ docs/project_defn.rst | 42 ++++--------------- docs/run_sims.rst | 17 ++++---- setup.py | 1 - 16 files changed, 49 insertions(+), 261 deletions(-) delete mode 100644 buildstockbatch/eagle.sh delete mode 100644 buildstockbatch/eagle_postprocessing.sh diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 5061ffa2..8b667f12 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -29,7 +29,7 @@ log file here ``` **Platform (please complete the following information):** -- Simulation platform: [e.g. Kestrel, Eagle, AWS, local docker; please label with this as well] +- Simulation platform: [e.g. Kestrel, AWS, local docker; please label with this as well] - BuildStockBatch version, branch, or sha: - resstock or comstock repo version, branch, or sha: - Local Desktop OS: [e.g. Windows, Mac, Linux, especially important if running locally] diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 5794d24f..f5d064fe 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -14,5 +14,5 @@ Not all may apply - [ ] All other unit and integration tests passing - [ ] Update validation for project config yaml file changes - [ ] Update existing documentation -- [ ] Run a small batch run on Kestrel/Eagle to make sure it all works if you made changes that will affect Kestrel/Eagle +- [ ] Run a small batch run on Kestrel to make sure it all works if you made changes that will affect Kestrel - [ ] Add to the changelog_dev.rst file and propose migration text in the pull request diff --git a/buildstockbatch/eagle.sh b/buildstockbatch/eagle.sh deleted file mode 100644 index 8a5aca08..00000000 --- a/buildstockbatch/eagle.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -#SBATCH --nodes=1 -#SBATCH --ntasks=1 - -echo "Job ID: $SLURM_JOB_ID" -echo "Hostname: $HOSTNAME" -echo "QOS: $SLURM_JOB_QOS" - -df -i -df -h - -module load conda singularity-container -source activate "$MY_CONDA_ENV" -source /shared-projects/buildstock/aws_credentials.sh - -time python -u -m buildstockbatch.hpc eagle "$PROJECTFILE" diff --git a/buildstockbatch/eagle_postprocessing.sh b/buildstockbatch/eagle_postprocessing.sh deleted file mode 100644 index 26f9b638..00000000 --- a/buildstockbatch/eagle_postprocessing.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -echo "begin eagle_postprocessing.sh" - -echo "Job ID: $SLURM_JOB_ID" -echo "Hostname: $HOSTNAME" - -df -i -df -h - -module load conda singularity-container -source activate "$MY_CONDA_ENV" -source /shared-projects/buildstock/aws_credentials.sh - -export POSTPROCESS=1 - -echo "UPLOADONLY: ${UPLOADONLY}" -echo "MEMORY: ${MEMORY}" -echo "NPROCS: ${NPROCS}" - -SCHEDULER_FILE=$OUT_DIR/dask_scheduler.json - -echo "head node" -echo $SLURM_JOB_NODELIST_PACK_GROUP_0 -echo "workers" -echo $SLURM_JOB_NODELIST_PACK_GROUP_1 - -pdsh -w $SLURM_JOB_NODELIST_PACK_GROUP_1 "free -h" -pdsh -w $SLURM_JOB_NODELIST_PACK_GROUP_1 "df -i; df -h" - -$MY_CONDA_ENV/bin/dask scheduler --scheduler-file $SCHEDULER_FILE &> $OUT_DIR/dask_scheduler.out & -pdsh -w $SLURM_JOB_NODELIST_PACK_GROUP_1 "source /shared-projects/buildstock/aws_credentials.sh; $MY_CONDA_ENV/bin/dask worker --scheduler-file $SCHEDULER_FILE --local-directory /tmp/scratch/dask --nworkers ${NPROCS} --nthreads 1 --memory-limit ${MEMORY}MB" &> $OUT_DIR/dask_workers.out & - -time python -u -m buildstockbatch.hpc eagle "$PROJECTFILE" diff --git a/buildstockbatch/gcp/gcp.py b/buildstockbatch/gcp/gcp.py index 18a40137..4f27b00c 100644 --- a/buildstockbatch/gcp/gcp.py +++ b/buildstockbatch/gcp/gcp.py @@ -807,7 +807,7 @@ def process_results(self, skip_combine=False, use_dask_cluster=True): Storage. The BSB implementation tries to write both indirectly (via `postprocessing.combine_results()`, using `get_fs()`), and directly (through `upload_results`). Which way the results end up on S3 depends on whether the script was run - via aws.py (indirect write), or locally or Eagle (direct upload). + via aws.py (indirect write), or locally or Kestrel (direct upload). Here, where writing to GCS is (currently) coupled to running on GCS, the writing to GCS will happen indirectly (via `postprocessing.combine_results()`), and we don't need to diff --git a/buildstockbatch/hpc.py b/buildstockbatch/hpc.py index e13ea3cb..b3f3c58c 100644 --- a/buildstockbatch/hpc.py +++ b/buildstockbatch/hpc.py @@ -3,7 +3,7 @@ """ buildstockbatch.hpc ~~~~~~~~~~~~~~~ -This class contains the object & methods that allow for usage of the library with Eagle and Kestrel +This class contains the object & methods that allow for usage of the library with Kestrel :author: Noel Merket :copyright: (c) 2018 by The Alliance for Sustainable Energy @@ -732,38 +732,6 @@ def rerun_failed_jobs(self, hipri=False): self.queue_post_processing(job_ids, hipri=hipri) -class EagleBatch(SlurmBatch): - DEFAULT_SYS_IMAGE_DIR = "/shared-projects/buildstock/singularity_images" - HPC_NAME = "eagle" - CORES_PER_NODE = 36 - MIN_SIMS_PER_JOB = 36 * 2 - DEFAULT_POSTPROCESSING_NODE_MEMORY_MB = 85248 - DEFAULT_NODE_MEMORY_MB = 85248 # standard node on Eagle - DEFAULT_POSTPROCESSING_N_PROCS = 18 - DEFAULT_POSTPROCESSING_N_WORKERS = 2 - - @classmethod - def validate_output_directory_eagle(cls, project_file): - cfg = get_project_configuration(project_file) - output_dir = path_rel_to_file(project_file, cfg["output_directory"]) - if not re.match(r"/(lustre/eaglefs/)?(scratch|projects)", output_dir): - raise ValidationError( - f"`output_directory` must be in /scratch or /projects," f" `output_directory` = {output_dir}" - ) - - @classmethod - def validate_project(cls, project_file): - super(cls, cls).validate_project(project_file) - cls.validate_output_directory_eagle(project_file) - logger.info("Eagle Validation Successful") - return True - - @staticmethod - def _queue_jobs_env_vars() -> dict: - env = {"MY_CONDA_ENV": os.environ["CONDA_PREFIX"]} - return env - - class KestrelBatch(SlurmBatch): DEFAULT_SYS_IMAGE_DIR = "/kfs2/shared-projects/buildstock/apptainer_images" HPC_NAME = "kestrel" @@ -824,17 +792,13 @@ def _queue_jobs_env_vars() -> dict: } -def eagle_cli(argv=sys.argv[1:]): - user_cli(EagleBatch, argv) - - def kestrel_cli(argv=sys.argv[1:]): user_cli(KestrelBatch, argv) def user_cli(Batch: SlurmBatch, argv: list): """ - This is the user entry point for running buildstockbatch on Eagle/Kestrel + This is the user entry point for running buildstockbatch on Kestrel """ # set up logging, currently based on within-this-file hard-coded config logging.config.dictConfig(logging_config) @@ -916,7 +880,7 @@ def main(): - upload results to Athena (job_array_number == 0 and POSTPROCESS and UPLOADONLY) The context for the work is deinfed by the project_filename (project .yml file), - which is used to initialize an EagleBatch object. + which is used to initialize an KestrelBatch object. """ # set up logging, currently based on within-this-file hard-coded config @@ -924,16 +888,11 @@ def main(): # only direct script argument is the project .yml file parser = argparse.ArgumentParser() - parser.add_argument("hpc_name", choices=["eagle", "kestrel"]) parser.add_argument("project_filename") args = parser.parse_args() - # initialize the EagleBatch/KestrelBatch object - if args.hpc_name == "eagle": - batch = EagleBatch(args.project_filename) - else: - assert args.hpc_name == "kestrel" - batch = KestrelBatch(args.project_filename) + # initialize the KestrelBatch object + batch = KestrelBatch(args.project_filename) # other arguments/cues about which part of the process we are in are # encoded in slurm job environment variables job_array_number = int(os.environ.get("SLURM_ARRAY_TASK_ID", 0)) @@ -966,9 +925,7 @@ def main(): if __name__ == "__main__": bsb_cli = os.environ.get("BUILDSTOCKBATCH_CLI") - if bsb_cli == "eagle": - eagle_cli() - elif bsb_cli == "kestrel": + if bsb_cli == "kestrel": kestrel_cli() else: main() diff --git a/buildstockbatch/postprocessing.py b/buildstockbatch/postprocessing.py index 4461cc2e..09fb339c 100644 --- a/buildstockbatch/postprocessing.py +++ b/buildstockbatch/postprocessing.py @@ -537,7 +537,7 @@ def combine_results(fs, results_dir, cfg, do_timeseries=True): # Determine how many files should be in each partition and group the files parquet_memory = int( - cfg.get("eagle", {}).get("postprocessing", {}).get("parquet_memory_mb", MAX_PARQUET_MEMORY) + cfg.get("kestrel", {}).get("postprocessing", {}).get("parquet_memory_mb", MAX_PARQUET_MEMORY) ) logger.info(f"Max parquet memory: {parquet_memory} MB") max_files_per_partition = max(1, math.floor(parquet_memory / (mean_mem / 1e6))) diff --git a/buildstockbatch/schemas/v0.4.yaml b/buildstockbatch/schemas/v0.4.yaml index d199b725..01263d55 100644 --- a/buildstockbatch/schemas/v0.4.yaml +++ b/buildstockbatch/schemas/v0.4.yaml @@ -1,11 +1,10 @@ -schema_version: enum('0.4') +schema_version: enum('0.5') buildstock_directory: str() project_directory: str(required=True) weather_files_path: str(required=False) weather_files_url: str(required=False) sampler: include('sampler-spec', required=True) workflow_generator: include('workflow-generator-spec', required=True) -eagle: include('hpc-spec', required=False) kestrel: include('hpc-spec', required=False) gcp: include('gcp-spec', required=False) aws: include('aws-spec', required=False) diff --git a/buildstockbatch/test/conftest.py b/buildstockbatch/test/conftest.py index 97f61f26..b3fa8f3b 100644 --- a/buildstockbatch/test/conftest.py +++ b/buildstockbatch/test/conftest.py @@ -11,7 +11,7 @@ def basic_residential_project_file(): with tempfile.TemporaryDirectory() as test_directory: - def _basic_residential_project_file(update_args={}, raw=False, hpc_name="eagle"): + def _basic_residential_project_file(update_args={}, raw=False, hpc_name="kestrel"): output_dir = "simulations_job0" if raw else "simulation_output" buildstock_directory = os.path.join(test_directory, "openstudio_buildstock") shutil.copytree( diff --git a/buildstockbatch/test/test_hpc.py b/buildstockbatch/test/test_hpc.py index 967596b3..4ca823f6 100644 --- a/buildstockbatch/test/test_hpc.py +++ b/buildstockbatch/test/test_hpc.py @@ -9,7 +9,7 @@ from unittest.mock import patch import gzip -from buildstockbatch.hpc import eagle_cli, kestrel_cli, EagleBatch, KestrelBatch, SlurmBatch # noqa: F401 +from buildstockbatch.hpc import kestrel_cli, KestrelBatch, SlurmBatch # noqa: F401 from buildstockbatch.base import BuildStockBatchBase from buildstockbatch.utils import get_project_configuration, read_csv @@ -95,11 +95,8 @@ def test_hpc_run_building(mock_subprocess, monkeypatch, basic_residential_projec def _test_env_vars_passed(mock_subprocess, hpc_name): env_vars_to_check = ["PROJECTFILE", "MEASURESONLY", "SAMPLINGONLY"] - if hpc_name == "eagle": - env_vars_to_check.append("MY_CONDA_ENV") - else: - assert hpc_name == "kestrel" - env_vars_to_check.append("MY_PYTHON_ENV") + assert hpc_name == "kestrel" + env_vars_to_check.append("MY_PYTHON_ENV") export_found = False for arg in mock_subprocess.run.call_args[0][0]: if arg.startswith("--export"): @@ -110,7 +107,7 @@ def _test_env_vars_passed(mock_subprocess, hpc_name): assert exported_env_vars.issuperset(env_vars_to_check) -@pytest.mark.parametrize("hpc_name", ["eagle", "kestrel"]) +@pytest.mark.parametrize("hpc_name", ["kestrel"]) def test_user_cli(basic_residential_project_file, monkeypatch, mocker, hpc_name): mock_subprocess = mocker.patch("buildstockbatch.hpc.subprocess") mock_validate_apptainer_image = mocker.patch("buildstockbatch.hpc.SlurmBatch.validate_apptainer_image_hpc") @@ -125,13 +122,9 @@ def test_user_cli(basic_residential_project_file, monkeypatch, mocker, hpc_name) project_filename, results_dir = basic_residential_project_file(hpc_name=hpc_name) shutil.rmtree(results_dir) - if hpc_name == "eagle": - monkeypatch.setenv("CONDA_PREFIX", "something") - cli = eagle_cli - else: - assert hpc_name == "kestrel" - monkeypatch.setenv("VIRTUAL_ENV", "something") - cli = kestrel_cli + assert hpc_name == "kestrel" + monkeypatch.setenv("VIRTUAL_ENV", "something") + cli = kestrel_cli argv = [project_filename] cli(argv) mock_subprocess.run.assert_called_once() @@ -188,7 +181,7 @@ def test_user_cli(basic_residential_project_file, monkeypatch, mocker, hpc_name) assert "0" == mock_subprocess.run.call_args[1]["env"]["MEASURESONLY"] -@pytest.mark.parametrize("hpc_name", ["eagle", "kestrel"]) +@pytest.mark.parametrize("hpc_name", ["kestrel"]) def test_qos_high_job_submit(basic_residential_project_file, monkeypatch, mocker, hpc_name): mock_subprocess = mocker.patch("buildstockbatch.hpc.subprocess") mock_subprocess.run.return_value.stdout = "Submitted batch job 1\n" @@ -198,11 +191,8 @@ def test_qos_high_job_submit(basic_residential_project_file, monkeypatch, mocker mocker.patch.object(SlurmBatch, "weather_dir", None) project_filename, results_dir = basic_residential_project_file(hpc_name=hpc_name) shutil.rmtree(results_dir) - if hpc_name == "eagle": - monkeypatch.setenv("CONDA_PREFIX", "something") - else: - assert hpc_name == "kestrel" - monkeypatch.setenv("VIRTUAL_ENV", "something") + assert hpc_name == "kestrel" + monkeypatch.setenv("VIRTUAL_ENV", "something") monkeypatch.setenv("SLURM_JOB_QOS", "high") batch = Batch(project_filename) @@ -224,7 +214,7 @@ def test_qos_high_job_submit(basic_residential_project_file, monkeypatch, mocker assert "--qos=high" in mock_subprocess.run.call_args[0][0] -@pytest.mark.parametrize("hpc_name", ["eagle", "kestrel"]) +@pytest.mark.parametrize("hpc_name", ["kestrel"]) def test_queue_jobs_minutes_per_sim(mocker, basic_residential_project_file, monkeypatch, hpc_name): mock_subprocess = mocker.patch("buildstockbatch.hpc.subprocess") Batch = eval(f"{hpc_name.capitalize()}Batch") @@ -242,11 +232,9 @@ def test_queue_jobs_minutes_per_sim(mocker, basic_residential_project_file, monk } ) shutil.rmtree(results_dir) - if hpc_name == "eagle": - monkeypatch.setenv("CONDA_PREFIX", "something") - else: - assert hpc_name == "kestrel" - monkeypatch.setenv("VIRTUAL_ENV", "something") + + assert hpc_name == "kestrel" + monkeypatch.setenv("VIRTUAL_ENV", "something") batch = Batch(project_filename) for i in range(1, 11): @@ -255,7 +243,7 @@ def test_queue_jobs_minutes_per_sim(mocker, basic_residential_project_file, monk json.dump({"batch": list(range(1000))}, f) batch.queue_jobs() mock_subprocess.run.assert_called_once() - n_minutes = 14 if hpc_name == "eagle" else 5 + n_minutes = 5 assert f"--time={n_minutes}" in mock_subprocess.run.call_args[0][0] diff --git a/buildstockbatch/test/test_validation.py b/buildstockbatch/test/test_validation.py index b224b83c..42abc5c8 100644 --- a/buildstockbatch/test/test_validation.py +++ b/buildstockbatch/test/test_validation.py @@ -17,7 +17,7 @@ import tempfile import json import pathlib -from buildstockbatch.hpc import EagleBatch, SlurmBatch, KestrelBatch +from buildstockbatch.hpc import SlurmBatch, KestrelBatch from buildstockbatch.aws.aws import AwsBatch from buildstockbatch.local import LocalBatch from buildstockbatch.base import BuildStockBatchBase, ValidationError @@ -53,8 +53,8 @@ def test_base_schema_validation_is_static(): assert isinstance(BuildStockBatchBase.validate_project_schema, types.FunctionType) -def test_eagle_validation_is_classmethod(): - assert inspect.ismethod(EagleBatch.validate_project) +def test_kestrel_validation_is_classmethod(): + assert inspect.ismethod(KestrelBatch.validate_project) def test_local_docker_validation_is_classmethod(): @@ -106,7 +106,7 @@ def test_xor_violations_fail(project_file, expected): @pytest.mark.parametrize( - "project_file, base_expected, eagle_expected", + "project_file, base_expected, kestrel_expected", [ ( os.path.join(example_yml_dir, "missing-required-schema.yml"), @@ -127,7 +127,7 @@ def test_xor_violations_fail(project_file, expected): (os.path.join(example_yml_dir, "minimal-schema.yml"), True, ValidationError), ], ) -def test_validation_integration(project_file, base_expected, eagle_expected): +def test_validation_integration(project_file, base_expected, kestrel_expected): # patch the validate_options_lookup function to always return true for this case with patch.object(BuildStockBatchBase, "validate_options_lookup", lambda _: True), patch.object( BuildStockBatchBase, "validate_measure_references", lambda _: True @@ -138,7 +138,7 @@ def test_validation_integration(project_file, base_expected, eagle_expected): ): for cls, expected in [ (BuildStockBatchBase, base_expected), - (EagleBatch, eagle_expected), + (KestrelBatch, kestrel_expected), ]: if expected is not True: with pytest.raises(expected): @@ -495,28 +495,6 @@ def test_dask_config(): with pytest.raises(ValidationError, match=r"needs to be a multiple of 1024"): AwsBatch.validate_dask_settings(test3_filename) - -def test_validate_eagle_output_directory(): - minimal_yml = pathlib.Path(example_yml_dir, "minimal-schema.yml") - with pytest.raises(ValidationError, match=r"must be in /scratch or /projects"): - EagleBatch.validate_output_directory_eagle(str(minimal_yml)) - with tempfile.TemporaryDirectory() as tmpdir: - dirs_to_try = [ - "/scratch/username/out_dir", - "/projects/projname/out_dir", - "/lustre/eaglefs/scratch/username/out_dir", - "/lustre/eaglefs/projects/projname/out_dir", - ] - for output_directory in dirs_to_try: - with open(minimal_yml, "r") as f: - cfg = yaml.load(f, Loader=yaml.SafeLoader) - cfg["output_directory"] = output_directory - temp_yml = pathlib.Path(tmpdir, "temp.yml") - with open(temp_yml, "w") as f: - yaml.dump(cfg, f, Dumper=yaml.SafeDumper) - EagleBatch.validate_output_directory_eagle(str(temp_yml)) - - def test_validate_kestrel_output_directory(): minimal_yml = pathlib.Path(example_yml_dir, "minimal-schema.yml") with pytest.raises(ValidationError, match=r"must be in /scratch or /projects"): diff --git a/docs/conf.py b/docs/conf.py index 94ca7931..57ee43c4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,7 +60,7 @@ "sampler", "workflow", "resstock", - "eagle", + "hpc", "local", "aws", "postprocessing", @@ -228,7 +228,7 @@ nitpick_ignore = [ ("py:func", "BuildStockBatchBase.validate_precomputed_sample"), ("py:func", "BuildStockBatchBase.validate_xor_nor_schema_keys"), - ("py:func", "EagleBatch.run_building"), + ("py:func", "KestrelBatch.run_building"), ("py:class", "sampler.CommercialSobolSingularitySampler"), ("py:class", "sampler.CommercialSobolDockerSampler"), ("py:class", "workflow_generator.CommercialDefaultWorkflowGenerator"), diff --git a/docs/installation.rst b/docs/installation.rst index 435adcf1..13243160 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -175,59 +175,6 @@ The ``-d`` and ``-e`` flags can also be combined if desired bash create_kestrel_env.sh -d -e /projects/enduse/envs mydevenv -.. _eagle_install: - -Eagle -~~~~~ - -buildstockbatch is preinstalled on Eagle. To use it, `ssh into Eagle`_, -activate the appropriate conda environment: - -.. _ssh into Eagle: https://www.nrel.gov/hpc/eagle-user-basics.html - -:: - - module load conda - source activate /shared-projects/buildstock/envs/buildstock-X.X - -You can get a list of installed environments by looking in the envs directory - -:: - - ls /shared-projects/buildstock/envs - -Developer installation -...................... - -For those doing development work on buildstockbatch (not most users), a new -conda environment that includes buildstockbatch is created with the bash -script `create_eagle_env.sh` in the git repo that will need to be cloned onto -Eagle. The script is called as follows: - -:: - - bash create_eagle_env.sh env-name - -This will create a directory ``/shared-projects/buildstock/envs/env-name`` that -contains the conda environment with BuildStock Batch installed. This environment -can then be used by any user. - -If you pass the ``-d`` flag to that script, it will install the buildstockbatch -package in development mode meaning that any changes you make in your cloned -repo will immediately be available to that environment. However, it means that -only the user who installed the environment can use it. - -If you pass the flag ``-e /projects/someproject/envs``, it will install the -environment there instead of the default location. This is useful if you need a -specific installation for a particular project. - -The ``-d`` and ``-e`` flags can also be combined if desired - -:: - - bash create_eagle_env.sh -d -e /projects/enduse/envs mydevenv - - Amazon Web Services (Beta) ~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/project_defn.rst b/docs/project_defn.rst index d57aa7da..3863ba48 100644 --- a/docs/project_defn.rst +++ b/docs/project_defn.rst @@ -91,12 +91,12 @@ Information about baseline simulations are listed under the ``baseline`` key. - ``skip_sims``: Include this key to control whether the set of baseline simulations are run. The default (i.e., when this key is not included) is to run all the baseline simulations. No results csv table with baseline characteristics will be provided when the baseline simulations are skipped. -- ``custom_gems``: true or false. **ONLY WORKS ON EAGLE, KESTREL, AND LOCAL** +- ``custom_gems``: true or false. **ONLY WORKS ON KESTREL AND LOCAL** When true, buildstockbatch will call the OpenStudio CLI commands with the ``bundle`` and ``bundle_path`` options. These options tell the CLI to load a custom set of gems rather than those included in the OpenStudio CLI. For both - Eagle, Kestrel, and local Docker runs, these gems are first specified in the - ``buildstock\resources\Gemfile``. For Eagle, Kestrel, when the apptainer image + Kestrel and local Docker runs, these gems are first specified in the + ``buildstock\resources\Gemfile``. For Kestrel, when the apptainer image is built, these gems are added to the image. For local Docker, when the containers are started, the gems specified in the Gemfile are installed into a Docker volume on the local computer. This volume is mounted by each container @@ -154,34 +154,6 @@ Output Directory ``output_directory``: specifies where the outputs of the simulation should be stored. The last folder in the path will be used as the table name in Athena (if aws configuration is present under postprocessing) so needs to be lowercase, start from letters and contain only letters, numbers and underscore character. `Athena requirement. `_ -.. _eagle-config: - -Eagle Configuration -~~~~~~~~~~~~~~~~~~~ - -Under the ``eagle`` key is a list of configuration for running the batch job on -the Eagle supercomputer. - -* ``n_jobs``: Number of eagle jobs to parallelize the simulation into -* ``minutes_per_sim``: Required. Maximum allocated simulation time in minutes. -* ``account``: Required. Eagle allocation account to charge the job to. -* ``sampling``: Configuration for the sampling in eagle - - * ``time``: Maximum time in minutes to allocate to sampling job - -* ``postprocessing``: Eagle configuration for the postprocessing step - - * ``time``: Maximum time in minutes to allocate postprocessing job - * ``n_workers``: Number of eagle nodes to parallelize the postprocessing - job into. Max supported is 32. Default is 2. - * ``n_procs``: Number of CPUs to use within each eagle nodes. Max is 36. - Default is 18. Try reducing this if you get OOM error. - * ``node_memory_mb``: The memory (in MB) to request for eagle node for - postprocessing. The valid values are 85248, 180224 and 751616. Default is - 85248. - * ``parquet_memory_mb``: The size (in MB) of the combined parquet file in - memory. Default is 1000. - .. _kestrel-config: Kestrel Configuration @@ -290,7 +262,7 @@ using `GCP Batch `_ and `Cloud Run ` * `Detailed instructions from AWS `_ diff --git a/docs/run_sims.rst b/docs/run_sims.rst index 16466630..281a9c75 100644 --- a/docs/run_sims.rst +++ b/docs/run_sims.rst @@ -20,13 +20,10 @@ Running a project file is straightforward. Call the ``buildstock_local`` command Running the simulation with ``--postprocessonly`` when there is already postprocessed results from previous run will overwrite those results. -NREL HPC (Eagle or Kestrel) -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -After you have :ref:`activated the appropriate conda environment on Eagle `, -you can submit a project file to be simulated by passing it to the ``buildstock_eagle`` command. - -.. command-output:: buildstock_eagle --help - :ellipsis: 0,8 +NREL HPC (Kestrel) +~~~~~~~~~~~~~~~~~~~ +After you have :ref:`activated the appropriate conda environment on Kestrel `, +you can submit a project file to be simulated by passing it to the ``buildstock_kestrel`` command. .. command-output:: buildstock_kestrel --help :ellipsis: 0, 8 @@ -40,19 +37,19 @@ you can submit a project file to be simulated by passing it to the ``buildstock_ Project configuration ..................... -To run a project on Kestrel or Eagle, you will need to make a few changes to +To run a project on Kestrel, you will need to make a few changes to your :doc:`project_defn`. First, the ``output_directory`` should be in ``/scratch/your_username/some_directory`` or in ``/projects`` somewhere. Building stock simulations generate a lot of output quickly and the ``/scratch`` or ``/projects`` filesystem are equipped to handle that kind of I/O throughput where your ``/home`` directory is not. -Next, you will need to add a :ref:`kestrel-config` or :ref:`eagle-config` top level key to the +Next, you will need to add a :ref:`kestrel-config` top level key to the project file, which will look something like this: .. code-block:: yaml - kestrel: # or eagle + kestrel: account: your_hpc_allocation n_jobs: 100 # the number of concurrent nodes to use minutes_per_sim: 2 diff --git a/setup.py b/setup.py index e4b062d9..bd11e182 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,6 @@ entry_points={ "console_scripts": [ "buildstock_local=buildstockbatch.local:main", - "buildstock_eagle=buildstockbatch.hpc:eagle_cli", "buildstock_kestrel=buildstockbatch.hpc:kestrel_cli", "buildstock_gcp=buildstockbatch.gcp.gcp:main", "buildstock_aws=buildstockbatch.aws.aws:main", From bf97ca7510c35bdd84d64f3a9290caa867fda216 Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Tue, 5 Nov 2024 16:59:33 -0600 Subject: [PATCH 2/7] Changelog update and put back hpc_name argument --- buildstockbatch/hpc.py | 2 ++ docs/changelog/changelog_dev.rst | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/buildstockbatch/hpc.py b/buildstockbatch/hpc.py index b3f3c58c..500b463d 100644 --- a/buildstockbatch/hpc.py +++ b/buildstockbatch/hpc.py @@ -888,10 +888,12 @@ def main(): # only direct script argument is the project .yml file parser = argparse.ArgumentParser() + parser.add_argument("hpc_name", choices=["kestrel"]) parser.add_argument("project_filename") args = parser.parse_args() # initialize the KestrelBatch object + assert args.hpc_name == "kestrel" batch = KestrelBatch(args.project_filename) # other arguments/cues about which part of the process we are in are # encoded in slurm job environment variables diff --git a/docs/changelog/changelog_dev.rst b/docs/changelog/changelog_dev.rst index f9dcb9ae..4d462592 100644 --- a/docs/changelog/changelog_dev.rst +++ b/docs/changelog/changelog_dev.rst @@ -123,3 +123,9 @@ Development Changelog :pullreq: 472 Remove support for Python 3.8, 3.9, and 3.10. Add support for Python 3.12. + + .. change:: + :tags: general + :pullreq: 472 + + Remove eagle related code and references since eagle is retired. \ No newline at end of file From 38646556ffc26f37d6d02087593073757d5a0e9f Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Tue, 5 Nov 2024 17:02:22 -0600 Subject: [PATCH 3/7] Put eagle in schema v0.4 and remove in v0.5 --- buildstockbatch/schemas/v0.4.yaml | 3 +- buildstockbatch/schemas/v0.5.yaml | 168 ++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 buildstockbatch/schemas/v0.5.yaml diff --git a/buildstockbatch/schemas/v0.4.yaml b/buildstockbatch/schemas/v0.4.yaml index 01263d55..0e4d7b78 100644 --- a/buildstockbatch/schemas/v0.4.yaml +++ b/buildstockbatch/schemas/v0.4.yaml @@ -1,4 +1,4 @@ -schema_version: enum('0.5') +schema_version: enum('0.4') buildstock_directory: str() project_directory: str(required=True) weather_files_path: str(required=False) @@ -6,6 +6,7 @@ weather_files_url: str(required=False) sampler: include('sampler-spec', required=True) workflow_generator: include('workflow-generator-spec', required=True) kestrel: include('hpc-spec', required=False) +eagle: include('hpc-spec', required=False) gcp: include('gcp-spec', required=False) aws: include('aws-spec', required=False) output_directory: regex('^(.*\/)?[a-z][a-z0-9_]*\/?$', required=True) diff --git a/buildstockbatch/schemas/v0.5.yaml b/buildstockbatch/schemas/v0.5.yaml new file mode 100644 index 00000000..01263d55 --- /dev/null +++ b/buildstockbatch/schemas/v0.5.yaml @@ -0,0 +1,168 @@ +schema_version: enum('0.5') +buildstock_directory: str() +project_directory: str(required=True) +weather_files_path: str(required=False) +weather_files_url: str(required=False) +sampler: include('sampler-spec', required=True) +workflow_generator: include('workflow-generator-spec', required=True) +kestrel: include('hpc-spec', required=False) +gcp: include('gcp-spec', required=False) +aws: include('aws-spec', required=False) +output_directory: regex('^(.*\/)?[a-z][a-z0-9_]*\/?$', required=True) +sys_image_dir: str(required=False) +baseline: include('sim-spec', required=True) +os_version: str(required=True) +os_sha: str(required=True) +max_minutes_per_sim: int(max=1440, required=False) +upgrades: list(include('upgrade-spec'), required=False) +postprocessing: include('postprocessing-spec', required=False) +references: map(required=False) +--- +gcp-spec: + # The GCP job ID (for Batch and Cloud Run) pattern is `^[a-z]([a-z0-9-]{0,61}[a-z0-9])?$`. + # For postprocessing job id, we append 3 characters ("-pp"), so this can be up to 60 chars. + job_identifier: regex('^[a-z]([a-z0-9-]{0,58}[a-z0-9])?$', required=True) + project: str(required=True) + region: str(required=True) + service_account: str(required=False) + artifact_registry: include('gcp-ar-spec', required=True) + batch_array_size: num(min=1, max=10000, required=True) + parallelism: num(min=1, max=10000, required=False) + gcs: include('gcs-spec', required=True) + job_environment: include('gcp-job-environment-spec', required=False) + postprocessing_environment: include('gcp-postprocessing_environment-spec', required=False) + +gcs-spec: + bucket: str(required=True) + prefix: str(required=True) + upload_chunk_size_mib: num(min=5, max=5000, required=False) + +gcp-ar-spec: + repository: str(required=True) + +gcp-job-environment-spec: + vcpus: int(min=1, max=224, required=False) + memory_mib: int(min=512, required=False) + boot_disk_mib: int(required=False) + machine_type: str(required=False) + use_spot: bool(required=False) + minutes_per_sim: num(min=0.05, max=480, required=False) + +gcp-postprocessing_environment-spec: + # Limits documented at + # https://cloud.google.com/run/docs/configuring/services/memory-limits + # https://cloud.google.com/run/docs/configuring/services/cpu + cpus: int(min=1, max=8, required=False) + memory_mib: int(min=512, max=32768, required=False) + +aws-spec: + job_identifier: regex('^[a-zA-Z]\w{,9}$', required=True) + s3: include('s3-aws-postprocessing-spec', required=True) + region: str(required=True) + use_spot: bool(required=False) + spot_bid_percent: num(min=1, max=100, required=False) + batch_array_size: num(min=1, max=10000, required=True) + notifications_email: regex('^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$', name='email', required=True) + dask: include('aws-dask-spec', required=True) + job_environment: include('aws-job-environment', required=False) + tags: map(str(), str(), required=False) + +aws-job-environment: + vcpus: int(min=1, max=36, required=False) + memory: int(min=1024, required=False) + +aws-dask-spec: + scheduler_cpu: enum(1024, 2048, 4096, 8192, 16384, required=False) + scheduler_memory: int(min=1024, required=False) + worker_cpu: enum(1024, 2048, 4096, 8192, 16384, required=False) + worker_memory: int(min=1024, required=False) + n_workers: int(min=1, required=True) + +hpc-spec: + account: str(required=True) + minutes_per_sim: num(min=0.05, max=480, required=True) + n_jobs: int(required=False) + postprocessing: include('hpc-postprocessing-spec', required=False) + sampling: include('sampling-spec', required=False) + +hpc-postprocessing-spec: + time: int(required=True) + n_workers: int(min=1, max=32, required=False) + node_memory_mb: int(min=85248, max=751616, required=False) + n_procs: int(min=1, max=36, required=False) + parquet_memory_mb: int(min=100, max=4096, required=False) + + +sampler-spec: + type: str(required=True) + args: map(key=regex(r'^[a-zA-Z_]\w*$', name='valid variable name'), required=False) + +workflow-generator-spec: + type: enum('residential_hpxml', 'commercial_default', required=True) + version: str(required=True) + args: map(key=regex(r'^[a-zA-Z_]\w*$', name='valid variable name'), required=False) + +sampling-spec: + time: int(required=True) + +sim-spec: + n_buildings_represented: int(required=True) + skip_sims: bool(required=False) + custom_gems: bool(required=False) + +upgrade-spec: + upgrade_name: str(required=True) + options: list(include('option-spec'), required=True) + package_apply_logic: include('apply-logic-spec', required=False) + reference_scenario: str(required=False) + +option-spec: + option: include('param_option-spec', required=True) + apply_logic: include('apply-logic-spec', required=False) + costs: list(include('cost-spec'), required=False) + lifetime: num(required=False) + +param_option-spec: str(exclude=':(){}[]') + +apply-logic-spec: > + any( + list( + include('and-spec'), + include('or-spec'), + include('not-spec'), + include('param_option-spec'), + ), + include('and-spec'), + include('or-spec'), + include('not-spec'), + include('param_option-spec') + ) +or-spec: + or: list(include('apply-logic-spec')) +and-spec: + and: list(include('apply-logic-spec')) +not-spec: + not: any(include('apply-logic-spec'), list(include('apply-logic-spec'))) + +cost-spec: + value: num(required=True) + multiplier: str(required=True) + +postprocessing-spec: + partition_columns: list(str(), required=False) + aws: include('aws-postprocessing-spec', required=False) + keep_individual_timeseries: bool(required=False) + +aws-postprocessing-spec: + region_name: str(required=False) + s3: include('s3-aws-postprocessing-spec', required=True) + athena: include('athena-aws-postprocessing-spec', required=False) + +s3-aws-postprocessing-spec: + bucket: str(required=True) + prefix: str(required=True) + +athena-aws-postprocessing-spec: + glue_service_role: str(required=False) + database_name: regex('^[a-z][a-z0-9_]*$', required=True) + max_crawling_time: num(requried=False) From 69b379f41412ae535a5326d33eadb9d43824179a Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Tue, 5 Nov 2024 17:02:50 -0600 Subject: [PATCH 4/7] Black formatting fix --- buildstockbatch/test/test_validation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/buildstockbatch/test/test_validation.py b/buildstockbatch/test/test_validation.py index 42abc5c8..cec046d6 100644 --- a/buildstockbatch/test/test_validation.py +++ b/buildstockbatch/test/test_validation.py @@ -495,6 +495,7 @@ def test_dask_config(): with pytest.raises(ValidationError, match=r"needs to be a multiple of 1024"): AwsBatch.validate_dask_settings(test3_filename) + def test_validate_kestrel_output_directory(): minimal_yml = pathlib.Path(example_yml_dir, "minimal-schema.yml") with pytest.raises(ValidationError, match=r"must be in /scratch or /projects"): From d9630b28711dd1623b67ccddd20e11c638fedbb1 Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Wed, 6 Nov 2024 10:01:30 -0600 Subject: [PATCH 5/7] Remove EagleBatch from documentation --- docs/changelog/changelog_0_18.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog/changelog_0_18.rst b/docs/changelog/changelog_0_18.rst index 09fd510d..274d3211 100644 --- a/docs/changelog/changelog_0_18.rst +++ b/docs/changelog/changelog_0_18.rst @@ -60,9 +60,9 @@ Added option to activate custom gems in the singularity container in eagle-based simulations by setting the ``custom_gem`` key in the ``baseline`` section of the configuration yaml to ``True``. This is implemented via - the ``bundle`` and ``bundle_path`` options in the OpenStudio CLI. See :func:`~.EagleBatch.run_building` for - additional implementation details. - + the ``bundle`` and ``bundle_path`` options in the OpenStudio CLI. + See `run_building `_ + for additional implementation details. .. change:: :tags: change, schema From a5c82a2a3e25b324a8e1d75f5f6411a3059a8d7b Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Wed, 6 Nov 2024 10:37:58 -0600 Subject: [PATCH 6/7] remove dead link --- docs/changelog/migration_2022_10_0.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/changelog/migration_2022_10_0.rst b/docs/changelog/migration_2022_10_0.rst index 690f57c4..a8df68aa 100644 --- a/docs/changelog/migration_2022_10_0.rst +++ b/docs/changelog/migration_2022_10_0.rst @@ -33,5 +33,4 @@ generators. See :doc:`../workflow_generators/index` for more information. There are some new, non breaking postprocessing inputs. The number of processors per worker and parquet memory size on Eagle are now configurable. There is also a new configuration capability to partition the postprocessed output for more -efficient querying in Athena. See :ref:`eagle-config` and -:ref:`post-config-opts` for details. +efficient querying in Athena. From 069d0f57eba03a0a46283e150a4b1449ccf671ed Mon Sep 17 00:00:00 2001 From: Rajendra Adhikari Date: Wed, 6 Nov 2024 11:05:30 -0600 Subject: [PATCH 7/7] Remove one more eagle file --- create_eagle_env.sh | 38 -------------------------------------- 1 file changed, 38 deletions(-) delete mode 100755 create_eagle_env.sh diff --git a/create_eagle_env.sh b/create_eagle_env.sh deleted file mode 100755 index 965fcdf8..00000000 --- a/create_eagle_env.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -DEV=0 -while getopts de: option -do -case "${option}" -in -d) DEV=1;; -e) CONDA_ENVS_DIR=${OPTARG};; -esac -done - -if [ -z "$CONDA_ENVS_DIR" ] -then - CONDA_ENVS_DIR=/shared-projects/buildstock/envs -fi - -MY_CONDA_ENV_NAME=${@:$OPTIND:1} -if [ -z "$MY_CONDA_ENV_NAME" ] -then - echo "Environment name not provided" - exit 1 -fi - -MY_CONDA_PREFIX="$CONDA_ENVS_DIR/$MY_CONDA_ENV_NAME" -echo "Creating $MY_CONDA_PREFIX" -module load conda -conda remove -y --prefix "$MY_CONDA_PREFIX" --all -mamba create -y --prefix "$MY_CONDA_PREFIX" -c conda-forge "python=3.10" "pyarrow" "numpy" "pandas" "dask>=2022.10.0" "distributed" "ruby" -conda deactivate -conda activate "$MY_CONDA_PREFIX" -which pip -if [ $DEV -eq 1 ] -then - pip install --no-cache-dir -e ".[dev]" -else - pip install --no-cache-dir . -fi