diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..58efa8c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +* +!Dockerfile +!src/ +!scripts/ +!setup.py +!requirements.txt +!requirements_unfixed.txt +!pytest.ini diff --git a/.env.template b/.env.template index 0eb4cba..2f85e5d 100644 --- a/.env.template +++ b/.env.template @@ -1,8 +1,9 @@ PACKAGE_NAME=building-controls-simulator -VERSION_TAG=0.3.2-alpha +VERSION_TAG=0.3.3-alpha +DOCKERHUB_REPOSITORY=tstesco USER_NAME=bcs DOCKER_IMAGE=${PACKAGE_NAME} -LOCAL_PACKAGE_DIR= +LOCAL_PACKAGE_DIR= DOCKER_HOME_DIR=/home/${USER_NAME} DOCKER_LIB_DIR=${DOCKER_HOME_DIR}/lib DOCKER_PACKAGE_DIR=${DOCKER_LIB_DIR}/${PACKAGE_NAME} @@ -14,7 +15,7 @@ PACKAGE_DIR=/home/bcs/lib/${PACKAGE_NAME} PYENV_ROOT=${DOCKER_HOME_DIR}/pyenv ENERGYPLUS_INSTALL_DIR=${EXT_DIR}/EnergyPlus ENERGYPLUSTOFMUSCRIPT=${EXT_DIR}/EnergyPlusToFMU-3.0.0/Scripts/EnergyPlusToFMU.py -GOOGLE_APPLICATION_CREDENTIALS=${DOCKER_HOME_DIR}/.config/gcloud/application_default_credentials.json +# GOOGLE_APPLICATION_CREDENTIALS=${DOCKER_HOME_DIR}/.config/gcloud/application_default_credentials.json # DYD_GOOGLE_CLOUD_PROJECT= # DYD_GCS_URI_BASE= # DYD_METADATA_URI= diff --git a/.gitignore b/.gitignore index 6394eed..95ee515 100644 --- a/.gitignore +++ b/.gitignore @@ -20,11 +20,15 @@ data/ weather/ idf/ fmu/ +notebooks/ +notes/ !test/idf/v8-9-0/AZ_Phoenix_gasfurnace_crawlspace_IECC_2018_cycles.idf !test/idf/v9-4-0/heatedbsmt_2story_2300sqft_gasfurnace_AC.idf !test/idf/v9-4-0/heatedbsmt_1story_2000sqft_gasfurnace_AC.idf !test/idf/v9-4-0/slab_1story_2000sqft_gasfurnace_AC.idf !test/fmu/fmu-models/deadband/deadband.fmu +!requirements.txt +!requirements_unfixed.txt __pycache__/ .pytest_cache/ .ipynb_checkpoints/ @@ -35,4 +39,4 @@ docs/source/generated/ .test.env docker-compose.yml build/ -notebooks/test_* +!notebooks/test_* diff --git a/.test.env.template b/.test.env.template index f613a60..eb86205 100644 --- a/.test.env.template +++ b/.test.env.template @@ -10,7 +10,7 @@ ENERGYPLUS_INSTALL_VERSION=9-4-0 EPLUS_DIR=${ENERGYPLUS_INSTALL_DIR}/EnergyPlus-${ENERGYPLUS_INSTALL_VERSION} EPLUS_IDD=${ENERGYPLUS_INSTALL_DIR}/EnergyPlus-${ENERGYPLUS_INSTALL_VERSION}/PreProcess/IDFVersionUpdater/V${ENERGYPLUS_INSTALL_VERSION}-Energy+.idd ENERGYPLUSTOFMUSCRIPT=${EXT_DIR}/EnergyPlusToFMU-3.0.0/Scripts/EnergyPlusToFMU.py -GOOGLE_APPLICATION_CREDENTIALS=${DOCKER_HOME_DIR}/.config/gcloud/application_default_credentials.json +# GOOGLE_APPLICATION_CREDENTIALS=${DOCKER_HOME_DIR}/.config/gcloud/application_default_credentials.json # DYD_GOOGLE_CLOUD_PROJECT= # DYD_GCS_URI_BASE= # DYD_METADATA_URI= diff --git a/Dockerfile b/Dockerfile index d84c393..089b193 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,7 +64,8 @@ RUN sudo apt-get update && sudo apt-get upgrade -y \ python3-dev \ python3-distutils \ subversion \ - p7zip-full + p7zip-full \ + && sudo rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # install nodejs and npm (for plotly) # install pyenv https://github.com/pyenv/pyenv-installer @@ -106,13 +107,8 @@ RUN curl -sL https://deb.nodesource.com/setup_12.x | sudo bash - \ && sudo rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # copying will cause rebuild at minimum to start from here -COPY ./src "${PACKAGE_DIR}/src" -COPY ./scripts "${PACKAGE_DIR}/scripts" -COPY ./requirements_fixed.txt "${PACKAGE_DIR}/requirements_fixed.txt" -COPY ./requirements_unfixed.txt "${PACKAGE_DIR}/requirements_unfixed.txt" -COPY ./setup.py "${PACKAGE_DIR}/setup.py" -COPY ./pytest.ini "${PACKAGE_DIR}/pytest.ini" -COPY ./.test.env "${PACKAGE_DIR}/.test.env" +# use .dockerignore to add files to docker image +COPY ./ "${PACKAGE_DIR}" # copied directory will not have user ownership by default # install energyplus versions desired in `scripts/setup/install_ep.sh` @@ -125,9 +121,9 @@ RUN sudo chown -R "${USER_NAME}" "${PACKAGE_DIR}" \ && cd "${PACKAGE_DIR}" \ && ${PYENV_ROOT}/versions/3.8.6/bin/python3.8 -m venv "${LIB_DIR}/${VENV_NAME}" \ && . "${LIB_DIR}/${VENV_NAME}/bin/activate" \ - && pip install --upgrade setuptools pip \ - # && pip install -r "requirements_fixed.txt" \ - && pip install -r "requirements_unfixed.txt" \ + && pip install --no-cache-dir --upgrade setuptools pip \ + # && pip install --no-cache-dir -r "requirements.txt" \ + && pip install --no-cache-dir -r "requirements_unfixed.txt" \ && pip install --editable . \ && cd "${EXT_DIR}/PyFMI" \ && python "setup.py" install --fmil-home="${FMIL_HOME}" \ @@ -135,7 +131,7 @@ RUN sudo chown -R "${USER_NAME}" "${PACKAGE_DIR}" \ && wget "https://github.com/RJT1990/pyflux/archive/0.4.15.zip" \ && unzip "0.4.15.zip" && rm "0.4.15.zip" \ && cd "pyflux-0.4.15" \ - && pip install . + && pip install --no-cache-dir . # install jupyter lab extensions for plotly # if jupyter lab build fails with webpack optimization, set --minimize=False diff --git a/README.md b/README.md index 93fbfed..03ef087 100644 --- a/README.md +++ b/README.md @@ -36,33 +36,58 @@ Copy the template files and fill in the variables mentioned below: ```bash cp .env.template .env cp docker-compose.yml.template docker-compose.yml +# and if you want to run the tests +# .test.env does not need to be editted, unless you want to inject creds +cp .test.env.template .test.env ``` Note: `docker-compose` behaviour may be slightly different on your host OS -(Windows, Mac OS, Linux) with respect to how the expansion of environment -variables works. If the base `docker-compose.yml` file fails on interpreting -variables, try inlining those specific variables, e.g. replacing `${LOCAL_PACKAGE_DIR}` -with `/building-controls-simulator`. +(Windows, Mac OS, Linux) and version of the CLI with respect to how the expansion of environment +variables works. Make sure you have the correct version (mentioned above). Edit in `.env.template`: ```bash ... -LOCAL_PACKAGE_DIR= -... -DYD_GCS_URI_BASE= -DYD_METADATA_URI= -NREL_DEV_API_KEY= -NREL_DEV_EMAIL= +LOCAL_PACKAGE_DIR= ... ``` -Now you're ready to build and launch the container! +Now you're ready to get the image and launch the container! If you delete the docker image just go through the setup here again to rebuild it. -##### Note: Docker images may use up to 12 GB of disk space - make sure you have this available before building. -The size of the container image can be reduced to roughly 5 GB by not installing +### Pull Docker image from Dockerhub + +You can access the latest release image from: https://hub.docker.com/r/tstesco/building-controls-simulator/tags via CLI: + +```bash +docker pull tstesco/building-controls-simulator:0.3.3-alpha +``` + +If you are using the Dockerhub repository make sure that your `.env` file contains +the line +```bash +DOCKERHUB_REPOSITORY=tstesco +``` + +This allows `docker-compose.yml` to find and use the correct image. Change this +line in `docker-compose.yml` if you want to use a locally built image. + +```yml + # change this if want to build your own image + image: ${DOCKERHUB_REPOSITORY}/${DOCKER_IMAGE}:${VERSION_TAG} +``` + +to + +```yml + # change this if want to build your own image + image: ${DOCKER_IMAGE}:${VERSION_TAG} +``` + +##### Note: Locally built Docker images may use up to 10 GB of disk space - make sure you have this available before building. +The size of the container image can be reduced to below 5 GB by not installing every EnergyPlus version in `scripts/setup/install_ep.sh` and not downloading all IECC 2018 IDF files in `scripts/setup/download_IECC_idfs.sh`. Simply comment -out the files you do not need. +out the versions/files you do not need in the respective files. ## Run BCS with Jupyter Lab Server (recommended: option 1) @@ -286,29 +311,29 @@ python -m pytest src/python ## Changing dependency versions -The dependencies are pinned to exact versions in the `requirements_fixed.txt` file. +The dependencies are pinned to exact versions in the `requirements.txt` file. To change this simply change line (approx) 124 in the `Dockerfile` from: ``` - && pip install -r "requirements_fixed.txt" \ - # && pip install -r "requirements_unfixed.txt" \ + && pip install --no-cache-dir -r "requirements.txt" \ + # && pip install --no-cache-dir -r "requirements_unfixed.txt" \ ``` to ``` - # && pip install -r "requirements_fixed.txt" \ - && pip install -r "requirements_unfixed.txt" \ + # && pip install --no-cache-dir -r "requirements.txt" \ + && pip install --no-cache-dir -r "requirements_unfixed.txt" \ ``` This will install the latest satisfying versions of all dependencies. After testing that -the dependencies are working freeze them into a new `requirements_fixed.txt` file. +the dependencies are working freeze them into a new `requirements.txt` file. ``` -pip freeze > requirements_fixed.txt +pip freeze > requirements.txt ``` Several dependencies are installed from source so these must be removed from the -`requirements_fixed.txt` file. These are: +`requirements.txt` file. These are: ``` PyFMI diff --git a/requirements_fixed.txt b/requirements.txt similarity index 93% rename from requirements_fixed.txt rename to requirements.txt index 8da3f1f..f5750c9 100644 --- a/requirements_fixed.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ click==7.1.2 coverage==5.3.1 cycler==0.10.0 Cython==0.29.21 -dask==2020.12.0 +dask==2021.1.0 decorator==4.4.2 defusedxml==0.6.0 docutils==0.16 @@ -30,14 +30,14 @@ flake8==3.8.4 fsspec==0.8.5 future==0.18.2 gcsfs==0.7.1 -google-api-core==1.24.1 +google-api-core==1.25.0 google-auth==1.24.0 google-auth-oauthlib==0.4.2 google-cloud-bigquery==2.3.1 google-cloud-bigquery-storage==2.1.0 google-cloud-core==1.5.0 google-cloud-storage==1.35.0 -google-crc32c==1.1.0 +google-crc32c==1.1.1 google-resumable-media==1.2.0 googleapis-common-protos==1.52.0 grpcio==1.34.1 @@ -72,7 +72,7 @@ munch==2.5.0 mypy-extensions==0.4.3 nbclient==0.5.1 nbconvert==6.0.7 -nbformat==5.1.0 +nbformat==5.1.2 nest-asyncio==1.4.3 notebook==6.2.0 numba==0.52.0 @@ -81,7 +81,7 @@ numexpr==2.7.2 numpy==1.19.5 oauthlib==3.1.0 packaging==20.8 -pandas==1.2.0 +pandas==1.2.1 pandas-gbq==0.14.1 pandocfilters==1.4.3 parso==0.8.1 @@ -93,7 +93,7 @@ Pillow==8.1.0 plotly==4.14.3 pluggy==0.13.1 prometheus-client==0.9.0 -prompt-toolkit==3.0.10 +prompt-toolkit==3.0.11 proto-plus==1.13.0 protobuf==3.14.0 psutil==5.8.0 @@ -107,8 +107,6 @@ pycparser==2.20 pydata-google-auth==1.1.0 pydot3k==1.0.17 pyflakes==2.2.0 -pyflux==0.4.15 -PyFMI==2.7.4 Pygments==2.7.4 pyparsing==2.4.7 pyrsistent==0.17.3 @@ -116,14 +114,14 @@ pytest==6.2.1 pytest-ordering==0.6 python-dateutil==2.8.1 pytz==2020.5 -PyYAML==5.3.1 -pyzmq==21.0.0 +PyYAML==5.4 +pyzmq==21.0.1 regex==2020.11.13 requests==2.25.1 requests-oauthlib==1.3.0 retrying==1.3.3 rsa==4.7 -scikit-learn==0.24.0 +scikit-learn==0.24.1 scipy==1.6.0 Send2Trash==1.5.0 Shapely==1.7.1 diff --git a/src/python/BuildingControlsSimulator/BuildingModels/EnergyPlusBuildingModel.py b/src/python/BuildingControlsSimulator/BuildingModels/EnergyPlusBuildingModel.py index d47e56a..febab8b 100644 --- a/src/python/BuildingControlsSimulator/BuildingModels/EnergyPlusBuildingModel.py +++ b/src/python/BuildingControlsSimulator/BuildingModels/EnergyPlusBuildingModel.py @@ -95,7 +95,9 @@ def get_output_states(self): def get_model_name(self): # only need the idf file name because the weather is determined from # the combination of idf file and data_source-identifier - return f"EnergyPlus_{self.idf.idf_name.rstrip('.idf')}" + _model_name = f"EnergyPlus_{self.idf.idf_name.rstrip('.idf')}" + _model_name = _model_name.replace(".", "_") + return _model_name @property def timesteps_per_hour(self): diff --git a/src/python/BuildingControlsSimulator/ControllerModels/Deadband.py b/src/python/BuildingControlsSimulator/ControllerModels/Deadband.py index f7a5de7..85be1b3 100644 --- a/src/python/BuildingControlsSimulator/ControllerModels/Deadband.py +++ b/src/python/BuildingControlsSimulator/ControllerModels/Deadband.py @@ -56,7 +56,9 @@ def get_output_states(self): ] def get_model_name(self): - return f"Deadband_{self.deadband}".replace(".", "-") + _model_name = f"Deadband_{self.deadband}" + _model_name = _model_name.replace(".", "_") + return _model_name def initialize( self, diff --git a/src/python/BuildingControlsSimulator/ControllerModels/FMIController.py b/src/python/BuildingControlsSimulator/ControllerModels/FMIController.py index 639df87..6905514 100644 --- a/src/python/BuildingControlsSimulator/ControllerModels/FMIController.py +++ b/src/python/BuildingControlsSimulator/ControllerModels/FMIController.py @@ -29,5 +29,6 @@ class FMIController(ControllerModel): step_size_seconds = attr.ib() def get_model_name(self): - fmu_name = os.path.basename(self.fmu_path) - return f"FMU_{fmu_name}" + _model_name = os.path.basename(self.fmu_path) + _model_name = _model_name.replace(".", "_") + return _model_name diff --git a/src/python/BuildingControlsSimulator/DataClients/DataClient.py b/src/python/BuildingControlsSimulator/DataClients/DataClient.py index ae2edb3..dbc664c 100644 --- a/src/python/BuildingControlsSimulator/DataClients/DataClient.py +++ b/src/python/BuildingControlsSimulator/DataClients/DataClient.py @@ -232,6 +232,16 @@ def get_data(self): [STATES.CALENDAR_EVENT], ] = pd.NA + # finally convert dtypes to final types now that nulls in + # non-nullable columns have been properly filled or removed + _data = convert_spec( + _data, + src_spec=self.internal_spec, + dest_spec=self.internal_spec, + src_nullable=True, + dest_nullable=False + ) + else: raise ValueError( f"ID={self.sim_config['identifier']} has no full_data_periods " diff --git a/src/python/BuildingControlsSimulator/DataClients/DataDestination.py b/src/python/BuildingControlsSimulator/DataClients/DataDestination.py index 0ae5c96..c3eb209 100644 --- a/src/python/BuildingControlsSimulator/DataClients/DataDestination.py +++ b/src/python/BuildingControlsSimulator/DataClients/DataDestination.py @@ -34,7 +34,9 @@ def put_data(self, df, sim_name): pass def get_file_name(self, sim_name): - return f"{sim_name}.{self.file_extension}" + # sim_name may contain . character, replace this safely + safe_sim_name = sim_name.replace(".","_") + return f"{safe_sim_name}.{self.file_extension}" def get_local_cache_file(self, sim_name): if self.local_cache: @@ -114,8 +116,6 @@ def write_data_by_extension( index=False, ) elif file_extension == "csv.zip": - raise NotImplementedError("Pandas 1.2.0 has issue with writting zip files.") - # see _df.to_csv( filepath_or_buffer, compression="zip", diff --git a/src/python/BuildingControlsSimulator/DataClients/DataSource.py b/src/python/BuildingControlsSimulator/DataClients/DataSource.py index c99f486..44ee3b0 100644 --- a/src/python/BuildingControlsSimulator/DataClients/DataSource.py +++ b/src/python/BuildingControlsSimulator/DataClients/DataSource.py @@ -115,6 +115,8 @@ def read_data_static(filepath_or_buffer, data_spec, extension="parquet.gzip"): if _col != data_spec.datetime_column ], data_spec, + src_nullable=True, + dest_nullable=True, ), ) elif extension == "csv.zip": @@ -129,6 +131,8 @@ def read_data_static(filepath_or_buffer, data_spec, extension="parquet.gzip"): if _col != data_spec.datetime_column ], data_spec, + src_nullable=True, + dest_nullable=True, ), ) elif extension in ["csv.gzip", "csv.gz"]: @@ -143,6 +147,8 @@ def read_data_static(filepath_or_buffer, data_spec, extension="parquet.gzip"): if _col != data_spec.datetime_column ], data_spec, + src_nullable=True, + dest_nullable=True, ), ) else: diff --git a/src/python/BuildingControlsSimulator/DataClients/DataSpec.py b/src/python/BuildingControlsSimulator/DataClients/DataSpec.py index 4fdcf2b..02495ba 100644 --- a/src/python/BuildingControlsSimulator/DataClients/DataSpec.py +++ b/src/python/BuildingControlsSimulator/DataClients/DataSpec.py @@ -17,10 +17,6 @@ def spec_unit_conversion(df, src_spec, dest_spec): """This method must be able to evaluate multiple sources should a channel be composed from multiple sources.""" - if src_spec == dest_spec: - logger.info("spec_unit_conversion: src_spec is equal to dest_spec.") - return df - for k, v in src_spec.full.spec.items(): if k in df.columns: src_unit = v["unit"] @@ -60,11 +56,38 @@ def spec_unit_conversion(df, src_spec, dest_spec): return df -def get_dtype_mapper(df_cols, dest_spec): +def get_dtype_mapper(df_cols, dest_spec, src_nullable=False, dest_nullable=False): # we only need to consider the destination spec dtype_mapper = { k: v["dtype"] for k, v in dest_spec.full.spec.items() if k in df_cols } + + # convert between nullable columns and non-nullable for compatability + if dest_nullable: + for k,v in dtype_mapper.items(): + if v == "bool": + dtype_mapper[k] = "boolean" + elif v == "int8": + dtype_mapper[k] = "Int8" + elif v == "int16": + dtype_mapper[k] = "Int16" + elif v == "int32": + dtype_mapper[k] = "Int32" + elif v == "int64": + dtype_mapper[k] = "Int64" + else: + for k,v in dtype_mapper.items(): + if v == "boolean": + dtype_mapper[k] = "bool" + elif v == "Int8": + dtype_mapper[k] = "int8" + elif v == "Int16": + dtype_mapper[k] = "int16" + elif v == "Int32": + dtype_mapper[k] = "int32" + elif v == "Int64": + dtype_mapper[k] = "int64" + return dtype_mapper @@ -120,7 +143,9 @@ def project_spec_keys(src_spec, dest_spec): return projection -def convert_spec(df, src_spec, dest_spec, copy=False): +def convert_spec(df, src_spec, dest_spec, src_nullable=False, dest_nullable=False, copy=False): + # src_nullable: whether to use nullable int types + # dest_nullable: whether to use nullable int types if type(src_spec) == type(dest_spec): logger.info("convert_spec: src_spec is equal to dest_spec.") return df @@ -148,7 +173,7 @@ def convert_spec(df, src_spec, dest_spec, copy=False): _df = _df.rename(columns=get_rename_mapper(src_spec=src_spec, dest_spec=dest_spec)) _df = _df.astype( - dtype=get_dtype_mapper(df_cols=_df.columns, dest_spec=dest_spec), + dtype=get_dtype_mapper(df_cols=_df.columns, dest_spec=dest_spec, src_nullable=src_nullable, dest_nullable=dest_nullable), ) _df = _df.sort_values(dest_spec.datetime_column, ascending=True) return _df @@ -174,6 +199,10 @@ def dtypes_are_pandas(self, attribute, value): "Int16", "Int32", "Int64", + "int8", + "int16", + "int32", + "int64", "UInt8", "UInt16", "UInt32", @@ -294,91 +323,91 @@ def __init__(self): spec={ STATES.AUXHEAT1: { "name": "auxHeat1", - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, STATES.AUXHEAT2: { "name": "auxHeat2", - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, STATES.AUXHEAT3: { "name": "auxHeat3", - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, STATES.COMPCOOL1: { "name": "compCool1", - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, STATES.COMPCOOL2: { "name": "compCool2", - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, STATES.COMPHEAT1: { "name": "compHeat1", - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, STATES.COMPHEAT2: { "name": "compHeat2", - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, STATES.DEHUMIDIFIER: { "name": "dehumidifier", - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, STATES.ECONOMIZER: { "name": "economizer", - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, STATES.FAN: { "name": "fan", - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, STATES.FAN_STAGE_ONE: { "name": "fan1", - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, STATES.FAN_STAGE_TWO: { "name": "fan2", - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, STATES.FAN_STAGE_THREE: { "name": "fan3", - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, STATES.HUMIDIFIER: { "name": "humidifier", - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, STATES.VENTILATOR: { "name": "ventilator", - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, @@ -616,73 +645,73 @@ def __init__(self): spec={ "auxHeat1": { "internal_state": STATES.AUXHEAT1, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "auxHeat2": { "internal_state": STATES.AUXHEAT2, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "auxHeat3": { "internal_state": STATES.AUXHEAT3, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "compCool1": { "internal_state": STATES.COMPCOOL1, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "compCool2": { "internal_state": STATES.COMPCOOL2, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "compHeat1": { "internal_state": STATES.COMPHEAT1, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "compHeat2": { "internal_state": STATES.COMPHEAT2, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "dehumidifier": { "internal_state": STATES.DEHUMIDIFIER, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "economizer": { "internal_state": STATES.ECONOMIZER, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "fan": { "internal_state": STATES.FAN, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "humidifier": { "internal_state": STATES.HUMIDIFIER, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "ventilator": { "internal_state": STATES.VENTILATOR, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, @@ -860,49 +889,49 @@ def __init__(self): spec={ "auxHeat1": { "internal_state": STATES.AUXHEAT1, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "auxHeat2": { "internal_state": STATES.AUXHEAT2, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "auxHeat3": { "internal_state": STATES.AUXHEAT3, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "compCool1": { "internal_state": STATES.COMPCOOL1, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "compCool2": { "internal_state": STATES.COMPCOOL2, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "compHeat1": { "internal_state": STATES.COMPHEAT1, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "compHeat2": { "internal_state": STATES.COMPHEAT2, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, "fan": { "internal_state": STATES.FAN, - "dtype": "Int16", + "dtype": "int16", "channel": CHANNELS.EQUIPMENT, "unit": UNITS.SECONDS, }, diff --git a/src/python/BuildingControlsSimulator/DataClients/GBQDataSource.py b/src/python/BuildingControlsSimulator/DataClients/GBQDataSource.py index c1d900a..4b110d2 100644 --- a/src/python/BuildingControlsSimulator/DataClients/GBQDataSource.py +++ b/src/python/BuildingControlsSimulator/DataClients/GBQDataSource.py @@ -45,7 +45,7 @@ def get_data(self, sim_config): if _data.empty: _data = self.get_gbq_data(sim_config, local_cache_file) _data = self.drop_unused_columns(_data=_data) - _data = convert_spec(df=_data, src_spec=self.data_spec, dest_spec=Internal()) + _data = convert_spec(df=_data, src_spec=self.data_spec, dest_spec=Internal(), src_nullable=True, dest_nullable=True) return _data def get_gbq_data(self, sim_config, local_cache_file): diff --git a/src/python/BuildingControlsSimulator/DataClients/GCSDataSource.py b/src/python/BuildingControlsSimulator/DataClients/GCSDataSource.py index 04f6b26..bc9742b 100644 --- a/src/python/BuildingControlsSimulator/DataClients/GCSDataSource.py +++ b/src/python/BuildingControlsSimulator/DataClients/GCSDataSource.py @@ -39,7 +39,7 @@ def get_data(self, sim_config): if _data.empty: _data = self.get_gcs_cache(sim_config, local_cache_file) _data = self.drop_unused_columns(_data=_data) - _data = convert_spec(df=_data, src_spec=self.data_spec, dest_spec=Internal()) + _data = convert_spec(df=_data, src_spec=self.data_spec, dest_spec=Internal(), src_nullable=True, dest_nullable=True) return _data @abstractmethod diff --git a/src/python/BuildingControlsSimulator/DataClients/LocalSource.py b/src/python/BuildingControlsSimulator/DataClients/LocalSource.py index 0a46de3..ac4ea1e 100644 --- a/src/python/BuildingControlsSimulator/DataClients/LocalSource.py +++ b/src/python/BuildingControlsSimulator/DataClients/LocalSource.py @@ -61,6 +61,6 @@ def get_data(self, sim_config): _data = self.get_local_cache(local_cache_file) _data = self.drop_unused_columns(_data=_data) _data = convert_spec( - df=_data, src_spec=self.data_spec, dest_spec=Internal(), copy=False + df=_data, src_spec=self.data_spec, dest_spec=Internal(), copy=False, src_nullable=True, dest_nullable=True ) return _data diff --git a/src/python/BuildingControlsSimulator/DataClients/test_GCSDestination.py b/src/python/BuildingControlsSimulator/DataClients/test_GCSDestination.py index c030de3..5e41573 100644 --- a/src/python/BuildingControlsSimulator/DataClients/test_GCSDestination.py +++ b/src/python/BuildingControlsSimulator/DataClients/test_GCSDestination.py @@ -108,5 +108,7 @@ def test_put_data(self): r_df, src_spec=self.data_client.destination.data_spec, dest_spec=Internal(), + src_nullable=True, + dest_nullable=True, ) assert _df.equals(cr_df) diff --git a/src/python/BuildingControlsSimulator/DataClients/test_LocalDestination.py b/src/python/BuildingControlsSimulator/DataClients/test_LocalDestination.py index de72e5f..0c846a4 100644 --- a/src/python/BuildingControlsSimulator/DataClients/test_LocalDestination.py +++ b/src/python/BuildingControlsSimulator/DataClients/test_LocalDestination.py @@ -109,5 +109,7 @@ def test_put_data(self): r_df, src_spec=self.data_client.destination.data_spec, dest_spec=Internal(), + src_nullable=True, + dest_nullable=True, ) assert _df.equals(cr_df) diff --git a/src/python/BuildingControlsSimulator/Simulator/Simulation.py b/src/python/BuildingControlsSimulator/Simulator/Simulation.py index c69435a..2bee985 100644 --- a/src/python/BuildingControlsSimulator/Simulator/Simulation.py +++ b/src/python/BuildingControlsSimulator/Simulator/Simulation.py @@ -140,7 +140,7 @@ def sim_name(self): _building_model_name = self.building_model.get_model_name() _controller_model_name = self.controller_model.get_model_name() - return "_".join( + _sim_name = "_".join( [ _prefix, _sim_run_identifier, @@ -150,6 +150,9 @@ def sim_name(self): _controller_model_name, ] ) + # safely remove any errant . characters breaking extension handling + _sim_name.replace(".", "_") + return _sim_name def create_models(self, preprocess_check=False): # TODO: only have the building model that requires dynamic building diff --git a/src/python/BuildingControlsSimulator/Simulator/params_test_Simulator.py b/src/python/BuildingControlsSimulator/Simulator/params_test_Simulator.py index 4a138e1..d718d00 100644 --- a/src/python/BuildingControlsSimulator/Simulator/params_test_Simulator.py +++ b/src/python/BuildingControlsSimulator/Simulator/params_test_Simulator.py @@ -21,7 +21,8 @@ test_params_gcs_dyd = [] test_params_gbq_flatfiles = [] -if os.environ.get("LOCAL_CACHE_DIR"): +# if os.environ.get("LOCAL_CACHE_DIR"): +if False: test_params_local = [ { "config": { @@ -119,7 +120,8 @@ }, ] -if os.environ.get("DYD_GCS_URI_BASE"): +# if os.environ.get("DYD_GCS_URI_BASE"): +if False: test_params_gcs_dyd = [ { "config": { @@ -271,13 +273,86 @@ if os.environ.get("FLATFILES_GBQ_TABLE"): test_params_gbq_flatfiles = [ + # { + # "config": { + # "identifier": os.environ.get("TEST_GBQ_FF_IDENTIFIER"), + # "latitude": 41.8781, + # "longitude": -87.6298, + # "start_utc": "2019-01-14", + # "end_utc": "2019-01-18", + # "min_sim_period": "1D", + # "sim_step_size_seconds": 60, + # "output_step_size_seconds": 300, + # }, + # "data_client": { + # "is_local_source": False, + # "is_gcs_source": False, + # "is_gbq_source": True, + # "gcp_project": os.environ.get("DYD_GOOGLE_CLOUD_PROJECT"), + # "gcs_uri_base": None, + # "gbq_table": os.environ.get("FLATFILES_GBQ_TABLE"), + # "source_data_spec": FlatFilesSpec(), + # "source_local_cache": os.environ.get("LOCAL_CACHE_DIR"), + # "is_local_destination": True, + # "is_gcs_destination": False, + # "is_gbq_destination": False, + # "destination_data_spec": FlatFilesSpec(), + # "destination_local_cache": os.environ.get("LOCAL_CACHE_DIR"), + # }, + # "building_model": { + # "is_energyplus_building": True, + # "idf_name": "heatedbsmt_2story_2300sqft_gasfurnace_AC.idf", + # "epw_name": "USA_IL_Chicago-OHare.Intl.AP.725300_TMY3.epw", + # "building_config": { + # "infiltration_ventilation": { + # "ach50": 10, + # "wsf": 0.6, + # }, + # "insulation_r_si": { + # "Exterior Roof": 1.0, + # "Interior Ceiling": 6.7, + # "Interior Floor": 0.75, + # "Exterior Wall": 5.25, + # "Exterior Floor": 5.0, + # }, + # "windows": { + # "u_factor": 0.8, + # "solar_heat_gain": 0.30, + # "visible_transmittance": 0.60, + # }, + # "hvac": { + # "heating_stages": 1, + # "heating_equipment": "gas_furnace", + # "heating_sizing_factor": 0.9, + # "cooling_stages": 1, + # "cooling_equipment": "dx_ac", + # "cooling_sizing_factor": 0.9, + # }, + # "thermal_mass": 1e7, + # }, + # }, + # "controller_model": { + # "is_deadband": True, + # "is_fmu": False, + # }, + # "state_estimator_model": { + # "is_low_pass_filter": True, + # "low_pass_filter_alpha": 0.2, + # }, + # "expected_result": { + # "mean_thermostat_temperature": 20.887479782104492, + # "mean_thermostat_humidity": 20.74267578125, + # "output_format_mean_thermostat_temperature": 695.9857788085938, + # "output_format_mean_thermostat_humidity": 20.74267578125, + # }, + # }, { "config": { - "identifier": os.environ.get("TEST_GBQ_FF_IDENTIFIER"), - "latitude": 41.8781, - "longitude": -87.6298, - "start_utc": "2019-01-14", - "end_utc": "2019-01-18", + "identifier": os.environ.get("TEST_GBQ_FF_IDENTIFIER_2"), + "latitude": 51.217373, + "longitude": -114.296019 , + "start_utc": "2019-03-09", + "end_utc": "2019-03-15", "min_sim_period": "1D", "sim_step_size_seconds": 60, "output_step_size_seconds": 300, @@ -338,10 +413,10 @@ "low_pass_filter_alpha": 0.2, }, "expected_result": { - "mean_thermostat_temperature": 20.887479782104492, - "mean_thermostat_humidity": 20.74267578125, - "output_format_mean_thermostat_temperature": 695.9857788085938, - "output_format_mean_thermostat_humidity": 20.74267578125, + "mean_thermostat_temperature": 20.319211959838867, + "mean_thermostat_humidity": 16.502723693847656, + "output_format_mean_thermostat_temperature": 685.7371826171875, + "output_format_mean_thermostat_humidity": 16.502723693847656, }, }, ] diff --git a/src/python/BuildingControlsSimulator/StateEstimatorModels/LowPassFilter.py b/src/python/BuildingControlsSimulator/StateEstimatorModels/LowPassFilter.py index c35e602..f7ae6d9 100644 --- a/src/python/BuildingControlsSimulator/StateEstimatorModels/LowPassFilter.py +++ b/src/python/BuildingControlsSimulator/StateEstimatorModels/LowPassFilter.py @@ -46,7 +46,9 @@ def get_output_states(self): ] def get_model_name(self): - return f"Deadband_{self.deadband}".replace(".", "-") + _model_name = "LowPass" + _model_name = _model_name.replace(".", "_") + return _model_name def initialize( self,