From 80fe6e66e1b97919fe440a8cbdf697502a9a39a1 Mon Sep 17 00:00:00 2001 From: grgmiller Date: Thu, 22 Feb 2024 09:21:32 -0800 Subject: [PATCH 1/5] Update conda environment --- README.md | 39 +++++++++++++++++++++++++-------------- environment.yml | 18 +++++++----------- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index d597c6ea..e5031741 100644 --- a/README.md +++ b/README.md @@ -10,17 +10,9 @@ Please check out [our documentation](https://docs.singularity.energy/docs/open-g The Open Grid Emissions Dataset can be [downloaded here](https://singularity.energy/open-grid-emissions). An archive of previous versions of the dataset and intermediate data outputs (for research and validation purposes) can be found on [Zenodo](https://zenodo.org/communities/singularity-energy?page=1&size=20). ## Installing and running the data pipeline -A couple of installation methods are available. First, navigate to the folder where you want to save the repository and run the following commands: +To manage the code environment necessary to run the OGE data pipeline, either `pipenv` or `conda` may be used. Currently, we utilize `pipenv` as our preferred environment manager for running the pipeline that is used for data releases, but `conda` will also work if you are more familiar with `conda`. -### If you are using conda -```bash -conda install git -git clone https://github.com/singularity-energy/open-grid-emissions.git -conda update conda -cd open-grid-emissions -conda env create -f environment.yml -conda activate open_grid_emissions -``` +First, navigate to the folder where you want to save the repository and run the following commands: ### If you are using pipenv Note that this option requires to have Python and git installed on your machine. @@ -35,6 +27,19 @@ python -m build pip install . ``` +### If you are using conda +```bash +conda install git +git clone https://github.com/singularity-energy/open-grid-emissions.git +conda update conda +cd open-grid-emissions +conda env create -f environment.yml +conda activate open_grid_emissions +pip install build +python -m build +pip install . +``` + The pipeline can be run as follows: ```bash cd src/oge @@ -133,11 +138,17 @@ Open anaconda prompt, use `cd` to navigate to the directory where your local fil conda update conda conda env create -f environment.yml ``` -and activate the environment: +Installation requires that the conda channel-priority be set to "flexible". This is the default behavior, so if you've never manually changed this, you shouldn't have to worry about this. However, if you receive an error message like "Found conflicts!" when trying to install the environment, try setting your channel priority to flexible by running the following command:`conda config --set channel_priority flexible` and then re-running the above commands. + +The final step is to install the `oge` package itself in the conda environment. To do so, run: ```bash conda activate open_grid_emissions +pip install build +python -m build +pip install . ``` -Installation requires that the conda channel-priority be set to "flexible". This is the default behavior, so if you've never manually changed this, you shouldn't have to worry about this. However, if you receive an error message like "Found conflicts!" when trying to install the environment, try setting your channel priority to flexible by running the following command:`conda config --set channel_priority flexible` and then re-running the above commands. + +The open_grid_emissions conda environment should now be set up and ready to run. ### Setup with pipenv #### Install python and git @@ -182,9 +193,9 @@ pip install –-editable . If you ever need to remove and reinstall the environment, run `pipenv --rm` from the root directory then follow the directions above. ### Running the complete data pipeline -If you would like to run the full data pipeline to generate all intermediate outputs and results files, navigate to `open-grid-emissions/src`, and run the following (replacing 2021 with whichever year you want to run): +If you would like to run the full data pipeline to generate all intermediate outputs and results files, navigate to `open-grid-emissions/src/oge`, and run the following (replacing 2022 with whichever year you want to run): ``` -python data_pipeline.py --year 2021 +python data_pipeline.py --year 2022 ``` ### Keeping the code updated diff --git a/environment.yml b/environment.yml index 4331e3de..9282602e 100644 --- a/environment.yml +++ b/environment.yml @@ -3,34 +3,30 @@ channels: - defaults - conda-forge dependencies: - - blas=*=openblas # prevent mkl implementation of blas - cvxopt - - cvxpy=1.2.1 # used by gridemissions, newer version not working as of 12/12/2022 + - cvxpy + - osqp - ipykernel - - nomkl # prevent mkl implementation of blas - notebook - numpy - openpyxl - - pandas + - pandas>=2.0 - pip - plotly - pyarrow - pytest - python>=3.11,<3.12 # require 3.11.x for compatibility with pudl - - python-snappy # used for pudl - - qdldl-python==0.1.5,!=0.1.5.post2 # used for gridemissions, newer version not working as of 12/12/2022 - - requests>=2.28.1 + - qdldl-python + - requests - ruff - s3fs - seaborn # used by gridemissions - - setuptools # used for pudl - sqlalchemy - - sqlite # used for pudl - statsmodels - coloredlogs # used for prettier logging - pip: # --editable ../pudl #NOTE: this is for development use - - git+https://github.com/singularity-energy/pudl.git@main#egg=catalystcoop.pudl + - git+https://github.com/singularity-energy/pudl.git@oge_release#egg=catalystcoop.pudl #- --editable ../gridemissions # NOTE: this is for development use - - git+https://github.com/gailin-p/gridemissions#egg=gridemissions + - git+https://github.com/singularity-energy/gridemissions#egg=gridemissions From eaf2b07026df9521f63eb4538a6742a03a019cb3 Mon Sep 17 00:00:00 2001 From: grgmiller Date: Thu, 22 Feb 2024 09:51:40 -0800 Subject: [PATCH 2/5] update development install instructions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e5031741..dc6a428d 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ The final step is to install the `oge` package itself in the conda environment. conda activate open_grid_emissions pip install build python -m build -pip install . +pip install --editable . ``` The open_grid_emissions conda environment should now be set up and ready to run. From 5afd369334877884bddbb16c5766e57f3ed24b7c Mon Sep 17 00:00:00 2001 From: grgmiller Date: Fri, 23 Feb 2024 15:13:49 -0800 Subject: [PATCH 3/5] fix filepaths for s3 --- src/oge/filepaths.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/oge/filepaths.py b/src/oge/filepaths.py index 3b74c271..38c4b3d5 100644 --- a/src/oge/filepaths.py +++ b/src/oge/filepaths.py @@ -23,24 +23,24 @@ def top_folder(rel=""): def reference_table_folder(rel=""): - return os.path.join(top_folder("reference_tables"), rel) + return os.path.join(top_folder("reference_tables"), rel).replace("\\","/") def data_folder(rel=""): """Returns a path relative to the `data` folder.""" - return os.path.join(get_data_store(), rel) + return os.path.join(get_data_store(), rel).replace("\\","/") def downloads_folder(rel=""): - return os.path.join(data_folder("downloads"), rel) + return os.path.join(data_folder("downloads"), rel).replace("\\","/") def outputs_folder(rel=""): - return os.path.join(data_folder("outputs"), rel) + return os.path.join(data_folder("outputs"), rel).replace("\\","/") def results_folder(rel=""): - return os.path.join(data_folder("results"), rel) + return os.path.join(data_folder("results"), rel).replace("\\","/") def containing_folder(filepath: str) -> str: From 6e5fd8c8d513f988c81078b5a0128b4632b5e688 Mon Sep 17 00:00:00 2001 From: grgmiller Date: Fri, 23 Feb 2024 15:27:03 -0800 Subject: [PATCH 4/5] format with ruff --- src/oge/filepaths.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/oge/filepaths.py b/src/oge/filepaths.py index 38c4b3d5..24614f25 100644 --- a/src/oge/filepaths.py +++ b/src/oge/filepaths.py @@ -23,24 +23,24 @@ def top_folder(rel=""): def reference_table_folder(rel=""): - return os.path.join(top_folder("reference_tables"), rel).replace("\\","/") + return os.path.join(top_folder("reference_tables"), rel).replace("\\", "/") def data_folder(rel=""): """Returns a path relative to the `data` folder.""" - return os.path.join(get_data_store(), rel).replace("\\","/") + return os.path.join(get_data_store(), rel).replace("\\", "/") def downloads_folder(rel=""): - return os.path.join(data_folder("downloads"), rel).replace("\\","/") + return os.path.join(data_folder("downloads"), rel).replace("\\", "/") def outputs_folder(rel=""): - return os.path.join(data_folder("outputs"), rel).replace("\\","/") + return os.path.join(data_folder("outputs"), rel).replace("\\", "/") def results_folder(rel=""): - return os.path.join(data_folder("results"), rel).replace("\\","/") + return os.path.join(data_folder("results"), rel).replace("\\", "/") def containing_folder(filepath: str) -> str: From 906d9f0beee24874b376166e208d6a04c49e11bb Mon Sep 17 00:00:00 2001 From: grgmiller Date: Sat, 24 Feb 2024 10:57:19 -0800 Subject: [PATCH 5/5] update version to 0.3.3 --- CITATION.cff | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index bcf717b1..a4812d01 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -22,6 +22,6 @@ authors: identifiers: - type: doi value: 'https://zenodo.org/doi/10.5281/zenodo.7062459' -version: 0.3.2 +version: 0.3.3 license: MIT -date-released: '2024-02-14' +date-released: '2024-02-27' diff --git a/pyproject.toml b/pyproject.toml index 21c4c61e..cdff1c1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "oge" -version = "0.3.2" +version = "0.3.3" requires-python = ">=3.11,<3.12" readme = "README.md" authors = [