From cc73ae08f354112c137ac6727762eb8a1497a91a Mon Sep 17 00:00:00 2001 From: Jackson Burns Date: Sun, 16 Apr 2023 18:55:33 -0400 Subject: [PATCH 01/24] run CI on macos-11, change timeout to step level instead of command macos does not have the timeout command, just use the timeout in GitHub actions steps instead --- .github/workflows/CI.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 326c34e24e..b3a049dbe3 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -38,7 +38,10 @@ concurrency: jobs: build-and-test-linux: - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, macos-11] + runs-on: ${{ matrix.os }} # skip scheduled runs on main from forks if: ${{ !( github.repository != 'ReactionMechanismGenerator/RMG-Py' && github.event_name == 'schedule' ) }} env: # update this if needed to match a pull request on the RMG-database @@ -103,10 +106,11 @@ jobs: # Regression Testing - Test Execution - name: Regression Tests - Execution + timeout-minutes: 60 run: | for regr_test in aromatics liquid_oxidation nitrogen oxidation sulfur superminimal; do - if timeout 600 python-jl rmg.py test/regression/"$regr_test"/input.py; then + if python-jl rmg.py test/regression/"$regr_test"/input.py; then echo "$regr_test" "Executed Successfully" else echo "$regr_test" "Failed to Execute" From 7efc2ded0cdad17a0058721e6a6a2c5c4ca2360b Mon Sep 17 00:00:00 2001 From: Jackson Burns Date: Sun, 16 Apr 2023 18:55:46 -0400 Subject: [PATCH 02/24] treat ubuntu results as baseline --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index b3a049dbe3..03d4de2c2c 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -135,7 +135,7 @@ jobs: # Upload Regression Results as Stable if Scheduled or Push to Main - name: Upload Results as Reference # upload the results for scheduled CI (on main) and pushes to main - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' && runner.os =='ubuntu' uses: actions/upload-artifact@v3 with: name: stable_regression_results From ad7bdddcf4b9a898473779ee62600bc665528d8c Mon Sep 17 00:00:00 2001 From: Jackson Burns Date: Sun, 16 Apr 2023 20:32:13 -0400 Subject: [PATCH 03/24] switch bash syntax to hopefully be compatible with mac --- .github/workflows/CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 03d4de2c2c..0844f64e31 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -117,7 +117,7 @@ jobs: export FAILED=Yes fi done - if [[ -v FAILED ]]; then + if [[ ${FAILED} ]]; then echo "One or more regression tests could not be executed." echo "Please download the failed results or check the above log to see why." exit 1 @@ -219,7 +219,7 @@ jobs: fi fi done - if [[ -v FAILED ]]; then + if [[ ${FAILED} ]]; then echo "One or more regression tests failed." echo "Please download the failed results and run the tests locally or check the above log to see why." exit 1 From e5cee33c5434a59f753270ab563b369e6faabe52 Mon Sep 17 00:00:00 2001 From: Jackson Burns <33505528+JacksonBurns@users.noreply.github.com> Date: Sun, 16 Apr 2023 22:05:25 -0400 Subject: [PATCH 04/24] try updating xcode on macos latest to deal with binary signing bug --- .github/workflows/CI.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0844f64e31..42310fe6cd 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -40,7 +40,7 @@ jobs: build-and-test-linux: strategy: matrix: - os: [ubuntu-latest, macos-11] + os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} # skip scheduled runs on main from forks if: ${{ !( github.repository != 'ReactionMechanismGenerator/RMG-Py' && github.event_name == 'schedule' ) }} @@ -51,6 +51,12 @@ jobs: shell: bash -l {0} steps: - uses: actions/checkout@v3 + + - name: Update Xcode on MacOS + if: runner.os == 'macos' + uses: maxim-lobanov/setup-xcode@v1 + with: + xcode-version: latest-stable # configures the mamba environment manager and builds the environment - name: Setup Mambaforge Python 3.7 From 4a4d8eed3b129812f7dd394db859a85943806bca Mon Sep 17 00:00:00 2001 From: Jackson Burns Date: Tue, 18 Apr 2023 20:43:00 -0400 Subject: [PATCH 05/24] remove xcode update --- .github/workflows/CI.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 42310fe6cd..6c61beb1c5 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -51,12 +51,6 @@ jobs: shell: bash -l {0} steps: - uses: actions/checkout@v3 - - - name: Update Xcode on MacOS - if: runner.os == 'macos' - uses: maxim-lobanov/setup-xcode@v1 - with: - xcode-version: latest-stable # configures the mamba environment manager and builds the environment - name: Setup Mambaforge Python 3.7 From 90ce8b63de5509b0ac13c3c702b727ca13b1cc42 Mon Sep 17 00:00:00 2001 From: Jackson Burns Date: Tue, 18 Apr 2023 20:49:48 -0400 Subject: [PATCH 06/24] install a global version of Julia on MacOS --- .github/workflows/CI.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 6c61beb1c5..eb5385f70b 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -88,6 +88,12 @@ jobs: make clean make + - name: Add Global Julia Install on MacOS + if: runner.os == 'macos' + uses: julia-actions/setup-julia@v1 + with: + version: "1.8.5" + # RMS installation and linking to Julia # Allow these installs to 'fail' (as they do in RMG-Tests) with the command || True trick - name: Install and link Julia dependencies From 5a56fd05a3d6794ae10774bc54442608bdab6292 Mon Sep 17 00:00:00 2001 From: Jackson Burns <33505528+JacksonBurns@users.noreply.github.com> Date: Tue, 2 May 2023 20:22:31 -0400 Subject: [PATCH 07/24] manually add PyCall to the MacOS global Julia install IIRC this is done somewhere in the rmg channel Julia custom build --- .github/workflows/CI.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index eb5385f70b..1e0d9d665d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -94,6 +94,11 @@ jobs: with: version: "1.8.5" + - name: Install PyCall on MacOS Global Julia + if: runner.os == 'macos' + run: | + julia -e 'using Pkg; Pkg.add("PyCall"); using PyCall' || true + # RMS installation and linking to Julia # Allow these installs to 'fail' (as they do in RMG-Tests) with the command || True trick - name: Install and link Julia dependencies From 39fea326be292dac5e025d2a4f71cf1c2dde332e Mon Sep 17 00:00:00 2001 From: JacksonBurns Date: Sun, 7 May 2023 21:46:42 -0400 Subject: [PATCH 08/24] can we use official Julia? is Killed 9 due to code signing? --- .github/workflows/CI.yml | 11 ----------- environment.yml | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 1e0d9d665d..6c61beb1c5 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -88,17 +88,6 @@ jobs: make clean make - - name: Add Global Julia Install on MacOS - if: runner.os == 'macos' - uses: julia-actions/setup-julia@v1 - with: - version: "1.8.5" - - - name: Install PyCall on MacOS Global Julia - if: runner.os == 'macos' - run: | - julia -e 'using Pkg; Pkg.add("PyCall"); using PyCall' || true - # RMS installation and linking to Julia # Allow these installs to 'fail' (as they do in RMG-Tests) with the command || True trick - name: Install and link Julia dependencies diff --git a/environment.yml b/environment.yml index f16342f1c5..09da47266a 100644 --- a/environment.yml +++ b/environment.yml @@ -37,7 +37,7 @@ dependencies: - rmg::pydas >=1.0.3 - pydot - rmg::pydqed >=1.0.3 - - rmg::pyjulia + - conda-forge::julia=1.8.5 - pymongo - pyparsing - rmg::pyrdl From 3fe9300f58d386bd36c37806e28806ad95234f13 Mon Sep 17 00:00:00 2001 From: Jackson Burns <33505528+JacksonBurns@users.noreply.github.com> Date: Thu, 11 May 2023 10:03:09 -0400 Subject: [PATCH 09/24] Set Julia path variables manually in GitHub runner our build of Julia was setting these two environment variables automatically by putting them into the conda activate environment script. we should try to replicate that here, but for now I am hardcoding the result for the runner to see if it will fix the precompilation issues from the previous actions run --- .github/workflows/CI.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 6c61beb1c5..f1e686834e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -76,10 +76,12 @@ jobs: git clone -b $RMG_DATABASE_BRANCH https://github.com/ReactionMechanismGenerator/RMG-database.git # modify env variables as directed in the RMG installation instructions - - name: Set PYTHONPATH and PATH + - name: Set Environment Variables run: | RUNNER_CWD=$(pwd) echo "PYTHONPATH=$RUNNER_CWD/RMG-Py:$PYTHONPATH" >> $GITHUB_ENV + echo "JULIA_DEPOT_PATH=/usr/share/miniconda3/envs/rmg_env/share/julia/site" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=/usr/share/miniconda3/envs/rmg_env/lib" >> $GITHUB_ENV echo "$RUNNER_CWD/RMG-Py" >> $GITHUB_PATH # RMG build step From 27da1c4159d3406e80f468a531ab64aa8d2d9ea8 Mon Sep 17 00:00:00 2001 From: Jackson Burns Date: Thu, 11 May 2023 11:08:59 -0400 Subject: [PATCH 10/24] add missing sparse matrix libs needed by conda julia --- .github/workflows/CI.yml | 2 -- environment.yml | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f1e686834e..1a4c287d8b 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -80,8 +80,6 @@ jobs: run: | RUNNER_CWD=$(pwd) echo "PYTHONPATH=$RUNNER_CWD/RMG-Py:$PYTHONPATH" >> $GITHUB_ENV - echo "JULIA_DEPOT_PATH=/usr/share/miniconda3/envs/rmg_env/share/julia/site" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=/usr/share/miniconda3/envs/rmg_env/lib" >> $GITHUB_ENV echo "$RUNNER_CWD/RMG-Py" >> $GITHUB_PATH # RMG build step diff --git a/environment.yml b/environment.yml index 09da47266a..dd97f5a776 100644 --- a/environment.yml +++ b/environment.yml @@ -36,8 +36,9 @@ dependencies: - psutil - rmg::pydas >=1.0.3 - pydot - - rmg::pydqed >=1.0.3 - conda-forge::julia=1.8.5 + - conda-forge::suitesparse + - rmg::pydqed >=1.0.3 - pymongo - pyparsing - rmg::pyrdl From f89c0d1b5e6955b88af5d876a27c46c786ef4a40 Mon Sep 17 00:00:00 2001 From: Jackson Burns Date: Thu, 11 May 2023 18:36:04 -0400 Subject: [PATCH 11/24] add a 'summary' job to CI so that we can still apply branch protection we cannot use branch protection rules on jobs that have matrix in them, so we add a summary step that will wait for both the Mac and Linux builds+tests to execute and then return pass or fail. we will need to update the branch protection rule for main to look for ci-report-status instead of -build-and-test-linux --- .github/workflows/CI.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 1a4c287d8b..7b59770230 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -37,7 +37,7 @@ concurrency: cancel-in-progress: true jobs: - build-and-test-linux: + build-and-test-unix: strategy: matrix: os: [ubuntu-latest, macos-latest] @@ -231,6 +231,21 @@ jobs: mamba install -y -c conda-forge codecov codecov + # This allows us to have a branch protection rule for tests and deploys with matrix + # + # taken from https://github.com/orgs/community/discussions/4324#discussioncomment-3477871 + ci-report-status: + runs-on: ubuntu-20.04 + needs: build-and-test-unix + if: always() + steps: + - name: Successful CI + if: ${{ !(contains(needs.build-and-test-unix.result, 'failure')) }} + run: exit 0 + - name: Failing CI + if: ${{ contains(needs.build-and-test-unix.result, 'failure') }} + run: exit 1 + build-and-push-docker: # after testing and on pushes to main, build and push docker image # technically we could live without the 'needs' since _in theory_ @@ -238,7 +253,7 @@ jobs: # who knows ¯\_(ツ)_/¯ # # taken from https://github.com/docker/build-push-action - needs: build-and-test-linux + needs: build-and-test-unix runs-on: ubuntu-latest if: github.ref == 'refs/heads/main' && github.repository == 'ReactionMechanismGenerator/RMG-Py' steps: From 3ef7225e95c176d1d257fdb4268000986b290989 Mon Sep 17 00:00:00 2001 From: Jackson Burns Date: Thu, 11 May 2023 18:58:39 -0400 Subject: [PATCH 12/24] update developer install docs for conda Julia and special M1 instruction --- .../rmg/installation/anacondaDeveloper.rst | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/documentation/source/users/rmg/installation/anacondaDeveloper.rst b/documentation/source/users/rmg/installation/anacondaDeveloper.rst index 63d1716e74..25c096d87b 100644 --- a/documentation/source/users/rmg/installation/anacondaDeveloper.rst +++ b/documentation/source/users/rmg/installation/anacondaDeveloper.rst @@ -47,17 +47,6 @@ Installation by Source Using Anaconda Environment for Unix-based Systems: Linux to install this is to simply run one of the commands in the terminal, e.g. ``git``. The terminal will then prompt you on whether or not you would like to install the Command Line Tools. - For MacOS users only, download and install MacOS Julia 1.8 from here: . Then add Julia to PATH by running the following command: :: - - echo 'export PATH="/Applications/Julia-1.8.app/Contents/Resources/julia/bin:$PATH"' >> ~/.bash_profile - - export PATH="/Applications/Julia-1.8.app/Contents/Resources/julia/bin:$PATH" - - If using MacOS Catalina or newer your terminal may be using ``zsh`` by default, in which case you should replace ``.bash_profile`` with ``.zshrc``. - - Note that this Julia install will not respect conda environmental boundaries meaning only one conda environment can be linked to it at a time. - - #. Install the latest versions of RMG and RMG-database through cloning the source code via Git. Make sure to start in an appropriate local directory where you want both RMG-Py and RMG-database folders to exist. Github has deprecated password authentication from the command line, so it @@ -84,6 +73,16 @@ Installation by Source Using Anaconda Environment for Unix-based Systems: Linux cd RMG-Py conda env create -f environment.yml +.. warning:: Apple Silicon (M1+) Users + Execute the following commands instead of ``conda env create -f environment.yml``: :: + + conda create -n rmg_env + conda activate rmg_env + conda config --env --set subdir osx-64 + conda install python=3.7 + conda env update -f environment.yml +.. + If either of these commands return an error due to being unable to find the ``conda`` command, try to either close and reopen your terminal to refresh your environment variables or type the following command. If on Linux or pre-Catalina MacOS :: @@ -128,6 +127,14 @@ Installation by Source Using Anaconda Environment for Unix-based Systems: Linux julia -e 'using Pkg; Pkg.add(PackageSpec(name="ReactionMechanismSimulator",rev="main")); using ReactionMechanismSimulator;' +.. warning:: Apple Silicon (M1+) Users + Execute the following commands instead of ``conda env create -f environment.yml``: :: + + julia -e 'using Pkg; Pk.add("PyCall");Pkg.build("PyCall");Pkg.add(PackageSpec(name="ReactionMechanismSimulator",rev="main")); using ReactionMechanismSimulator;' + + python -c "import julia; julia.install(); import diffeqpy; diffeqpy.install()" +.. + Note that this links your python to python-jl enabling calls to Julia through pyjulia. Occasionally programs will interact with python-jl differently than the default python. If this occurs for you we recommend doing that operation in a different conda environment. However, if convenient you can undo this linking by replacing python-jl with From adf003fbf28d56d38f586b222ba3986839e9cd40 Mon Sep 17 00:00:00 2001 From: Jackson Burns Date: Mon, 15 May 2023 23:23:35 -0400 Subject: [PATCH 13/24] add EXTENSIVE inline docs, make the dependence on pyjulia explicit --- environment.yml | 102 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 72 insertions(+), 30 deletions(-) diff --git a/environment.yml b/environment.yml index dd97f5a776..62e2cc45e0 100644 --- a/environment.yml +++ b/environment.yml @@ -1,3 +1,20 @@ +# environment.yml - conda environment specification file for RMG-Py +# +# Usage: conda env create --file environment.yml +# +# This file contains all of the software packages needed to run RMG-Py. +# There is a mixture of the following +# - packlages would could be installed at OS level, but we install here +# for better version control +# - python tools +# - external software tools specific to chemistry +# - other software we maintain which RMG depends on +# + some other categories (see below) +# +# Changelog: +# - May 15, 2023 Added this changelog, added inline documentation, +# made depdency list more explicit (@JacksonBurns). +# name: rmg_env channels: - defaults @@ -5,51 +22,76 @@ channels: - conda-forge - cantera dependencies: +# System-level dependencies - we could install these at the OS level +# but by installing them in the conda environment we get better control - cairo - cairocffi + - ffmpeg + - xlrd + - xlwt + - h5py + - graphviz + - markupsafe + - psutil + - conda-forge::ncurses + - conda-forge::suitesparse + +# external software tools for chemistry + - coolprop - cantera::cantera=2.6 + - conda-forge::mopac - conda-forge::cclib >=1.6.3 - - rmg::chemprop - - coolprop + - conda-forge::openbabel >= 3 + +# general-purpose external software tools + - conda-forge::julia=1.8.5 + +# Python tools + - python >=3.7 - coverage - cython >=0.25.2 - - rmg::diffeqpy - - ffmpeg - - rmg::gprof2dot - - graphviz - - h5py + - scikit-learn + - scipy + - numpy >=1.10.0 + - pydot - jinja2 - jupyter - - rmg::lpsolve55 - - markupsafe + - pymongo + - pyparsing + - pyyaml + - networkx + - nose - matplotlib >=1.5 - - conda-forge::mopac - mpmath + - pandas + +# packages we maintain + - rmg::gprof2dot + - rmg::lpsolve55 - rmg::muq2 - - networkx - - nomkl - - nose - rmg::numdifftools - - numpy >=1.10.0 - - conda-forge::openbabel >= 3 - - pandas - - psutil - rmg::pydas >=1.0.3 - - pydot - - conda-forge::julia=1.8.5 - - conda-forge::suitesparse - rmg::pydqed >=1.0.3 - - pymongo - - pyparsing - rmg::pyrdl - rmg::pyrms - - python >=3.7 - - pyyaml - rmg::quantities - - rmg::rdkit >=2020.03.3.0 - - scikit-learn - - scipy - rmg::symmetry - - xlrd - - xlwt - - conda-forge::ncurses + - rmg::pyjulia + +# packages we would like to stop maintaining (and why) + - rmg::diffeqpy + # we should use the official verison https://github.com/SciML/diffeqpy), + # rather than ours (which is only made so that we can get it from conda) + # It is only on pip, so we will need to do something like: + # https://stackoverflow.com/a/35245610 + + - rmg::chemprop + # Our build of this is version 0.0.1 (!!) and we are using parts + # of the API that are now gone. Need a serious PR to fix this. + + - rmg::rdkit >=2020.03.3.0 + # We should use the official channel, not sure how difficult this + # change will be. + +# conda mutex metapackage + - nomkl From 814419c2372b083831710dfe2b660ac99fb11cd7 Mon Sep 17 00:00:00 2001 From: Jackson Burns Date: Mon, 15 May 2023 23:27:10 -0400 Subject: [PATCH 14/24] remove redundant line from developer instructions --- .../source/users/rmg/installation/anacondaDeveloper.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/documentation/source/users/rmg/installation/anacondaDeveloper.rst b/documentation/source/users/rmg/installation/anacondaDeveloper.rst index 25c096d87b..e3539521cc 100644 --- a/documentation/source/users/rmg/installation/anacondaDeveloper.rst +++ b/documentation/source/users/rmg/installation/anacondaDeveloper.rst @@ -79,7 +79,6 @@ Installation by Source Using Anaconda Environment for Unix-based Systems: Linux conda create -n rmg_env conda activate rmg_env conda config --env --set subdir osx-64 - conda install python=3.7 conda env update -f environment.yml .. From dc03b096210d95d485682efebae0b62be50c0cee Mon Sep 17 00:00:00 2001 From: Jackson Burns <33505528+JacksonBurns@users.noreply.github.com> Date: Tue, 16 May 2023 09:24:36 -0400 Subject: [PATCH 15/24] changes result aggregation job to use `ubuntu-latest` --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 7b59770230..0ad3b615a9 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -235,7 +235,7 @@ jobs: # # taken from https://github.com/orgs/community/discussions/4324#discussioncomment-3477871 ci-report-status: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest needs: build-and-test-unix if: always() steps: From a6f23cd2e9950af9e92fecc4bf361d224a5e8a18 Mon Sep 17 00:00:00 2001 From: Jackson Burns Date: Tue, 16 May 2023 10:39:02 -0400 Subject: [PATCH 16/24] set julia to `>=1.8.5` and use `conda-forge` distribution of `pyjulia` --- environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 62e2cc45e0..d8fb1a9f1c 100644 --- a/environment.yml +++ b/environment.yml @@ -44,7 +44,8 @@ dependencies: - conda-forge::openbabel >= 3 # general-purpose external software tools - - conda-forge::julia=1.8.5 + - conda-forge::julia>=1.8.5 + - conda-forge::pyjulia # Python tools - python >=3.7 @@ -76,7 +77,6 @@ dependencies: - rmg::pyrms - rmg::quantities - rmg::symmetry - - rmg::pyjulia # packages we would like to stop maintaining (and why) - rmg::diffeqpy From 8749d17bc99d897931b0665103a3d4b10041f8f3 Mon Sep 17 00:00:00 2001 From: Jackson Burns Date: Tue, 16 May 2023 11:06:16 -0400 Subject: [PATCH 17/24] developer installation instructions update, see extended description - break up the environment creation step s.t. the admonition for Mac M1 users are more clear - change the Julia and Python installation instructions so that they work for all platforms in the same order --- .../rmg/installation/anacondaDeveloper.rst | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/documentation/source/users/rmg/installation/anacondaDeveloper.rst b/documentation/source/users/rmg/installation/anacondaDeveloper.rst index e3539521cc..dd2bff3eb6 100644 --- a/documentation/source/users/rmg/installation/anacondaDeveloper.rst +++ b/documentation/source/users/rmg/installation/anacondaDeveloper.rst @@ -68,10 +68,9 @@ Installation by Source Using Anaconda Environment for Unix-based Systems: Linux conda install -n base conda-libmamba-solver conda config --set solver libmamba - Now create the conda environment for RMG-Py :: + Navigate to the RMG-Py directory :: cd RMG-Py - conda env create -f environment.yml .. warning:: Apple Silicon (M1+) Users Execute the following commands instead of ``conda env create -f environment.yml``: :: @@ -82,6 +81,10 @@ Installation by Source Using Anaconda Environment for Unix-based Systems: Linux conda env update -f environment.yml .. + Now create the conda environment for RMG-Py :: + + conda env create -f environment.yml + If either of these commands return an error due to being unable to find the ``conda`` command, try to either close and reopen your terminal to refresh your environment variables or type the following command. If on Linux or pre-Catalina MacOS :: @@ -120,19 +123,11 @@ Installation by Source Using Anaconda Environment for Unix-based Systems: Linux Be sure to either close and reopen your terminal to refresh your environment variables (``source ~/.bashrc`` or ``source ~/.zshrc``). -#. Install and Link Julia dependencies :: - - python -c "import julia; julia.install(); import diffeqpy; diffeqpy.install()" +#. Install and Link Julia dependencies: :: - julia -e 'using Pkg; Pkg.add(PackageSpec(name="ReactionMechanismSimulator",rev="main")); using ReactionMechanismSimulator;' + julia -e 'using Pkg; Pkg.add("PyCall");Pkg.build("PyCall");Pkg.add(PackageSpec(name="ReactionMechanismSimulator",rev="main")); using ReactionMechanismSimulator;' -.. warning:: Apple Silicon (M1+) Users - Execute the following commands instead of ``conda env create -f environment.yml``: :: - - julia -e 'using Pkg; Pk.add("PyCall");Pkg.build("PyCall");Pkg.add(PackageSpec(name="ReactionMechanismSimulator",rev="main")); using ReactionMechanismSimulator;' - - python -c "import julia; julia.install(); import diffeqpy; diffeqpy.install()" -.. + python -c "import julia; julia.install(); import diffeqpy; diffeqpy.install()" Note that this links your python to python-jl enabling calls to Julia through pyjulia. Occasionally programs will interact with python-jl differently than the default python. If this occurs for you we recommend doing that operation From aedc44eab6f279a78defb6a3e6ed95271f077003 Mon Sep 17 00:00:00 2001 From: Richard West Date: Tue, 16 May 2023 13:45:42 -0400 Subject: [PATCH 18/24] Remove redundant explanation about linking python to python-jl This explanation was added in commit 5d208269ce5dabc73d3152170ac75f1dfbce4bd1 but I think it refers to the line ln -sfn $(which python-jl) $(which python) that was added in b6910a76fdef684995a886384f50bbdd2eace901 (2 minutes later) to the continuous integrations workflow, but not actually added to the user instructions. i.e. the linking step (that this explanation refers to) is not in the instructions. So I suggest we remove it. --- .../source/users/rmg/installation/anacondaDeveloper.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/documentation/source/users/rmg/installation/anacondaDeveloper.rst b/documentation/source/users/rmg/installation/anacondaDeveloper.rst index dd2bff3eb6..ac00f01aef 100644 --- a/documentation/source/users/rmg/installation/anacondaDeveloper.rst +++ b/documentation/source/users/rmg/installation/anacondaDeveloper.rst @@ -129,10 +129,6 @@ Installation by Source Using Anaconda Environment for Unix-based Systems: Linux python -c "import julia; julia.install(); import diffeqpy; diffeqpy.install()" - Note that this links your python to python-jl enabling calls to Julia through pyjulia. Occasionally programs will - interact with python-jl differently than the default python. If this occurs for you we recommend doing that operation - in a different conda environment. However, if convenient you can undo this linking by replacing python-jl with - python3 in the second command above. Just make sure to rerun the linking command once you are done. #. Finally, you can run RMG from any location by typing the following (given that you have prepared the input file as ``input.py`` in the current folder). :: @@ -141,8 +137,6 @@ Installation by Source Using Anaconda Environment for Unix-based Systems: Linux You may now use RMG-Py, Arkane, as well as any of the :ref:`Standalone Modules ` included in the RMG-Py package. - - Test Suite ========== From e0e2601dd9dbb8f6b3eed521c0d5153e3799c8d0 Mon Sep 17 00:00:00 2001 From: Richard West Date: Tue, 16 May 2023 13:57:24 -0400 Subject: [PATCH 19/24] Rewording install instructions a bit. Mostly whitespace and linebreaks. Added a new step number. --- .../rmg/installation/anacondaDeveloper.rst | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/documentation/source/users/rmg/installation/anacondaDeveloper.rst b/documentation/source/users/rmg/installation/anacondaDeveloper.rst index ac00f01aef..0c63ef6e53 100644 --- a/documentation/source/users/rmg/installation/anacondaDeveloper.rst +++ b/documentation/source/users/rmg/installation/anacondaDeveloper.rst @@ -18,7 +18,8 @@ Installation by Source Using Anaconda Environment for Unix-based Systems: Linux Note that you should restart your terminal in order for the changes to take effect, as the installer will tell you. #. There are a few system-level dependencies which are required and should not be installed via Anaconda. These include - `Git `_ for version control, `GNU Make `_, and the C and C++ compilers from the `GNU Compiler Collection (GCC) `_ for compiling RMG. + `Git `_ for version control, `GNU Make `_, + and the C and C++ compilers from the `GNU Compiler Collection (GCC) `_ for compiling RMG. For Linux users, you can check whether these are already installed by simply calling them via the command line, which will let you know if they are missing. To install any missing packages, you should use the appropriate package manager @@ -68,30 +69,33 @@ Installation by Source Using Anaconda Environment for Unix-based Systems: Linux conda install -n base conda-libmamba-solver conda config --set solver libmamba - Navigate to the RMG-Py directory :: +#. Navigate to the RMG-Py directory :: cd RMG-Py -.. warning:: Apple Silicon (M1+) Users - Execute the following commands instead of ``conda env create -f environment.yml``: :: - +#. Apple silicon (M1+) users only: execute the following commands + **instead of** the following `conda env create -f environment.yml` step. + (This will tell conda that we want to the environment to use x86 + architecture rather than the native ARM64 architecture) :: + conda create -n rmg_env conda activate rmg_env conda config --env --set subdir osx-64 conda env update -f environment.yml -.. - Now create the conda environment for RMG-Py :: +#. Create the conda environment for RMG-Py :: conda env create -f environment.yml - If either of these commands return an error due to being unable to find the ``conda`` command, try to either close and reopen your terminal to refresh your environment variables or type the following command. + If either of these commands return an error due to being unable to find the ``conda`` command, + try to either close and reopen your terminal to refresh your environment variables + or type the following command. - If on Linux or pre-Catalina MacOS :: + If on Linux or pre-Catalina MacOS (or if you have a bash shell):: source ~/.bashrc - If on MacOS Catalina or later :: + If on MacOS Catalina or later (or if you have a Z shell):: source ~/.zshrc @@ -111,10 +115,10 @@ Installation by Source Using Anaconda Environment for Unix-based Systems: Linux #. Modify environment variables. Add RMG-Py to the PYTHONPATH to ensure that you can access RMG modules from any folder. Also, add your RMG-Py folder to PATH to launch ``rmg.py`` from any folder. - In general, these commands should be placed in the appropriate shell initialization file. For Linux users using - bash (the default on distributions mentioned here), these should be placed in ``~/.bashrc``. For MacOS users using bash (default before MacOS Catalina), - these should be placed in ``~/.bash_profile``, which you should create if it doesn't exist. For MacOS users using zsh - (default beginning in MacOS Catalina), these should be placed in ``~/.zshrc``. :: + In general, these commands should be placed in the appropriate shell initialization file. + For Linux users using bash (the default on distributions mentioned here), these should be placed in ``~/.bashrc``. + For MacOS users using bash (default before MacOS Catalina), these should be placed in ``~/.bash_profile``, which you should create if it doesn't exist. + For MacOS users using zsh (default beginning in MacOS Catalina), these should be placed in ``~/.zshrc``. :: export PYTHONPATH=YourFolder/RMG-Py/:$PYTHONPATH export PATH=YourFolder/RMG-Py/:$PATH From f5e7971f0ce646acfe4a1bf98c2279518e3524ee Mon Sep 17 00:00:00 2001 From: Richard West Date: Tue, 16 May 2023 13:58:41 -0400 Subject: [PATCH 20/24] Remove the note about activating old anaconda environments. This changed many years ago now. Although some clusters (eg. at Northeastern) still have an old old anaconda, I no longer feel it's our job to explain how to use it, and just makes the instructions longer than needed. --- .../source/users/rmg/installation/anacondaDeveloper.rst | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/documentation/source/users/rmg/installation/anacondaDeveloper.rst b/documentation/source/users/rmg/installation/anacondaDeveloper.rst index 0c63ef6e53..8aabfb238b 100644 --- a/documentation/source/users/rmg/installation/anacondaDeveloper.rst +++ b/documentation/source/users/rmg/installation/anacondaDeveloper.rst @@ -102,12 +102,7 @@ Installation by Source Using Anaconda Environment for Unix-based Systems: Linux #. Activate conda environment :: conda activate rmg_env - - Note regarding differences between conda versions: Prior to Anaconda 4.4, the command to activate an environment was - ``source activate rmg_env``. It has since been changed to ``conda activate rmg_env`` due to underlying changes to - standardize operation across different operating systems. However, a prerequisite to using the new syntax is having - run the ``conda init`` setup routine, which can be done at the end of the install procedure if the user requests. - + #. Compile RMG-Py after activating the conda environment :: make From 39cb5455020d7f58d4629f458b2801cc67c24014 Mon Sep 17 00:00:00 2001 From: Jackson Burns Date: Tue, 16 May 2023 13:54:36 -0400 Subject: [PATCH 21/24] turn out we can't use conda-forge pyjulia, see extended desc for why i have added comments to the environment file, the issue is that our build of pyjulia does some `sed` magic that we do not replicate otherwise. would be good to get rid of this in the future, but this PR is already getting out of hand --- environment.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index d8fb1a9f1c..3a50658d92 100644 --- a/environment.yml +++ b/environment.yml @@ -45,7 +45,6 @@ dependencies: # general-purpose external software tools - conda-forge::julia>=1.8.5 - - conda-forge::pyjulia # Python tools - python >=3.7 @@ -93,5 +92,13 @@ dependencies: # We should use the official channel, not sure how difficult this # change will be. + - rmg::pyjulia + # This is identical to the conda-forge package, except that we run + # a hard to decipher sed command during the build process: + # https://github.com/ReactionMechanismGenerator/conda-recipes/blob/rmg-deps/pyjulia/build.sh#LL15C69-L15C69 + # + # We should either remove the need to use this command or add this + # to the installation steps + # conda mutex metapackage - nomkl From 45a6de6c2adeee95a7776dd9f9a59d446f179364 Mon Sep 17 00:00:00 2001 From: Jackson Burns Date: Wed, 17 May 2023 08:59:54 -0400 Subject: [PATCH 22/24] add further clarifying comments for pyjulia, ncurses, and gfortran - who needs pyjulia - why use conda-forge ncurses instead of default - gfortran is not listed in the env file, explain why --- environment.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/environment.yml b/environment.yml index 3a50658d92..13de5a5a8a 100644 --- a/environment.yml +++ b/environment.yml @@ -33,6 +33,8 @@ dependencies: - graphviz - markupsafe - psutil + # conda-forge not default, since default has a version information bug + # (see https://github.com/ReactionMechanismGenerator/RMG-Py/pull/2421) - conda-forge::ncurses - conda-forge::suitesparse @@ -99,6 +101,18 @@ dependencies: # # We should either remove the need to use this command or add this # to the installation steps + # + # Both pyrms and diffeqpy depend on this package. # conda mutex metapackage - nomkl + +# additional packages that are required, but not specified here (and why) + # pydqed, pydas, mopac, and likely others require a fortran compiler (specifically gfortran) + # in the environment. Normally we would add this to the environment file with + # - libgfortran-ng >= 10 + # but this exact package is only maintained for Linux, meaning that if we were to add + # it here the environment creation would fail on Mac. The way it ends up working behind + # the scenes is that conda will find a different package for Mac that provides gfortran, + # but because we cannot specify per-platform requirements in this file we simply leave + # it out. \ No newline at end of file From 1cafe962c40c457d40ed8d9f7e5a85a33b311072 Mon Sep 17 00:00:00 2001 From: Richard West Date: Wed, 17 May 2023 16:23:21 -0400 Subject: [PATCH 23/24] Emphasize adding RMG-Py to the PYTHONPATH. Otherwise the PyCall step installs it via conda! It looks like this: 279 dependencies successfully precompiled in 550 seconds. 14 already precompiled. [ Info: Installing rmgpy.molecule via the Conda rmg package... [ Info: Running `conda config --add channels rmg --file /opt/miniconda3/envs/rmg_env3/condarc-julia.yml --force` in root environment [ Info: Running `conda install -y rmg` in root environment (then you get the old binary from the rmg conda channel instead of your developer version) Or at least I *think* this is why that happened. When I did the whole thing again with this step included I didn't have the same problem. --- .../source/users/rmg/installation/anacondaDeveloper.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/source/users/rmg/installation/anacondaDeveloper.rst b/documentation/source/users/rmg/installation/anacondaDeveloper.rst index 8aabfb238b..74532c01f8 100644 --- a/documentation/source/users/rmg/installation/anacondaDeveloper.rst +++ b/documentation/source/users/rmg/installation/anacondaDeveloper.rst @@ -108,6 +108,7 @@ Installation by Source Using Anaconda Environment for Unix-based Systems: Linux make #. Modify environment variables. Add RMG-Py to the PYTHONPATH to ensure that you can access RMG modules from any folder. + *This is important before the next step in which julia dependencies are installed.* Also, add your RMG-Py folder to PATH to launch ``rmg.py`` from any folder. In general, these commands should be placed in the appropriate shell initialization file. From 25cbbc72f4f06b747dfb8f683362febcab2a960f Mon Sep 17 00:00:00 2001 From: Richard West Date: Wed, 17 May 2023 18:13:32 -0400 Subject: [PATCH 24/24] Dev install instructions: Switch the rmg_env to libmamba solver. The solver setting is environment-specific. If you set it before you make the rmg_env then it is not set IN the rmg_env and when you later do stuff to that environment, it's super slow. --- .../source/users/rmg/installation/anacondaDeveloper.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/documentation/source/users/rmg/installation/anacondaDeveloper.rst b/documentation/source/users/rmg/installation/anacondaDeveloper.rst index 74532c01f8..2b3edb6f0a 100644 --- a/documentation/source/users/rmg/installation/anacondaDeveloper.rst +++ b/documentation/source/users/rmg/installation/anacondaDeveloper.rst @@ -103,6 +103,10 @@ Installation by Source Using Anaconda Environment for Unix-based Systems: Linux conda activate rmg_env +#. Switch the conda solver to libmamba again, to accelerate any changes you might make to this conda environment in the future:: + + conda config --set solver libmamba + #. Compile RMG-Py after activating the conda environment :: make