Merge branch 'master' into swiss-gmpes

gem · Sep 21, 2023 · 0c69cda · 0c69cda
2 parents a7e0a88 + a9c6164
commit 0c69cda
Show file tree

Hide file tree

Showing 796 changed files with 158,205 additions and 89,477 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,3 @@
+* text=auto
+*.xml text eol=lf
+*.csv text eol=crlf
diff --git a/.github/workflows/cluster_ansible.yml b/.github/workflows/cluster_ansible.yml
diff --git a/.github/workflows/engine_test.yml b/.github/workflows/engine_test.yml
@@ -13,6 +13,8 @@ jobs:
       matrix:
         os: [ubuntu-22.04]
         python-version: ["3.10"]
+    env:
+      GITHUB_HEAD_REF:  ${{ github.head_ref }}
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python  ${{ matrix.python-version }}
@@ -21,20 +23,22 @@ jobs:
         python-version:  ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python install.py devel
+        python install.py devel --version=$GITHUB_HEAD_REF
     - name: Calculators and documentation tests
       run: |
         source ~/openquake/bin/activate
-        pip install pytest
+        pip install pytest https://wheelhouse.openquake.org/v3/py/rtgmpy-1.0.0-py3-none-any.whl
         oq dbserver start
-        pytest --doctest-modules -x --disable-warnings --color=yes --durations=10 openquake/calculators && pytest doc/adv-manual/*.rst
+        pytest --doctest-modules -x --disable-warnings --color=yes --durations=10 openquake/calculators -k 'not risk and not damage' && pytest doc/adv-manual/*.rst
 
   hazardlib:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest]
         python-version: ["3.10"]
+    env:
+      GITHUB_HEAD_REF:  ${{ github.head_ref }}
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python  ${{ matrix.python-version }}
@@ -43,21 +47,24 @@ jobs:
         python-version:  ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python install.py devel
+        python install.py devel --version=$GITHUB_HEAD_REF
     - name: Hazardlib tests
       run: |
         source ~/openquake/bin/activate
-        pip install pyshp pytest flake8
+        pip install pyshp pytest flake8 https://wheelhouse.openquake.org/v3/py/rtgmpy-1.0.0-py3-none-any.whl
         oq dbserver start
         cd openquake
-        pytest -x --doctest-modules --disable-warnings --color=yes --durations=10 hazardlib sep commands engine hmtk risklib commonlib baselib
+        pytest calculators -k 'risk or damage' -x --doctest-modules --disable-warnings --color=yes --durations=5
+        pytest -x --doctest-modules --disable-warnings --color=yes --durations=8 sep commands engine hmtk risklib commonlib baselib hazardlib
 
   server_public_mode:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest]
         python-version: ["3.10"]
+    env:
+      GITHUB_HEAD_REF:  ${{ github.head_ref }}
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python  ${{ matrix.python-version }}
@@ -66,19 +73,22 @@ jobs:
         python-version:  ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python install.py devel
+        python install.py devel --version=$GITHUB_HEAD_REF
     - name: Server 'PUBLIC' mode tests
       run: |
         source ~/openquake/bin/activate
         oq dbserver start
-        OQ_APPLICATION_MODE=public ./openquake/server/manage.py test tests.test_public_mode
+        # -v 2 also logs the test names
+        OQ_APPLICATION_MODE=public ./openquake/server/manage.py test -v 2 tests.test_public_mode
 
   server_read_only_mode:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest]
         python-version: ["3.10"]
+    env:
+      GITHUB_HEAD_REF:  ${{ github.head_ref }}
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python  ${{ matrix.python-version }}
@@ -87,19 +97,22 @@ jobs:
         python-version:  ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python install.py devel
+        python install.py devel --version=$GITHUB_HEAD_REF
     - name: Server 'READ_ONLY' mode tests
       run: |
         source ~/openquake/bin/activate
         oq dbserver start
-        OQ_APPLICATION_MODE=read_only ./openquake/server/manage.py test tests.test_read_only_mode
+        # -v 2 also logs the test names
+        OQ_APPLICATION_MODE=read_only ./openquake/server/manage.py test -v 2 tests.test_read_only_mode
 
   server_aelo_mode:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest]
         python-version: ["3.10"]
+    env:
+      GITHUB_HEAD_REF:  ${{ github.head_ref }}
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python  ${{ matrix.python-version }}
@@ -108,9 +121,10 @@ jobs:
         python-version:  ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python install.py devel
+        python install.py devel --version=$GITHUB_HEAD_REF
     - name: Server 'AELO' mode tests
       run: |
         source ~/openquake/bin/activate
         oq dbserver start
-        OQ_APPLICATION_MODE=aelo OQ_CONFIG_FILE=openquake/server/tests/data/openquake.cfg ./openquake/server/manage.py test tests.test_aelo_mode
+        # -v 2 also logs the test names
+        OQ_APPLICATION_MODE=aelo OQ_CONFIG_FILE=openquake/server/tests/data/openquake.cfg ./openquake/server/manage.py test -v 2 tests.test_aelo_mode
diff --git a/.github/workflows/macos_intel_install.yml b/.github/workflows/macos_intel_install.yml
@@ -9,15 +9,15 @@ name: MacOS Intel install and test
   schedule:
     - cron: "0 1 * * *"
   push:
-    branches: [ test_os ]
+    branches: [macos_test]
 
 jobs:
 
-  Install_and_test:
+  install_and_test:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-13, macos-12]
+        os: [macos-13]
         python-version: [3.9, "3.10"]
     steps:
     # This Checkout use git-ref keyword from dispatch
@@ -37,13 +37,6 @@ jobs:
         env: 
           BRANCH: ${{ github.event.inputs.git-ref }}
         run: |
-          #PY_VER=`echo py${{ matrix.python-version }} | tr -d .`
-          #echo $PY_VER
-          #pip3 install -r requirements-$PY_VER-macos.txt
-          #pip3 install -e .\[osgeo\]
-          echo "sqlite3 version"
-          which sqlite3
-          sqlite3 --version
           export PIP_DEFAULT_TIMEOUT=100
           pip3 install -U pip wheel setuptools
           #
@@ -54,15 +47,67 @@ jobs:
           else
             python install.py devel 
           fi
+          source ~/openquake/bin/activate
+          pip3 install pytest pyshp flake8
+          oq --version  # makes numba compile, so dbserver start becomes quicker
+          oq dbserver start
+          sleep 10
+
       - name: Run demos to test installation
         run: |
+          set -x
           source ~/openquake/bin/activate
-          pip3 install pytest
-          oq --version
-          oq dbserver start
-          #pytest -vsx --color=yes ~/work/oq-engine/oq-engine/openquake/hazardlib/tests/gsim/kotha_2020_test.py
-          #pytest -vsx --color=yes /Users/runner/work/oq-engine/oq-engine/openquake/sep
-          #pytest -vs  --color=yes /Users/runner/work/oq-engine/oq-engine/openquake/calculators
-          oq engine --run https://downloads.openquake.org/jobs/risk_test.zip
-          sleep 3
-          oq engine --run "https://github.com/gem/oq-engine/blob/master/openquake/server/tests/data/classical.zip?raw=true"
+          oq info venv
+          oq info cfg
+          cd ~/work/oq-engine/oq-engine/demos
+          ls -lrt
+          # run demos with job_hazard.ini and job_risk.ini
+          for demo_dir in $(find . -type d | sort); do
+             if [ -f $demo_dir/job_hazard.ini ]; then
+                 oq engine --run $demo_dir/job_hazard.ini --exports csv,hdf5
+                 oq engine --run $demo_dir/job_risk.ini --exports csv,hdf5 --hc -1
+             fi
+          done
+          # run the other demos
+          for ini in $(find . -name job.ini | sort); do
+              oq engine --run $ini --exports csv,hdf5
+          done
+
+      - name: Run tests for calculators and advanced manual to test installation
+        if: always()
+        run: |
+          source ~/openquake/bin/activate
+          cd ~/work/oq-engine/oq-engine
+          pytest --doctest-modules --disable-warnings --color=yes --durations=10 openquake/calculators && pytest doc/adv-manual/*.rst
+
+      - name: Run tests for hazardlib, sep, commands, engine, hmtk, risklib, commonlib and baselib to test installation
+        if: always()
+        run: |
+          source ~/openquake/bin/activate
+          cd ~/work/oq-engine/oq-engine/openquake
+          pytest --doctest-modules --disable-warnings --color=yes --durations=10 hazardlib sep commands engine hmtk risklib commonlib baselib
+
+      - name: Run tests for the engine server in public mode to test installation
+        if: always()
+        run: |
+          source ~/openquake/bin/activate
+          cd ~/work/oq-engine/oq-engine
+          # -v 2 also logs the test names
+          OQ_APPLICATION_MODE=public ./openquake/server/manage.py test -v 2 tests.test_public_mode
+
+      - name: Run tests for the engine server in read-only mode to test installation
+        if: always()
+        run: |
+          source ~/openquake/bin/activate
+          cd ~/work/oq-engine/oq-engine
+          # -v 2 also logs the test names
+          OQ_APPLICATION_MODE=read_only ./openquake/server/manage.py test -v 2 tests.test_read_only_mode
+
+      # NOTE: we shouldn't need to install the engine in AELO mode on mac
+      # - name: Run tests for the engine server in aelo mode to test installation
+      #   if: always()
+      #   run: |
+      #     source ~/openquake/bin/activate
+      #     cd ~/work/oq-engine/oq-engine
+      #     -v 2 also logs the test names
+      #     OQ_APPLICATION_MODE=aelo OQ_CONFIG_FILE=openquake/server/tests/data/openquake.cfg ./openquake/server/manage.py test -v 2 tests.test_aelo_mode
diff --git a/.github/workflows/macos_m1_monterey_install.yml b/.github/workflows/macos_m1_monterey_install.yml
@@ -7,7 +7,9 @@ name: MacOS Monterey M1
         default: master
         required: true
   schedule:
-    - cron: "15 12 * * *"
+    - cron: "15 23 * * *"
+  push:
+    branches: [macos_test]
 
 jobs:
 
@@ -16,8 +18,7 @@ jobs:
     strategy:
       matrix:
         os: [m1]
-        #python-version: [3.9.13, "3.10.9"]
-        python-version: [3.9.13]
+        python-version: [3.9.13, "3.10.11"]
     steps:
     # This Checkout use git-ref keyword from dispatch
       - name: Clone Repository (Master)
@@ -54,7 +55,7 @@ jobs:
           if [ -e "$inst_cmd" ]; then
               sh "$inst_cmd"
           fi
-      - name: Run Installation of oq-engine in devel mode
+      - name: Install oq-engine in devel mode
         env:
           BRANCH: ${{ github.event.inputs.git-ref }}
         run: |
@@ -74,20 +75,65 @@ jobs:
           else
             $python install.py devel
           fi
-      - name: Run demos to test installation
-        run: |
           source ~/openquake/bin/activate
-          pip3 install pytest
-          oq --version
+          pip3 install pytest pyshp flake8
           oq dbserver start
           sleep 10
-          # test for engine
-          #pytest -vsx --color=yes /Users/runner/runner-isolation/actions-runner/_work/oq-engine/oq-engine/openquake/sep
-          #
-          #pytest -vsx --color=yes /Users/runner/runner-isolation/actions-runner/_work/oq-engine/oq-engine/openquake/hazardlib/tests/gsim/kotha_2020_test.py
-          #
-          #pytest -vsx --color=yes /Users/runner/runner-isolation/actions-runner/_work/oq-engine/oq-engine/openquake/calculators
-          # Run calcs
-          oq engine --run https://downloads.openquake.org/jobs/risk_test.zip
-          sleep 15
-          oq engine --run /Users/runner/runner-isolation/actions-runner/_work/oq-engine/oq-engine/demos/hazard/AreaSourceClassicalPSHA/job.ini
+
+      - name: Run demos to test installation
+        run: |
+          set -x
+          source ~/openquake/bin/activate
+          oq info venv
+          oq info cfg
+          cd /Users/runner/runner-isolation/actions-runner/_work/oq-engine/oq-engine/demos
+          ls -lrt
+          # run demos with job_hazard.ini and job_risk.ini
+          for demo_dir in $(find . -type d | sort); do
+             if [ -f $demo_dir/job_hazard.ini ]; then
+                 oq engine --run $demo_dir/job_hazard.ini --exports csv,hdf5
+                 oq engine --run $demo_dir/job_risk.ini --exports csv,hdf5 --hc -1
+             fi
+          done
+          # run the other demos
+          for ini in $(find . -name job.ini | sort); do
+              oq engine --run $ini --exports csv,hdf5
+          done
+
+      - name: Run tests for calculators and advanced manual to test installation
+        if: always()
+        run: |
+          source ~/openquake/bin/activate
+          cd /Users/runner/runner-isolation/actions-runner/_work/oq-engine/oq-engine/
+          pytest --doctest-modules --disable-warnings --color=yes --durations=10 openquake/calculators && pytest doc/adv-manual/*.rst
+
+      - name: Run tests for hazardlib, sep, commands, engine, hmtk, risklib, commonlib and baselib to test installation
+        if: always()
+        run: |
+          source ~/openquake/bin/activate
+          cd /Users/runner/runner-isolation/actions-runner/_work/oq-engine/oq-engine/openquake/
+          pytest --doctest-modules --disable-warnings --color=yes --durations=10 hazardlib sep commands engine hmtk risklib commonlib baselib
+
+      - name: Run tests for the engine server in public mode to test installation
+        if: always()
+        run: |
+          source ~/openquake/bin/activate
+          cd /Users/runner/runner-isolation/actions-runner/_work/oq-engine/oq-engine/
+          # -v 2 also logs the test names
+          OQ_APPLICATION_MODE=public ./openquake/server/manage.py test -v 2 tests.test_public_mode
+
+      - name: Run tests for the engine server in read-only mode to test installation
+        if: always()
+        run: |
+          source ~/openquake/bin/activate
+          cd /Users/runner/runner-isolation/actions-runner/_work/oq-engine/oq-engine/
+          # -v 2 also logs the test names
+          OQ_APPLICATION_MODE=read_only ./openquake/server/manage.py test -v 2 tests.test_read_only_mode
+
+      - name: Run tests for the engine server in aelo mode to test installation
+        if: always()
+        run: |
+          source ~/openquake/bin/activate
+          cd /Users/runner/runner-isolation/actions-runner/_work/oq-engine/oq-engine/
+          # -v 2 also logs the test names
+          OQ_APPLICATION_MODE=aelo OQ_CONFIG_FILE=openquake/server/tests/data/openquake.cfg ./openquake/server/manage.py test -v 2 tests.test_aelo_mode