IPDPS2021 Merge pull request #34 from devitocodes/timetiling_on_fwi

Timetiling on fwi
georgebisbas · Feb 19, 2021 · 8fdc21b · 8fdc21b
2 parents 588f257 + acd4f00
commit 8fdc21b
Show file tree

Hide file tree

Showing 205 changed files with 21,016 additions and 3,236 deletions.
diff --git a/.github/workflows/asv.yml b/.github/workflows/asv.yml
@@ -1,64 +1,28 @@
-# Runner informaiton:
-# Standard F8s_v2 (8 vcpus, 16 GiB memory)
-# Intel Xeon® Platinum 8168 (SkyLake)
+# Runner information:
+# CPU: Intel(R) Core(TM) i7-6700K CPU @ 4.00GHz
+# GPU: NVIDIA GeForce RTX 2060
 
 name: asv-benchmarks
 
 env:
   OUTPUT_PATH: ${{ github.workspace }}
-  RESOURCE_GROUP: devito-regression
-  VM_NAME: asv-runner-00
+  ENVHOME: "/home/devito/environments"
 
 on:
+  workflow_dispatch:
+    inputs:
+      tags:
+        description: 'Run ASV'
   # Trigger the workflow on push to the master branch
   push:
     branches:
       - master
 
 jobs:
-  # Boot the self-hosted runner and start runner app
-  start-runner:
-    name: start-runner
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: checkout repo
-      uses: actions/checkout@v2.3.2
-
-    - name: start VM
-      env:
-        SP_APPID: ${{ secrets.SERVICE_PRINCIPAL_APPID }}
-        SP_SECRET: ${{ secrets.SERVICE_PRINCIPAL_SECRET }}
-        TENANT_ID: ${{ secrets.SERVICE_PRINCIPAL_TENANTID }}
-        SUB_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-      run: >
-        pwsh -command "& '${{ env.OUTPUT_PATH }}\.github\azure\startVM.ps1'"
-        -servicePrincipal $SP_APPID
-        -servicePrincipalSecret $SP_SECRET
-        -servicePrincipalTenantId $TENANT_ID
-        -azureSubscriptionName $SUB_ID
-        -resourceGroupName $RESOURCE_GROUP
-        -vmName $VM_NAME
-
-    - name: set host
-      run: echo ::set-output name=action_host::$(az vm show -d -g $RESOURCE_GROUP -n $VM_NAME --query publicIps -o tsv)
-      id: host
-
-    - name: start actions runner app
-      uses: fifsky/ssh-action@master
-      with:
-        command: |
-          #!/bin/bash
-          nohup actions-runner/run.sh >/dev/null 2>&1 &
-        host: ${{ steps.host.outputs.action_host }}
-        user: ${{ secrets.CI_GPU_VM_ADMIN_LOGIN }}
-        pass: ${{ secrets.CI_GPU_VM_ADMIN_PASSWORD }}
-        args: "-tt"
 
   # Run the asv benchmarks on the self-hosted runner
   benchmarks:
     name: benchmarks
-    needs: start-runner
     runs-on: [self-hosted, asv]
 
     env:
@@ -74,6 +38,15 @@ jobs:
     - name: Checkout devito
       uses: actions/checkout@v2.3.2
 
+    - name: Set VIRTUAL_ENV
+      run: |
+        echo "VIRTUAL_ENV=$ENVHOME/asv" >> $GITHUB_ENV
+        echo "PATH=$VIRTUAL_ENV/bin:$PATH" >> $GITHUB_ENV
+
+    - name: Set PATH
+      run: |
+        echo "PATH=$VIRTUAL_ENV/bin:$PATH" >> $GITHUB_ENV
+
     - name: Install dependencies
       run: |
         pip install --upgrade pip
@@ -82,11 +55,11 @@ jobs:
 
     - name: Setup asv
       run: |
-        asv machine --config benchmarks/regression/asv.conf.json --machine F8s_v2 --os ubuntu-18.04 --arch x86-64 --cpu IntelXeon8168 --num_cpu 8 --ram 16GB
+        asv machine --config benchmarks/regression/asv.conf.json --machine i7-6700K --os ubuntu-20.10 --arch x86-64 --cpu i7-6700K --num_cpu 8 --ram 16GB
 
     - name: Run benchmarks
       run: |
-        asv run -v --strict --show-stderr --config benchmarks/regression/asv.conf.json --cpu-affinity 0-7 --machine F8s_v2
+        asv run -v --strict --show-stderr --config benchmarks/regression/asv.conf.json --cpu-affinity 0-7 --machine i7-6700K
 
     - name: Checkout asv-results branch
       uses: actions/checkout@v2.3.2
@@ -120,29 +93,3 @@ jobs:
         EXTERNAL_REPOSITORY: devitocodes/devito-performance
         PUBLISH_BRANCH: gh-pages
         PUBLISH_DIR: ./benchmarks/regression/.asv/html
-
-  # Stop the runner
-  stop-runner:
-    name: stop-runner
-    if: ${{ always() }}
-    needs: benchmarks
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: checkout repo
-      uses: actions/checkout@v2.3.2
-
-    - name: stop VM
-      env:
-        SP_APPID: ${{ secrets.SERVICE_PRINCIPAL_APPID }}
-        SP_SECRET: ${{ secrets.SERVICE_PRINCIPAL_SECRET }}
-        TENANT_ID: ${{ secrets.SERVICE_PRINCIPAL_TENANTID }}
-        SUB_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-      run: >
-        pwsh -command "& '${{ env.OUTPUT_PATH }}\.github\azure\stopVM.ps1'"
-        -servicePrincipal $SP_APPID
-        -servicePrincipalSecret $SP_SECRET
-        -servicePrincipalTenantId $TENANT_ID
-        -azureSubscriptionName $SUB_ID
-        -resourceGroupName $RESOURCE_GROUP
-        -vmName $VM_NAME
diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
@@ -8,59 +8,93 @@ on:
       - master         # Push events on master branch
 
 jobs:
-  deploy-docker:
+  deploy-docker-cpu:
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout devito
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
 
       - name: Check event name
         run: echo ${{ github.event_name }}
 
-      - name: CPU image
-        if: github.event_name == 'push'
-        uses: docker/build-push-action@v1
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-          repository: devitocodes/devito
-          dockerfile: ./docker/Dockerfile
-          tags: cpu-dev
+
+      - name: CPU image
+        if: github.event_name == 'push'
+        uses: docker/build-push-action@v2.2.1
+        with:
+          context: .
+          file: ./docker/Dockerfile
+          push: true
+          tags: devitocodes/devito:cpu-dev
 
       - name: CPU image release
         if: github.event_name == 'release'
-        uses: docker/build-push-action@v1
+        uses: docker/build-push-action@v2.2.1
+        with:
+          context: .
+          file: ./docker/Dockerfile
+          push: true
+          tags: |
+            devitocodes/devito:cpu-latest
+            devitocodes/devito:cpu-${{ github.event.release.tag_name }}
+
+  deploy-docker-gpu:
+    runs-on: [self-hosted, gpu, docker]
+
+    steps:
+      - name: Checkout devito
+        uses: actions/checkout@v2
+
+      - name: Check event name
+        run: echo ${{ github.event_name }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-          repository: devitocodes/devito
-          dockerfile: ./docker/Dockerfile
-          tags: cpu-latest,'cpu-${{ github.event.release.tag_name }}'
 
       - name: GPU image
         if: github.event_name == 'push'
-        uses: docker/build-push-action@v1
+        uses: docker/build-push-action@v2.2.1
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          repository: devitocodes/devito
-          dockerfile: ./docker/Dockerfile.nvidia
-          tags: gpu-dev
+          context: .
+          file: ./docker/Dockerfile.nvidia
+          push: true
+          tags: devitocodes/devito:gpu-dev
 
       - name: GPU image release
         if: github.event_name == 'release'
-        uses: docker/build-push-action@v1
+        uses: docker/build-push-action@v2.2.1
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          repository: devitocodes/devito
-          dockerfile: ./docker/Dockerfile.nvidia
-          tags: gpu-latest,'gpu-${{ github.event.release.tag_name }}'
+          context: .
+          file: ./docker/Dockerfile.nvidia
+          push: true
+          tags: |
+            devitocodes/devito:gpu-latest
+            devitocodes/devito:gpu-${{ github.event.release.tag_name }}
 
   # Comment out 'cpu-latest' until next release
   test-cpu-image:
-    needs: deploy-docker
+    needs: deploy-docker-cpu
     runs-on: ubuntu-latest
 
     strategy:
@@ -75,18 +109,23 @@ jobs:
              docker pull 'devitocodes/devito:${{ matrix.im-name }}'
              docker run --rm --name testrun 'devitocodes/devito:${{ matrix.im-name }}' pytest tests/test_operator.py
 
-  # Comment out until gpu docker image problem resolved
-  #test-gpu-image:
-    #needs: deploy-docker
-    #runs-on: [self-hosted, gpu]
+  test-gpu-image:
+    needs: deploy-docker-gpu
+    runs-on: [self-hosted, gpu, docker]
 
-    #strategy:
-      #fail-fast: false
-      #matrix:
+    strategy:
+      fail-fast: false
+      matrix:
         #im-name: [gpu-dev, gpu-latest]
+        im-name: [gpu-dev]
+
+    steps:
+        - name: Run simple test
+          run: |
+             docker pull 'devitocodes/devito:${{ matrix.im-name }}'
+             docker run --gpus all --rm --name testrun 'devitocodes/devito:${{ matrix.im-name }}' pytest tests/test_gpu_openacc.py
 
-    #steps:
-        #- name: Run simple test
-          #run: |
-             #docker pull 'devitocodes/devito:${{ matrix.im-name }}'
-             #docker run --rm --name testrun 'devitocodes/devito:${{ matrix.im-name }}' pytest tests/test_gpu_openacc.py
+        # NOTE: Suitable for the time being but will need to modify when we switch runners
+        - name: Clean
+          run: |
+            docker system prune -a -f
diff --git a/.github/workflows/examples-mpi.yml b/.github/workflows/examples-mpi.yml
@@ -1,9 +1,12 @@
-# Runner informaiton:
-# Standard F4s_v2 (4 vcpus, 8 GiB memory)
-# Intel Xeon® Platinum 8168 (SkyLake)
+# Runner information:
+# CPU: Intel(R) Core(TM) i7-6700K CPU @ 4.00GHz
 
 name: Examples-mpi
 
+env:
+  OUTPUT_PATH: ${{ github.workspace }}
+  ENVHOME: "/home/devito/environments"
+
 on:
   # Trigger the workflow on push or pull request,
   # but only for the master branch
@@ -17,25 +20,33 @@ on:
 jobs:
   build:
     name: Examples with mpi
-    runs-on: [self-hosted, mpi]
+    runs-on: ["self-hosted", "mpi", "examples"]
 
     env:
       DEVITO_MPI: "1"
       DEVITO_LANGUAGE: "openmp"
       OMP_NUM_THREADS: "2"
-      DEVITO_ARCH: "gcc-7"
+      DEVITO_ARCH: "gcc"
       DEVITO_BACKEND: "core"
-      CC: "gcc-7"
-      CXX: "g++-7"
+      CC: "gcc"
+      CXX: "g++"
 
     steps:
     - name: Checkout devito
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2.3.2
 
+      # We can use the same env as for asv here
+    - name: Set VIRTUAL_ENV
+      run: |
+        echo "VIRTUAL_ENV=$ENVHOME/asv" >> $GITHUB_ENV
+        echo "PATH=$VIRTUAL_ENV/bin:$PATH" >> $GITHUB_ENV
+    - name: Set PATH
+      run: |
+        echo "PATH=$VIRTUAL_ENV/bin:$PATH" >> $GITHUB_ENV
     - name: Install dependencies
       run: |
-        pip3 install --upgrade pip
-        pip3 install -e .[extras]
+        pip install --upgrade pip
+        pip install -e .
 
     - name: Test mpi notebooks
       run : |
@@ -46,10 +57,10 @@ jobs:
 
     - name: Test seismic examples
       run: |
-        mpirun -n 4 py.test examples/seismic/tti/tti_example.py
-        mpirun -n 4 py.test examples/seismic/elastic/elastic_example.py
-        mpirun -n 4 py.test examples/seismic/viscoacoustic/viscoacoustic_example.py
-        mpirun -n 4 py.test examples/seismic/viscoelastic/viscoelastic_example.py
+        mpirun -n 4 pytest examples/seismic/tti/tti_example.py
+        mpirun -n 4 pytest examples/seismic/elastic/elastic_example.py
+        mpirun -n 4 pytest examples/seismic/viscoacoustic/viscoacoustic_example.py
+        mpirun -n 4 pytest examples/seismic/viscoelastic/viscoelastic_example.py
 
     - name: Test fwi examples with mpi
       run: |

diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
@@ -27,10 +27,10 @@ jobs:
 
     steps:
     - name: Checkout devito
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
 
     - name: Setup conda
-      uses: goanpeca/setup-miniconda@v1
+      uses: conda-incubator/setup-miniconda@v2
       with:
        activate-environment: devito
        environment-file: environment-dev.yml