CI/CD (#12)

* GA: adding actions * renaming workflow * GA: testing cibuildwheel * CI/CD\ * docs --------- Co-authored-by: Marius Isken <marius.isken@gfz-potsdam.de>
pyrocko · Mar 12, 2024 · bdb8bd2 · bdb8bd2
1 parent be42d95
commit bdb8bd2
Show file tree

Hide file tree

Showing 16 changed files with 187 additions and 64 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -1,26 +1,67 @@
-name: Build and test
+name: Deploy
 
-on: [push]
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "*"
 
 jobs:
-  build:
-    runs-on: ubuntu-latest
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ["3.10", "3.11"]
+        os: [ubuntu-20.04]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.16.5
+        with:
+          config-file: "{package}/pyproject.toml"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
+          path: ./wheelhouse/*.whl
 
+  make_sdist:
+    name: Make SDist
+    runs-on: ubuntu-latest
+    needs: build_wheels
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Optional, use if you use setuptools_scm
+          submodules: false # Optional, use if you have submodules
+
+      - name: Build SDist
+        run: pipx run build --sdist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-sdist
+          path: dist/*.tar.gz
+
+  pypi-publish:
+    name: Upload release to PyPI
+    if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes
+    needs: [build_wheels, make_sdist]
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/qseek
+    permissions:
+      id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+      - name: Download all the wheels and sdists
+        uses: actions/download-artifact@v3
         with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip" # caching pip dependencies
-          cache-dependency-path: "**/pyproject.toml"
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install .[dev]
-      - name: Test with pytest
-        run: |
-          pytest -m "not plot"
+          pattern: cibw-*
+          path: dist/
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
@@ -1,13 +1,14 @@
-name: docs
+name: Deploy Docs
 
 on:
   push:
-    branches:
-      - main
+    tags:
+      - "*"
+
 permissions:
   contents: write
 jobs:
-  deploy:
+  deploy-docs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -0,0 +1,59 @@
+name: Pre-commit and Tests
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+    tags:
+      - "*"
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+      - uses: pre-commit/action@v3.0.1
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip" # caching pip dependencies
+          cache-dependency-path: "**/pyproject.toml"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[dev]
+      - name: Test with pytest
+        run: |
+          pytest -m "not plot"
+
+  build_docs:
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          cache: "pip" # caching pip dependencies
+          cache-dependency-path: "**/pyproject.toml"
+      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
+      - uses: actions/cache@v3
+        with:
+          key: mkdocs-material-${{ env.cache_id }}
+          path: .cache
+          restore-keys: |
+            mkdocs-material-
+      - run: pip install .[docs]
+      - run: mkdocs build
diff --git a/README.md b/README.md
@@ -4,7 +4,7 @@
 
 [![Build and test](https://github.com/pyrocko/qseek/actions/workflows/build.yaml/badge.svg)](https://github.com/pyrocko/qseek/actions/workflows/build.yaml)
 [![Documentation](https://img.shields.io/badge/read-documentation-blue)](https://pyrocko.github.io/qseek/)
-![Python 3.10+](https://img.shields.io/badge/python-3.10%203.11-blue.svg)
+![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)
 <a href="https://github.com/psf/black"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg"></a>
 [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://pre-commit.com/)
 <!-- [![PyPI](https://img.shields.io/pypi/v/lassie)](https://pypi.org/project/lassie/) -->

diff --git a/docs/benchmark.md b/docs/benchmark.md
@@ -8,8 +8,8 @@ This enables high throughput of seismic data in different scenarios.
 
 | Number Stations | Throughput in MB | Throughput in Waveform data |
 | --------------- | ---------------- | ----------------------------|
-| 300+            | 40 MB/sec        | 12 hours/sec                |
-| 50              | 30 MB/sec        |  6 hours/sec                |
+| 300+            | 50 MB/sec        | 12 hours/sec                |
+| 50              | 200 MB/sec        |  6 hours/sec                |
 
 Scanning a 600 GB (~700 years of waveforms) data set costs **~2 days on a 64 cores machine equipped with an Nvidia A100 GPU**.
 

diff --git a/docs/components/configuration.md b/docs/components/configuration.md
@@ -6,6 +6,12 @@ Also see the [getting started guide](../getting_started.md) for a brief introduc
 
 [General configuration conventions](general.md) are outlines in this guide as well.
 
+!!! tip "Create a fresh config file."
+
+    ```sh title="Using the CLI"
+    qseek config
+    ```
+
 ## The Search
 
 The search configuration. **This is the entrypoint for the EQ detection and localisation**.

diff --git a/docs/components/station_corrections.md b/docs/components/station_corrections.md
@@ -2,13 +2,22 @@
 
 Station corrections can be extract from previous runs to refine the localisation accuracy. The corrections can also help to improve the semblance find more events in a dataset.
 
+## Station Specific Corrections
+
+![Source specific delay statistic](../images/station-delay-times.webp)
+*Statistics of station delay times.*
+
 ```python exec='on'
 from qseek.utils import generate_docs
 from qseek.insights import StationCorrections
 
 print(generate_docs(StationCorrections()))
 ```
 
+## Source Specific Corrections
+
+![Source specific corrections volume](../images/delay-volume.webp)
+*Delay volume for a selected stations.*
 
 ```python exec='on'
 from qseek.utils import generate_docs

diff --git a/docs/getting_started.md b/docs/getting_started.md
@@ -2,12 +2,18 @@
 
 ## Installation
 
-The installation is straight-forward:
+The installation is straight-forward using pip or pipx.
 
 ```sh title="From GitHub"
 pip install git+https://github.com/pyrocko/qseek
 ```
 
+or
+
+```sh title="Using pipx"
+pipx install git+https://github.com/pyrocko/qseek
+```
+
 ## Running Qseek
 
 The main entry point in the executeable is the `qseek` command. The provided command line interface (CLI) and a JSON config file is all what is needed to run the program.
@@ -26,8 +32,8 @@ qseek config > my-search.json
 
 Check out the `my-search.json` config file and add your waveform data and velocity models.
 
-??? abstract "Minimal Configuration Example"
-    Here is a minimal JSON configuration for Qseek
+??? quote "Minimal Configuration Example"
+    Here is a minimal JSON configuration for Qseek.
     ```bash exec='on' result='json'
     qseek config
     ```
@@ -36,8 +42,8 @@ For more details and information about the component, head over to [details of t
 
 ## Starting the Search
 
-Once happy, start the `qseek` CLI.
+Once happy with the configuration, start the `qseek` CLI.
 
-```sh title="Start earthquake detection"
+```sh title="Start the earthquake detection and localization"
 qseek search my-search.json
 ```
diff --git a/docs/index.md b/docs/index.md
@@ -1,4 +1,4 @@
-# Welcome to Qseek 🐕‍🦺
+# Welcome to Qseek 🔥
 
 Qseek is an earthquake detection and localisation framework. It combines modern **machine learning phase detection and robust migration and stacking techniques**.
 
@@ -7,7 +7,6 @@ The detector is leveraging [Pyrocko](https://pyrocko.org) and [SeisBench](https:
 !!! abstract "Citation"
     Marius Paul Isken, Peter Niemz, Jannes Münchmeyer, Sebastian Heimann, Simone Cesca, Torsten Dahm, Qseek: A data-driven Framework for Machine-Learning Earthquake Detection, Localization and Characterization, Seismica, 2024, *submitted*
 
-
 ![Reykjanes detections](images/reykjanes-demo.webp)
 
 *Seismic swarm activity at Iceland, Reykjanes Peninsula during a 2020 unrest. 15,000+ earthquakes detected, outlining a dike intrusion, preceeding the 2021 Fagradasfjall eruption. Visualized in [Pyrocko Sparrow](https://pyrocko.org).*
@@ -17,17 +16,16 @@ The detector is leveraging [Pyrocko](https://pyrocko.org) and [SeisBench](https:
 * [x] Earthquake phase detection using machine-learning pickers from [SeisBench](https://github.com/seisbench/seisbench)
 * [x] Octree localisation approach for efficient and accurate search
 * [x] Different velocity models:
-    * [x] Constant velocity
-    * [x] 1D Layered velocity model
-    * [x] 3D fast-marching velocity model (NonLinLoc compatible)
+  * [x] Constant velocity
+  * [x] 1D Layered velocity model
+  * [x] 3D fast-marching velocity model (NonLinLoc compatible)
 * [x] Extraction of earthquake event features:
-    * [x] Local magnitudes
-    * [x] Ground motion attributes
+  * [x] Local magnitudes
+  * [x] Ground motion attributes
 * [x] Automatic extraction of modelled and picked travel times
 * [x] Calculation and application of station corrections / station delay times
 * [ ] Real-time analytics on streaming data (e.g. SeedLink)
 
-
 [Get Started!](getting_started.md){ .md-button }
 
 ## Build with

diff --git a/mkdocs.yml b/mkdocs.yml
@@ -50,6 +50,9 @@ markdown_extensions:
       generic: true
   - pymdownx.tabbed:
       alternate_style: true
+  - pymdownx.emoji:
+      emoji_index: !!python/name:material.extensions.emoji.twemoji
+      emoji_generator: !!python/name:material.extensions.emoji.to_svg
   - admonition
   - def_list
   - attr_list
@@ -69,7 +72,7 @@ plugins:
   - markdown-exec
 
 nav:
-  - Earthquake Detector:
+  - The Earthquake Detector:
     - Welcome: index.md
     - Getting Started 🚀: getting_started.md
     - Visualising Detections: visualizing_results.md

diff --git a/pyproject.toml b/pyproject.toml
@@ -114,3 +114,6 @@ target-version = 'py311'
 
 [tool.pytest.ini_options]
 markers = ["plot: plot figures in tests"]
+
+[tool.cibuildwheel.macos]
+before-build = "brew install libomp"
diff --git a/src/qseek/models/semblance.py b/src/qseek/models/semblance.py
@@ -84,8 +84,6 @@ def _populate_table(self, table: Table) -> None:
 
 
 class SemblanceCache(dict[bytes, np.ndarray]):
-    _mask: dict[int, np.ndarray] = {}
-
     def get_mask(self, node_hashes: list[bytes]) -> np.ndarray:
         """Get the mask for the node hashes.
 
@@ -96,10 +94,7 @@ def get_mask(self, node_hashes: list[bytes]) -> np.ndarray:
             np.ndarray: The boolean mask for the node hashes.
         """
         # This is a "bit" of a hack to generate a hash from the node_hashes list
-        n_hashes = len(node_hashes)
-        if n_hashes not in self._mask:
-            self._mask[n_hashes] = np.array([hash in self for hash in node_hashes])
-        return self._mask[n_hashes]
+        return np.array([hash in self for hash in node_hashes])
 
 
 class Semblance:

diff --git a/src/qseek/octree.py b/src/qseek/octree.py
@@ -308,7 +308,7 @@ class Octree(BaseModel):
     )
     root_node_size: PositiveFloat = Field(
         default=2 * KM,
-        description="Initial size of the root octree node at level 0 in [m].",
+        description="Initial size of the root octree node at level 0 in meters.",
     )
     n_levels: int = Field(
         default=5,

diff --git a/src/qseek/search.py b/src/qseek/search.py
@@ -230,7 +230,7 @@ class Search(BaseModel):
     semblance_sampling_rate: SamplingRate = Field(
         default=100,
         description="Sampling rate for the semblance image function. "
-        "Choose from 10, 20, 25, 50, 100 Hz.",
+        "Choose from `10, 20, 25, 50, 100, 200` Hz.",
     )
     detection_threshold: PositiveFloat = Field(
         default=0.05,