refactor: Refactor pyrorisks (#83)

* refactor: Deprecate the outdated pyro_risks modules, CLI, and examples * refactor: Remove Heroku deployment files * refactor: Remove DVC / CML pipelines * chore: Add poetry for packaging and dependency management * chore: Bump Geopandas to 1.0.1, replace fiona backend with pyogrio * chore: Add rasterio manylinux wheel * chore: Add ruff for linting and formatting * chore: Update docker build and config * chore: Update CI workflows * chore: Add Mock predictor * fix: Update mypy config * fix: Fix type hints and linting * style: Fix type hints, formatting, and linting * docs: Remove deprecated modules from docs
pyronear · Jul 12, 2024 · 1b07dc2 · 1b07dc2
1 parent fcc633d
commit 1b07dc2
Show file tree

Hide file tree

Showing 107 changed files with 2,770 additions and 6,170 deletions.
diff --git a/.coveragerc b/.coveragerc
diff --git a/.dvc/.gitignore b/.dvc/.gitignore
diff --git a/.dvc/config b/.dvc/config
diff --git a/.dvcignore b/.dvcignore
diff --git a/.flake8 b/.flake8
diff --git a/.github/workflows/web-server.yml → .github/workflows/builds.yml b/.github/workflows/web-server.yml → .github/workflows/builds.yml
@@ -1,4 +1,4 @@
-name: web-server
+name: builds
 
 on:
   push:
@@ -7,14 +7,14 @@ on:
     branches: [ master ]
 
 jobs:
-  docker-ready:
+  docker:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Build & run docker
+      - uses: actions/checkout@v4
+      - name: Build & run docker container
         env:
           CDS_UID: ${{ secrets.CDS_UID }}
           CDS_API_KEY: ${{ secrets.CDS_API_KEY }}
         run: PORT=8003 docker-compose up -d --build
-      - name: Ping app inside the container
+      - name: Ping containerized app
         run: sleep 5 && nc -vz localhost 8003
diff --git a/.github/workflows/doc-deploy.yaml b/.github/workflows/doc-deploy.yaml
@@ -1,39 +1,27 @@
 name: doc-deploy
 on:
   push:
-    branches: master
+    branches: [master]
 
 jobs:
-  docs-publish:
+  publish:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
         with:
           persist-credentials: false
-      - name: Set up Python 3.7
+      - name: Set up Python 3.10.5
         uses: actions/setup-python@v1
         with:
-          python-version: 3.7
-          architecture: x64
-      - name: Cache python modules
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ hashFiles('requirements.txt') }}-${{ hashFiles('**/*.py') }}
-          restore-keys: |
-            ${{ runner.os }}-pkg-deps-${{ hashFiles('requirements.txt') }}-
-            ${{ runner.os }}-pkg-deps-
-            ${{ runner.os }}-
+          python-version: 3.10.5
+
       - name: Install dependencies
-        run: |
-          sudo apt install libspatialindex-dev python3-rtree
-          python -m pip install --upgrade pip
-          pip install -e .
-          pip install -r docs/requirements.txt
+        run: poetry install
+
 
       - name: Build documentation
         run: |
-          sphinx-build docs/source docs/build -a -v
+          poetry run sphinx-build docs/source docs/build -a -v
 
       - name: Install SSH Client 🔑
         uses: webfactory/ssh-agent@v0.4.1

diff --git a/.github/workflows/gh-page.yaml → .github/workflows/doc-status.yaml b/.github/workflows/gh-page.yaml → .github/workflows/doc-status.yaml
@@ -6,11 +6,11 @@ jobs:
   see-page-build-payload:
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+      - name: Set up Python 3.10.5
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.7
-          architecture: x64
+          python-version: 3.10.5
+
       - name: check status
         run: |
           import os

diff --git a/.github/workflows/get_today_effis_fwi.yml b/.github/workflows/get_today_effis_fwi.yml
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
diff --git a/.github/workflows/requirements.txt b/.github/workflows/requirements.txt
diff --git a/.github/workflows/scripts.yml b/.github/workflows/scripts.yml
@@ -0,0 +1,37 @@
+name: Scheduled Scripts
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  get-effis-fwi:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python Version
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.10.5
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.8.1
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Install dependencies
+        run: poetry install
+
+      - name: Get Today Effis Fwi
+        run: poetry run python pyrorisks/platform_fwi/main.py
+        env:
+          AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          BUCKET_NAME: risk
+          REGION_NAME: gra
+          ENDPOINT_URL: https://s3.gra.io.cloud.ovh.net/
+
diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
@@ -0,0 +1,80 @@
+name: style
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  linting:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python Version
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.10.5
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.8.1
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Install dependencies
+        run: poetry install
+
+      - name: Run Ruff linter
+        run: |
+          poetry run ruff --version
+          poetry run ruff check --diff .
+
+  formating:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python Version
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.10.5
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.8.1
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Install dependencies
+        run: poetry install
+
+      - name: Run Ruff formatter
+        run: |
+          poetry run ruff --version
+          poetry run ruff format --diff .
+
+  typing:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python Version
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.10.5
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.8.1
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Install dependencies
+        run: poetry install
+
+      - name: Run mypy 
+        run: |
+          poetry run mypy --version
+          poetry run mypy app pyrorisks
diff --git a/.gitignore b/.gitignore
@@ -8,7 +8,7 @@ __pycache__/
 
 # Distribution / packaging
 .Python
-build/
+docs/build/
 develop-eggs/
 dist/
 downloads/

diff --git a/Aptfile b/Aptfile
diff --git a/Dockerfile b/Dockerfile
@@ -1,28 +1,19 @@
-FROM python:3.8.1
+FROM python:3.10-buster
 
-# set work directory
-WORKDIR /usr/src/app
+RUN pip install poetry==1.8.1
 
-# set environment variables
-ENV PYTHONDONTWRITEBYTECODE 1
-ENV PYTHONUNBUFFERED 1
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache \
+    VIRTUAL_ENV=/app/.venv \
+    PATH="/app/.venv/bin:$PATH"
 
-# copy app requirements
-COPY ./requirements.txt requirements.txt
-COPY ./requirements-app.txt /usr/src/app/requirements-app.txt
-COPY ./setup.py setup.py
-COPY ./README.md README.md
-COPY ./pyro_risks pyro_risks
+WORKDIR /app
 
-# install dependencies
-RUN apt-get update && \
-    apt-get install --no-install-recommends -y libspatialindex-dev python3-rtree && \
-    pip install --upgrade pip setuptools wheel && \
-    pip install -e . && \
-    pip install -r /usr/src/app/requirements-app.txt && \
-    mkdir /usr/src/app/app && \
-    rm -rf /root/.cache/pip && \
-    rm -rf /var/lib/apt/lists/*
+COPY pyrorisks ./pyrorisks
+COPY app ./app
+COPY build ./build
+COPY pyproject.toml poetry.lock README.md ./
 
-# copy project
-COPY app/ /usr/src/app/app/
+RUN poetry install
diff --git a/Procfile b/Procfile
diff --git a/pyro_risks/__init__.py → app/__init__.py b/pyro_risks/__init__.py → app/__init__.py
diff --git a/pyro_risks/utils/__init__.py → app/api/__init__.py b/pyro_risks/utils/__init__.py → app/api/__init__.py
diff --git a/app/api/inference.py b/app/api/inference.py
@@ -3,10 +3,14 @@
 # This program is licensed under the Apache License version 2.
 # See LICENSE or go to <https://www.apache.org/licenses/LICENSE-2.0.txt> for full license details.
 
-from pyro_risks.models.predict import PyroRisk
-
 
 __all__ = ["predictor"]
 
 
-predictor = PyroRisk(which="RF")
+class Mock:
+    def predict(self, date):
+        _ = date
+        return {"01": {"score": 0.5, "explainability": "weather"}, "02": {"score": 0.5, "explainability": "weather"}}
+
+
+predictor = Mock()
diff --git a/app/api/routes/__init__.py b/app/api/routes/__init__.py
diff --git a/app/api/routes/risk.py b/app/api/routes/risk.py
@@ -20,7 +20,4 @@
 async def get_pyrorisk(country: str, date: str):
     """Using the country identifier, this will compute the wildfire risk for all known subregions"""
     preds = predictor.predict(date)
-    return [
-        RegionRisk(geocode=k, score=v["score"], explainability=v["explainability"])
-        for k, v in preds.items()
-    ]
+    return [RegionRisk(geocode=k, score=v["score"], explainability=v["explainability"]) for k, v in preds.items()]