diff --git a/.github/workflows/run_all_frameworks.yml b/.github/workflows/run_all_frameworks.yml
index 4f25e493e..69b48c574 100644
--- a/.github/workflows/run_all_frameworks.yml
+++ b/.github/workflows/run_all_frameworks.yml
@@ -33,7 +33,7 @@ jobs:
           ".github/runbenchmark/action.yml "\
           "runbenchmark.py "\
           "requirements.txt"
-          
+
           echo Common files: $common_files
           echo Changed files: $changed_files
 
@@ -100,7 +100,7 @@ jobs:
       - uses: actions/cache@v3
         id: cache
         with:
-          path: /home/runner/work/automlbenchmark/automlbenchmark/venv/lib/python3.9/site-packages
+          path: /home/runner/work/automlbenchmark/automlbenchmark/venv
           key: pip-v3-${{ hashFiles('**/requirements.txt') }}
           restore-keys: |
             pip-v3-
@@ -109,12 +109,18 @@ jobs:
         run: |
           source venv/bin/activate
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
-          pip show openml
+          python -m pip install -r requirements.txt
+          python -m pip install "coverage[toml]"
+          python -m pip show openml
       - name: Run constantpredictor on openml iris
         run: |
           source venv/bin/activate
-          python runbenchmark.py ${{ matrix.framework }} -t ${{ matrix.task }} -f 0 -e
+          coverage run -m runbenchmark ${{ matrix.framework }} -t ${{ matrix.task }} -f 0 -e
+          coverage xml
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   run_frameworks:
     name: ${{ matrix.framework }}/${{ matrix.task }}
@@ -143,7 +149,7 @@ jobs:
     - uses: actions/cache@v3
       id: cache
       with:
-        path: /home/runner/work/automlbenchmark/automlbenchmark/venv/lib/python3.9/site-packages
+        path: /home/runner/work/automlbenchmark/automlbenchmark/venv
         key: pip-v3-${{ hashFiles('**/requirements.txt') }}
         restore-keys: |
           pip-v3-
@@ -153,9 +159,15 @@ jobs:
         source venv/bin/activate
         python -m pip install --upgrade pip
         python -m pip install -r requirements.txt
+        python -m pip install "coverage[toml]"
     - name: Run ${{ matrix.framework }} on ${{ matrix.task }}
       run: |
         source venv/bin/activate
-        python runbenchmark.py ${{ matrix.framework }} ${{ matrix.benchmark }} test -f 0 -t ${{ matrix.task }} -e
+        coverage run -m runbenchmark ${{ matrix.framework }} ${{ matrix.benchmark }} test -f 0 -t ${{ matrix.task }} -e
+        coverage xml
       env:
         GITHUB_PAT: ${{ secrets.PUBLIC_ACCESS_GITHUB_PAT }}
+    - name: Upload coverage reports to Codecov
+      uses: codecov/codecov-action@v4
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
diff --git a/.github/workflows/run_pytest.yaml b/.github/workflows/run_pytest.yaml
index 52be3c836..d128d5167 100644
--- a/.github/workflows/run_pytest.yaml
+++ b/.github/workflows/run_pytest.yaml
@@ -18,7 +18,7 @@ jobs:
       - uses: actions/cache@v3
         id: cache
         with:
-          path: /home/runner/work/automlbenchmark/automlbenchmark/venv/lib/python3.9/site-packages
+          path: /home/runner/work/automlbenchmark/automlbenchmark/venv
           key: pip-v3-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/requirements-dev.txt') }}
           restore-keys: |
             pip-v3-
@@ -32,4 +32,9 @@ jobs:
       - name: Run unit tests
         run: |
           source venv/bin/activate
-          python -m pytest -vv -m "not stress and not openml_upload"
+          coverage run -m pytest -vv -m "not stress and not openml_upload"
+          coverage xml
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
diff --git a/pyproject.toml b/pyproject.toml
index ff6479037..9e54eb931 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -9,6 +9,13 @@ disable_error_code = "attr-defined"
 #  problem with ruamel: https://github.com/python/mypy/issues/12664
 incremental = false
 
+[tool.coverage.run]
+include=[
+    "tests/**",
+    "amlb/**",
+    "frameworks/shared/**"
+]
+
 # 3rd party libraries without type hints or stubs:
 [[tool.mypy.overrides]]
 module=[
@@ -23,4 +30,4 @@ module = "amlb.benchmark"
 disable_error_code = "union-attr"
 # special run-mode script makes some attributes optional,
 # TODO: refactor so that recovery script does not need special runtime or that it doesnt
-# abuse the benchmark class.
\ No newline at end of file
+# abuse the benchmark class.
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 725e46e03..f66954d0c 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,6 +1,7 @@
 pytest
 pytest-mock
 pip-tools
+coverage[toml]
 
 types-psutil
 types-xmltodict