Update packaging and Python versions

.github/workflows/main.yaml

@@ -37,19 +37,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: ['3.8', '3.9', '3.10']
+        python: ['3.9', '3.10', '3.11']
         CARBONPLAN_DATA: [
             # 'https://carbonplan.blob.core.windows.net/carbonplan-data',
             'gs://carbonplan-data',
           ]
     steps:
       - uses: actions/checkout@v3
-      - name: Install Conda environment
-        uses: mamba-org/provision-with-micromamba@main
+      - name: Install Conda environment from environment.yml
+        uses: mamba-org/setup-micromamba@v1
         with:
+          # environment-file is not assumed anymore
           environment-file: ci/environment.yaml
-          cache-downloads: true
-          extra-specs: |
+          create-args: >-
             python=${{ matrix.python-version }}
       - name: Install package
         run: |

.gitignore

@@ -29,6 +29,7 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+carbonplan_data/_version.py
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

.pre-commit-config.yaml

@@ -14,36 +14,38 @@ repos:
       - id: mixed-line-ending
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.10.1
+    rev: v3.8.0
     hooks:
       - id: pyupgrade
         args:
           - '--py38-plus'
 
   - repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 23.3.0
     hooks:
       - id: black
       - id: black-jupyter
 
-  - repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
+  - repo: https://github.com/keewis/blackdoc
+    rev: v0.3.8
     hooks:
-      - id: flake8
+      - id: blackdoc
 
-  - repo: https://github.com/asottile/seed-isort-config
-    rev: v2.2.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: 'v0.0.276'
     hooks:
-      - id: seed-isort-config
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+      - id: ruff
+        args: ['--fix']
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.0.0-alpha.9-for-vscode
     hooks:
-      - id: isort
+      - id: prettier
 
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+  - repo: https://github.com/kynan/nbstripout
+    rev: 0.6.1
     hooks:
-      - id: isort
+      - id: nbstripout
 
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: 'v3.0.3'

README.md

@@ -15,7 +15,6 @@
 ![PyPI](https://img.shields.io/pypi/v/carbonplan-data)
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
 
-
 This repository includes our main data catalog as well as our pre-processing utilities.
 
 ## install

carbonplan_data/tests/__init__.py

@@ -78,6 +78,7 @@ def parametrize_with_checks(catalog):
     >>> @parametrize_with_checks(cat)
     ... def test_catalog(entry, check):
     ...     check(entry)
+    ...
 
     """
     import pytest

carbonplan_data/tests/test_catalogs.py

@@ -1,6 +1,6 @@
 import os
 
-from carbonplan.data import MASTER_CATALOG_PATH, cat
+from carbonplan_data import MASTER_CATALOG_PATH, cat
 
 from . import parametrize_with_checks
 

carbonplan_data/utils.py

@@ -6,7 +6,7 @@
 import zipfile
 
 import numpy as np
-import urlpath
+import validators
 import wget
 import xarray as xr
 import yaml
@@ -155,7 +155,8 @@ def process_sources(name, workdir=None):
         # download
         if "download" in dset["actions"]:
             for url in dset["urlpath"]:
-                url = urlpath.URL(url)
+                if not validators.url(url):
+                    raise ValueError(f'url "{url}" not valid')
                 out = workdir / url.name
                 if not out.exists():
                     print(f"downloading {url}")

ci/environment.yaml

@@ -8,7 +8,7 @@ dependencies:
   - gcsfs
   - gdal
   - google-cloud-storage
-  - intake
+  - intake<=0.7.0
   - intake-geopandas
   - intake-parquet
   - intake-xarray
@@ -20,9 +20,7 @@ dependencies:
   - pydap
   - pytest
   - pytest-cov
+  - pre-commit
   - rasterio
-  - urlpath
   - xarray
   - zarr
-  - pip:
-      - carbonplan

pyproject.toml

@@ -1,7 +1,107 @@
+[build-system]
+requires = ["setuptools>=64", "setuptools-scm[toml]>=6.2", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "carbonplan-data"
+description = "Preprocessing utilities for CarbonPlan's data catalog"
+readme = "README.md"
+license = { text = "MIT" }
+authors = [{ name = "CarbonPlan", email = "tech@carbonplan.org" }]
+requires-python = ">=3.9"
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Topic :: Scientific/Engineering",
+]
+dynamic = ["version"]
+
+dependencies = [
+    "intake<=0.7.0",
+    "validators",
+    "wget",
+]
+
+[project.urls]
+repository = "https://github.com/carbonplan/data"
+
+[tool.setuptools.packages.find]
+include = ["carbonplan_data*"]
+
+[tool.setuptools_scm]
+version_scheme = "post-release"
+local_scheme = "node-and-date"
+fallback_version = "999"
+write_to = "carbonplan_data/_version.py"
+write_to_template = '__version__ = "{version}"'
+
+
+# [tool.setuptools.dynamic]
+# version = { attr = "carbonplan_data.__version__" }
+
+
 [tool.black]
 line-length = 100
-target-version = ['py38']
+target-version = ['py39']
 skip-string-normalization = true
 
-[build-system]
-requires = ["setuptools>=30.3.0", "wheel", "setuptools_scm"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py39"
+builtins = ["ellipsis"]
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+]
+per-file-ignores = {}
+# E402: module level import not at top of file
+# E501: line too long - let black worry about that
+# E731: do not assign a lambda expression, use a def
+ignore = ["E402", "E501", "E731"]
+select = [
+    # Pyflakes
+    "F",
+    # Pycodestyle
+    "E",
+    "W",
+    # isort
+    "I",
+    # Pyupgrade
+    "UP",
+]
+
+
+[tool.ruff.mccabe]
+max-complexity = 18
+
+[tool.ruff.isort]
+known-first-party = ["carbonplan_data"]
+
+[tool.pytest.ini_options]
+console_output_style = "count"
+addopts = "--cov=./ --cov-report=xml --verbose"

scripts/fluxnet/01_raw_to_parquet.ipynb

@@ -70,7 +70,6 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": true,
     "jupyter": {
      "outputs_hidden": true
     }

scripts/glas/01_cache_glas_data.ipynb

@@ -4,7 +4,6 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": true,
     "jupyter": {
      "outputs_hidden": true
     }

scripts/gridmet/01_gridmet_to_zarr.ipynb

@@ -84,7 +84,6 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": true,
     "jupyter": {
      "outputs_hidden": true
     }

scripts/mtbs/06_annual_downsampling.py

@@ -19,7 +19,7 @@
         print(source)
         crs, extent = projections("albers", region)
         resampling = "nearest"
-        cmd = ("gdalwarp " "-t_srs '%s' " "-te %s " "-tr %s %s " "-r %s " "%s " "%s") % (
+        cmd = ("gdalwarp " "-t_srs '{}' " "-te {} " "-tr {} {} " "-r {} " "{} " "{}").format(
             crs,
             extent,
             resolution,