Merge pull request #62 from cryoem-uoft/release-v4.4

.github/workflows/python-package.yml

@@ -25,6 +25,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: true
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
@@ -54,12 +56,12 @@ jobs:
       fail-fast: true
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        # os: ["ubuntu-latest"]
-        python-version: ["3.7", "3.10"]
-        # python-version: ["3.7"]
+        python-version: ["3.7", "3.12"]
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: true
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
@@ -89,10 +91,18 @@ jobs:
         os: ["ubuntu-20.04", "macos-11", "windows-2019"]
     steps:
       - uses: actions/checkout@v3
-      - uses: pypa/cibuildwheel@v2.11.2
+        with:
+          submodules: true
+      - name: Set up QEMU
+        if: runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
+      - uses: pypa/cibuildwheel@v2.16.2
         env:
-          CIBW_SKIP: cp36-* pp*-win* pp*-macosx* *_i686 cp3{8,9,10,11}-macosx_x86_64
-          CIBW_ARCHS_MACOS: "x86_64 universal2"
+          CIBW_SKIP: cp36-* pp*-win* pp*-macosx* *_i686
+          CIBW_ARCHS_LINUX: "x86_64 aarch64"
+          CIBW_ARCHS_MACOS: "x86_64 arm64 universal2"
           CIBW_ARCHS_WINDOWS: "AMD64 ARM64"
         with:
           output-dir: dist

.gitignore

@@ -67,4 +67,4 @@ ipython_config.py
 .vercel
 cryosparc/core.c
 *.dSYM
-Python-*
+cython_debug

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "cryosparc/include/lz4"]
+	path = cryosparc/include/lz4
+	url = https://github.com/lz4/lz4.git

.pre-commit-config.yaml

@@ -0,0 +1,26 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-toml
+      - id: check-yaml
+      - id: check-added-large-files
+  - repo: https://github.com/psf/black
+    rev: 23.11.0
+    hooks:
+      - id: black
+      - id: black-jupyter
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.4
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+  - repo: https://github.com/RobertCraigie/pyright-python
+    rev: v1.1.335
+    hooks:
+      - id: pyright
+        additional_dependencies: [cython, httpretty, numpy, pytest]

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## v4.4.0
+
+- Added: Python 3.11 and 3.12 support
+- Added: `overwrite` keyword argument for file upload functions for extra confirmation when uploading project files to a path that already exist (raises error if not specified)
+- Added: Smaller and faster compressed dataset format when saving with `CSDAT_FORMAT` (datasets saved in this format prior to this release may no longer be readable)
+- Updated: Calls to CryoSPARC’s command servers automatically retry while CryoSPARC is down for up to one minute
+- Updated: Raise `cryosparc.errors.CommandError` instead of `AssertionError` when a CryoSPARC command server request fails
+- Updated: Raise `cryosparc.errors.DatasetLoadError` to show target file path when `Dataset.load` encounters an exception
+- Updated: Improved server- and slot- related error messages
+- Updated: Warn when connecting to CryoSPARC version that doesn’t match cryosparc-tools version
+- Docs: Delete Rejected Exposures example
+- Docs: Instructions for plotting scale bars on 2D Classes
+- Docs: Revert downsampled, symmetry expanded particles example
+- Docs: Connect a volume series to Class3D example
+
 ## v4.3.1
 
 - Fixed: `Job.queue` method `lane` argument is now optional when queuing directly to master

MANIFEST.in

@@ -1,4 +1,8 @@
 include cryosparc/core.pyx
 include cryosparc/dataset.c
 include cryosparc/dataset.pxd
+include cryosparc/lz4.pxd
 include cryosparc/include/cryosparc-tools/dataset.h
+include cryosparc/include/lz4/lib/lz4.h
+include cryosparc/include/lz4/lib/lz4.c
+include cryosparc/include/lz4/lib/LICENSE

README.md

@@ -19,27 +19,33 @@ Toolkit for interfacing with CryoSPARC. Read the documentation at
 
 1. Clone this repository
    ```sh
-   git clone https://github.com/cryoem-uoft/cryosparc-tools.git
+   git clone --recursive https://github.com/cryoem-uoft/cryosparc-tools.git
    cd cryosparc-tools
    git lfs pull
    ```
-2. Create and activate a conda environment named "tools" with the desired python version. See the Run Example Notebooks section to install an environment
+2. (Optional) Create and activate a virtual environment
    ```sh
-   conda create -n tools python=3.7 -c conda-forge
-   conda activate tools
+   python3 -m venv .venv
+   source .venv/bin/activate  # macOS / Linux
+   # OR
+   .venv\Scripts\activate.bat  # Windows
    ```
 3. Install dev dependencies and build native modules
    ```sh
    pip install -U pip wheel
    pip install -e ".[dev]"
    ```
+4. Install pre-commit hooks
+   ```
+   pre-commit install
+   ```
 
 ### Re-compile native module
 
 Recompile native modules after making changes to C code:
 
 ```sh
-make
+python -m setup build_ext -i
 ```
 
 ## Build Packages for Publishing
@@ -92,12 +98,12 @@ dependencies to execute, including the following system configuration:
 
 - Nvidia GPU and driver
 - [Miniconda](https://docs.conda.io/en/latest/miniconda.html) installed
-- CryoSPARC running at `localhost:40000` or `cryoem5:40000` (can alias `cryoem5` to localhost)
+- CryoSPARC running at `localhost:40000` or `cryoem0:40000` (can alias `cryoem0` to localhost)
 
 Clean previous build artefacts:
 
 ```sh
-make clean
+rm -rf cryosparc/*.so build dist *.egg-info
 ```
 
 Install dependencies into a new conda environment:

cryosparc/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "4.3.1"
+__version__ = "4.4.0"
 
 
 def get_include():

cryosparc/command.py

@@ -4,12 +4,20 @@
 """
 from contextlib import contextmanager
 import json
+import os
 import socket
+import time
 import uuid
 from typing import Optional, Type
 from urllib.request import urlopen, Request
 from urllib.error import HTTPError, URLError
 from urllib.parse import urlencode
+from warnings import warn
+
+from .errors import CommandError
+
+MAX_ATTEMPTS = int(os.getenv("CRYOSPARC_COMMAND_RETRIES", 3))
+RETRY_INTERVAL = int(os.getenv("CRYOSPARC_COMMAND_RETRY_SECONDS", 30))
 
 
 class CommandClient:
@@ -67,13 +75,9 @@ class CommandClient:
 
     """
 
+    Error = CommandError
     service: str
 
-    class Error(Exception):
-        def __init__(self, parent: "CommandClient", reason: str, *args: object, url: str = "") -> None:
-            msg = f"*** {type(parent).__name__}: ({url}) {reason}"
-            super().__init__(msg, *args)
-
     def __init__(
         self,
         service: str = "command",
@@ -94,24 +98,35 @@ def __init__(
     def _get_callable(self, key):
         def func(*args, **kwargs):
             params = kwargs if len(kwargs) else args
-            data = {
-                "jsonrpc": "2.0",
-                "method": key,
-                "params": params,
-                "id": str(uuid.uuid4()),
-            }
+            data = {"jsonrpc": "2.0", "method": key, "params": params, "id": str(uuid.uuid4())}
             res = None
             try:
-                with make_json_request(self, "/api", data=data) as request:
+                with make_json_request(self, "/api", data=data, _stacklevel=4) as request:
                     res = json.loads(request.read())
-            except CommandClient.Error as err:
-                raise CommandClient.Error(
-                    self, f'Did not receive a JSON response from method "{key}" with params {params}', url=self._url
+            except CommandError as err:
+                raise CommandError(
+                    f'Encounted error from JSONRPC function "{key}" with params {params}',
+                    url=self._url,
+                    code=err.code,
+                    data=err.data,
                 ) from err
 
-            assert res, f'JSON response not received for method "{key}" with params {params}'
-            assert "error" not in res, f'Error for "{key}" with params {params}:\n' + format_server_error(res["error"])
-            return res["result"]
+            if not res:
+                raise CommandError(
+                    f'JSON response not received from JSONRPC function "{key}" with params {params}',
+                    url=self._url,
+                )
+            elif "error" in res:
+                error = res["error"]
+                raise CommandError(
+                    f'Encountered {error.get("name", "Error")} from JSONRPC function "{key}" with params {params}:\n'
+                    f"{format_server_error(error)}",
+                    url=self._url,
+                    code=error.get("code"),
+                    data=error.get("data"),
+                )
+            else:
+                return res["result"]  # OK
 
         return func
 
@@ -127,7 +142,14 @@ def __call__(self):
 
 @contextmanager
 def make_request(
-    client: CommandClient, method: str = "POST", url: str = "", query: dict = {}, data=None, headers: dict = {}
+    client: CommandClient,
+    method: str = "POST",
+    url: str = "",
+    *,
+    query: dict = {},
+    data=None,
+    headers: dict = {},
+    _stacklevel=2,  # controls warning line number
 ):
     """
     Create a raw HTTP request/response context with the given command client.
@@ -141,7 +163,7 @@ def make_request(
         headers (dict, optional): HTTP headers. Defaults to {}.
 
     Raises:
-        CommandClient.Error: General error such as timeout, URL or HTTP
+        CommandError: General error such as timeout, URL or HTTP
 
     Yields:
         http.client.HTTPResponse:  Use with a context manager to get HTTP response
@@ -157,42 +179,53 @@ def make_request(
     url = f"{client._url}{url}{'?' + urlencode(query) if query else ''}"
     headers = {"Originator": "client", **client._headers, **headers}
     attempt = 1
-    max_attempts = 3
     error_reason = "<unknown>"
-    while attempt < max_attempts:
+    code = 500
+    resdata = None
+    while attempt < MAX_ATTEMPTS:
         request = Request(url, data=data, headers=headers, method=method)
         response = None
         try:
             with urlopen(request, timeout=client._timeout) as response:
                 yield response
                 return
-        except (TimeoutError, socket.timeout):
-            error_reason = "Timeout Error"
-            print(
-                f"*** {type(client).__name__}: command ({url}) "
-                f"did not reply within timeout of {client._timeout} seconds, "
-                f"attempt {attempt} of {max_attempts}"
-            )
-            attempt += 1
-        except HTTPError as error:
+        except HTTPError as error:  # command server reported an error
+            code = error.code
             error_reason = (
                 f"HTTP Error {error.code} {error.reason}; "
                 f"please check cryosparcm log {client.service} for additional information."
             )
             if error.readable():
-                error_reason += "\nResponse from server: " + str(error.read())
+                resdata = error.read()
+                error_reason += f"\nResponse from server: {data}"
+            if resdata and error.headers.get_content_type() == "application/json":
+                resdata = json.loads(resdata)
 
-            print(f"*** {type(client).__name__}: ({url}) {error_reason}")
+            warn(f"*** {type(client).__name__}: ({url}) {error_reason}", stacklevel=_stacklevel)
             break
-        except URLError as error:
+        except URLError as error:  # command server may be down
             error_reason = f"URL Error {error.reason}"
-            print(f"*** {type(client).__name__}: ({url}) {error_reason}")
-            break
+            warn(
+                f"*** {type(client).__name__}: ({url}) {error_reason}, attempt {attempt} of {MAX_ATTEMPTS}. "
+                f"Retrying in {RETRY_INTERVAL} seconds",
+                stacklevel=_stacklevel,
+            )
+            time.sleep(RETRY_INTERVAL)
+            attempt += 1
+        except (TimeoutError, socket.timeout):  # slow network connection or request
+            error_reason = "Timeout Error"
+            warn(
+                f"*** {type(client).__name__}: command ({url}) "
+                f"did not reply within timeout of {client._timeout} seconds, "
+                f"attempt {attempt} of {MAX_ATTEMPTS}",
+                stacklevel=_stacklevel,
+            )
+            attempt += 1
 
-    raise CommandClient.Error(client, error_reason, url=url)
+    raise CommandError(error_reason, url=url, code=code, data=resdata)
 
 
-def make_json_request(client: CommandClient, url="", query={}, data=None, headers={}):
+def make_json_request(client: CommandClient, url="", *, query={}, data=None, headers={}, _stacklevel=3):
     """
     Similar to ``make_request``, except sends request body data JSON and
     receives arbitrary response.
@@ -208,6 +241,9 @@ def make_json_request(client: CommandClient, url="", query={}, data=None, header
     Yields:
         http.client.HTTPResponse: Use with a context manager to get HTTP response
 
+    Raises:
+        CommandError: General error such as timeout, URL or HTTP
+
     Example:
 
         >>> from cryosparc.command import CommandClient, make_json_request
@@ -218,10 +254,13 @@ def make_json_request(client: CommandClient, url="", query={}, data=None, header
     """
     headers = {"Content-Type": "application/json", **headers}
     data = json.dumps(data, cls=client._cls).encode()
-    return make_request(client, url=url, query=query, data=data, headers=headers)
+    return make_request(client, url=url, query=query, data=data, headers=headers, _stacklevel=_stacklevel)
 
 
 def format_server_error(error):
+    """
+    :meta private:
+    """
     err = error["message"] if "message" in error else str(error)
     if "data" in error and error["data"]:
         if isinstance(error["data"], dict) and "traceback" in error["data"]: