Merge pull request #19 from cryoem-uoft/release-v4.1

Release v4.1.3

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## v4.1.3
+
+- Fix error when creating a job with `CryoSPARC.create_job`
+- Fix Dataset allocation error due to slot generation overflow
+- Docs minor fixes
+
 ## v4.1.2
 
 - Preliminary access to CryoSPARC Live via CryoSPARC.rtp command client

CONTRIBUTING.md

@@ -16,7 +16,7 @@ Thanks for taking the time to contribute! See the [Table of Contents](#table-of-
 - [Commit Messages](#commit-messages)
 - [Writing tools, libraries and add-ons](#writing-tools-libraries-and-add-ons)
 
-## I Have a generally Question
+## I Have a General Question
 
 > If you want to ask a question, we assume that you have read the available [Documentation](https://tools.cryosparc.com).
 
@@ -30,7 +30,7 @@ If you still have a question and need clarification, we recommend the following:
 
 Structura will address the issue as soon as possible.
 
-## I Want To Contribute
+## I Want to Contribute
 
 > ### Legal Notice <!-- omit in toc -->
 >

README.md

@@ -108,7 +108,7 @@ conda create -n cryosparc-tools-example -c conda-forge \
     pyqt=5 libtiff wxPython=4.1.1 adwaita-icon-theme
 conda activate cryosparc-tools-example
 pip install -U pip
-pip install nvidia-pyindex matplotlib">=3.4,<3.5" pandas==1.1.4 notebook
+pip install nvidia-pyindex matplotlib~=3.4.0 pandas==1.1.4 notebook
 pip install "cryolo[c11]"
 pip install -e ".[build]"
 ```
@@ -133,4 +133,4 @@ Find examples in `docs/examples` directory
 
 ## License
 
-cryosparc-tools is licensed under the BSD-3-Clause.
+cryosparc-tools is licensed under the BSD-3-Clause license.

cryosparc/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "4.1.2"
+__version__ = "4.1.3"
 
 
 def get_include():

cryosparc/include/cryosparc-tools/dataset.h

@@ -362,8 +362,9 @@ moreslots (void) {
 	}
 }
 
-#define SHIFT_GEN (64-15)
-#define MASK_IDX  (0xffffffffffffffff >> 15)
+#define SHIFT_GEN (64-16)
+#define MASK_IDX  (0xffffffffffffffff >> 16)
+#define MAX_GEN   UINT16_MAX
 
 static inline uint64_t
 roundup(uint64_t value, uint64_t to)
@@ -405,6 +406,10 @@ dset_new_(size_t newsize, ds **allocation)
 	unlock();
 
 	memset(s->memory, 0, newsize);
+	if (s->generation >= MAX_GEN) {
+		// Generation limit reached, trigger overflow so that handle != 0
+		s->generation = 0;
+	}
 	gen = ++s->generation;
 
 	return i | (gen << SHIFT_GEN);

cryosparc/job.py

@@ -815,8 +815,7 @@ def mkdir(
     def cp(self, source_path_rel: Union[str, PurePosixPath], target_path_rel: Union[str, PurePosixPath]):
         """
         Copy a file or folder within a project to another location within that
-        same project. Note that argument order is reversed from
-        equivalent ``cp`` command.
+        same project.
 
         Args:
             source_path_rel (str | Path): Relative path in project of source
@@ -832,8 +831,7 @@ def cp(self, source_path_rel: Union[str, PurePosixPath], target_path_rel: Union[
     def symlink(self, source_path_rel: Union[str, PurePosixPath], target_path_rel: Union[str, PurePosixPath]):
         """
         Create a symbolic link in the given project. May only create links for
-        files within the project. Note that argument order is reversed from
-        ``ln -s``.
+        files within the project.
 
         Args:
             project_uid (str): Target project UID, e.g., "P3".

cryosparc/project.py

@@ -417,8 +417,7 @@ def mkdir(
     def cp(self, source_path_rel: Union[str, PurePosixPath], target_path_rel: Union[str, PurePosixPath]):
         """
         Copy a file or folder within a project to another location within that
-        same project. Note that argument order is reversed from
-        equivalent ``cp`` command.
+        same project.
 
         Args:
             source_path_rel (str | Path): Relative path in project of source
@@ -434,15 +433,14 @@ def cp(self, source_path_rel: Union[str, PurePosixPath], target_path_rel: Union[
     def symlink(self, source_path_rel: Union[str, PurePosixPath], target_path_rel: Union[str, PurePosixPath]):
         """
         Create a symbolic link in the given project. May only create links for
-        files within the project. Note that argument order is reversed from
-        ``ln -s``.
+        files within the project.
 
         Args:
             project_uid (str): Target project UID, e.g., "P3".
-            target_path_rel (str | Path): Relative path in project to new
-                symlink.
             source_path_rel (str | Path): Relative path in project to file from
                 which to create symlink.
+            target_path_rel (str | Path): Relative path in project to new
+                symlink.
         """
         self.cs.symlink(
             project_uid=self.uid,

cryosparc/tools.py

@@ -354,11 +354,16 @@ def create_job(
             #cryosparc.tools.CryoSPARC.get_job_sections
         """
         job_uid: str = self.cli.create_new_job(  # type: ignore
-            job_type=type, project_uid=project_uid, workspace_uid=workspace_uid, title=title, desc=desc
+            job_type=type,
+            project_uid=project_uid,
+            workspace_uid=workspace_uid,
+            created_by_user_id=self.user_id,
+            title=title,
+            desc=desc,
         )
         job = self.find_job(project_uid, job_uid)
         for input_name, (parent_job, output_name) in connections.items():
-            job.connect(parent_job, output_name, input_name, refresh=False)
+            job.connect(input_name, parent_job, output_name, refresh=False)
         for k, v in params.items():
             job.set_param(k, v, refresh=False)
         if connections or params:
@@ -773,8 +778,7 @@ def cp(
     ):
         """
         Copy a file or folder within a project to another location within that
-        same project. Note that argument order is reversed from
-        equivalent ``cp`` command.
+        same project.
 
         Args:
             project_uid (str): Target project UID, e.g., "P3".
@@ -793,8 +797,7 @@ def symlink(
     ):
         """
         Create a symbolic link in the given project. May only create links for
-        files within the project. Note that argument order is reversed from
-        ``ln -s``.
+        files within the project.
 
         Args:
             project_uid (str): Target project UID, e.g., "P3".

docs/examples/recenter-particles.ipynb

@@ -410,7 +410,7 @@
     "ax1.imshow(lowpassed, cmap=\"gray\", vmin=vmin, vmax=vmax, origin=\"lower\")\n",
     "ax2.imshow(lowpassed, cmap=\"gray\", vmin=vmin, vmax=vmax, origin=\"lower\")\n",
     "\n",
-    "# Plot old particles in magenta\n",
+    "# Plot old particles in yellow\n",
     "mic_old_particles = particles.query({\"location/micrograph_path\": mic_path})\n",
     "old_location_x = mic_old_particles[\"location/center_x_frac\"] * width\n",
     "old_location_y = mic_old_particles[\"location/center_y_frac\"] * height\n",

docs/intro.md

@@ -48,10 +48,10 @@ release version. i.e., if the CryoSPARC version is vX.Y.Z, use the latest vX.Y
 tools package. The Z component does not need to match.
 
 For example, if you are running CryoSPARC v4.1.2, install cryosparc-tools with
-`pip install "cryosparc-tools>=4.1,<4.2"`. If you later update to CryoSPARC
-v4.2.0 or v5.0.0, re-install the corresponding tools package with
-`pip install cryosparc-tools>=4.2,<4.3` or
-`pip install cryosparc-tools>=5.0,<5.1` respectively.
+`pip install cryosparc-tools~=4.1.0` (equivalent to `pip install "cryosparc-tools>=4.1.0,<4.2").
+If you later update to CryoSPARC v4.2.0 or v5.0.0, re-install the corresponding
+tools package with `pip install cryosparc-tools~=4.2.0` or
+`pip install cryosparc-tools~=5.0.0` respectively.
 ```
 
 ## Usage
@@ -118,7 +118,7 @@ environment with the required dependencies. Here the environment is named
 ```sh
 conda create -n cryosparc-tools-example -c conda-forge python=3 numpy==1.18.5
 conda activate cryosparc-tools-example
-pip install matplotlib">=3.4,<3.5" pandas==1.1.4 cryosparc-tools
+pip install matplotlib~=3.4.0 pandas==1.1.4 cryosparc-tools
 ```
 
 For speed, these do not include the dependencies for the crYOLO example
@@ -158,5 +158,5 @@ If you publish an open-source tool that uses this package to GitHub, add the
 
 ## License
 
-cryosparc-tools is licensed under the BSD-3-Clause.
+cryosparc-tools is licensed under the BSD-3-Clause license.
 [View full license text](https://github.com/cryoem-uoft/cryosparc-tools/blob/main/LICENSE).

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cryosparc-tools"
-version = "4.1.2"
+version = "4.1.3"
 description = "Toolkit for interfacing with CryoSPARC"
 readme = "README.md"
 requires-python = ">=3.7"

setup.py

@@ -23,7 +23,7 @@
 
 setup(
     name="cryosparc_tools",
-    version="4.1.2",
+    version="4.1.3",
     description="Toolkit for interfacing with CryoSPARC",
     headers=["cryosparc/include/cryosparc-tools/dataset.h"],
     ext_modules=cythonize(

tests/conftest.py

@@ -13,6 +13,7 @@
 from cryosparc.dataset import Dataset as BaseDataset, Row
 from cryosparc.util import default_rng
 
+
 # Always use this class for testing to ensure Dataset#items property is never
 # used internally. Downstream CryoSPARC relies on this.
 class Dataset(BaseDataset[Row]):
@@ -31,10 +32,20 @@ def request_callback_core(request, uri, response_headers):
     procs = {
         "hello_world": {"hello": "world"},
         "get_id_by_email_password": "6372a35e821ed2b71d9fe4e3",
-        "get_job": {"uid": "J1", "project_uid": "P1", "job_type": "homo_abinit"},
+        "get_job": {
+            "uid": "J1",
+            "project_uid": "P1",
+            "job_type": "homo_abinit",
+            "title": "New Job",
+            "description": "",
+            "created_by_user_id": "6372a35e821ed2b71d9fe4e3",
+        },
         "get_project_dir_abs": "/projects/my-project",
         "get_project": {"uid": "P1", "title": "My Project"},
+        "create_new_job": "J1",
         "job_send_streamlog": None,
+        "job_connect_group": True,
+        "job_set_param": True,
     }
     procs["system.describe"] = {"procs": [{"name": m} for m in procs]}
     response_headers["content-type"] = "application/json"

tests/test_dataset.py

@@ -267,11 +267,20 @@ def test_union_many_empty():
     assert len(Dataset.union_many().rows()) == 0
 
 
-def test_allocate_many():
+def test_allocate_many_separate():
+    for _ in range(66_000):
+        allocated = []
+        for _ in range(3):
+            allocated.append(Dataset(1))
+        assert len(allocated) == 3
+        del allocated
+
+
+def test_allocate_many_together():
     # Checks for logic issues when allocating a lot of datasets
     for _ in range(3):
         allocated = []
-        for _ in range(33_000):
+        for _ in range(66_000):
             allocated.append(Dataset(1))
-        assert len(allocated) == 33_000
+        assert len(allocated) == 66_000
         del allocated

tests/test_tools.py

@@ -0,0 +1,71 @@
+import httpretty
+from cryosparc.tools import CryoSPARC
+from cryosparc.project import Project
+from cryosparc.job import Job
+
+
+def test_create_job_basic(cs: CryoSPARC, project: Project):
+    job = cs.create_job(project.uid, "W1", "homo_abinit")
+    assert isinstance(job, Job)
+    assert job.uid == "J1"
+
+    latest_requests = httpretty.latest_requests()
+    create_job_request = latest_requests[-3]
+    get_job_request = latest_requests[-1]
+    assert create_job_request.parsed_body["method"] == "create_new_job"
+    assert create_job_request.parsed_body["params"] == {
+        "job_type": "homo_abinit",
+        "project_uid": project.uid,
+        "workspace_uid": "W1",
+        "created_by_user_id": cs.user_id,
+        "title": None,
+        "desc": None,
+    }
+    assert get_job_request.parsed_body["method"] == "get_job"
+    assert get_job_request.parsed_body["params"] == ["P1", "J1"]
+
+
+def test_create_job_connect_params(cs: CryoSPARC, project: Project):
+    job = cs.create_job(
+        project.uid,
+        "W1",
+        "homo_abinit",
+        connections={"particles": ("J2", "particles_selected")},
+        params={"abinit_K": 3},
+    )
+    assert isinstance(job, Job)
+    assert job.uid == "J1"
+
+    latest_requests = httpretty.latest_requests()
+    create_job_request = latest_requests[-9]
+    get_job_request = latest_requests[-7]
+    connect_request = latest_requests[-5]
+    set_param_request = latest_requests[-3]
+    refresh_request = latest_requests[-1]
+
+    assert create_job_request.parsed_body["method"] == "create_new_job"
+    assert create_job_request.parsed_body["params"] == {
+        "job_type": "homo_abinit",
+        "project_uid": project.uid,
+        "workspace_uid": "W1",
+        "created_by_user_id": cs.user_id,
+        "title": None,
+        "desc": None,
+    }
+    assert get_job_request.parsed_body["method"] == "get_job"
+    assert get_job_request.parsed_body["params"] == ["P1", "J1"]
+    assert connect_request.parsed_body["method"] == "job_connect_group"
+    assert connect_request.parsed_body["params"] == {
+        "project_uid": "P1",
+        "source_group": "J2.particles_selected",
+        "dest_group": "J1.particles",
+    }
+    assert set_param_request.parsed_body["method"] == "job_set_param"
+    assert set_param_request.parsed_body["params"] == {
+        "project_uid": "P1",
+        "job_uid": "J1",
+        "param_name": "abinit_K",
+        "param_new_value": 3,
+    }
+    assert refresh_request.parsed_body["method"] == "get_job"
+    assert refresh_request.parsed_body["params"] == ["P1", "J1"]