Merge primary_image and auxiliary_images. (#701)

1. There is a top-level field that is `images`, and that includes both primary images and auxiliary images. Auxiliary images are addressed as <fov_object>[<image_name>]. The primary image is a special case, where the image name = "primary". 2. Updated the sptx-format schema and examples. 3. Updated the notebooks. 4. Of note is that we are _not_ deprecating 4.0.0. The format is close enough I added a read path for it! I did not convert the iss_breast full experiment data set to the new format. Fixes #625 Depends on #698 Test plan: make -j tests run_notebooks
spacetx · Oct 17, 2018 · 16ecf10 · 16ecf10
1 parent d933c08
commit 16ecf10
Show file tree

Hide file tree

Showing 33 changed files with 203 additions and 181 deletions.
diff --git a/docs/source/debugging.rst b/docs/source/debugging.rst
@@ -18,8 +18,8 @@ data that was updated to work post-update:
 
 .. code-block:: diff
 
-    - http://czi.starfish.data.public.s3.amazonaws.com/browse/formatted/20180823/iss_breast/experiment.json
-    + http://czi.starfish.data.public.s3.amazonaws.com/browse/formatted/20180926/iss_breast/experiment.json
+    - http://spacetx.starfish.data.public.s3.amazonaws.com/browse/formatted/20180823/iss_breast/experiment.json
+    + http://spacetx.starfish.data.public.s3.amazonaws.com/browse/formatted/20180926/iss_breast/experiment.json
 
 If you're using your own data with starfish, you may need to re-run your data ingestion workflow
 based on :ref:`TileFetcher` and :ref:`FetchedTile` to generate up-to-date versions of spaceTx-format.

diff --git a/docs/source/usage/iss/iss_pipeline.py b/docs/source/usage/iss/iss_pipeline.py
@@ -9,7 +9,7 @@
 
 
 def iss_pipeline(fov, codebook):
-    primary_image = fov.primary_image
+    primary_image = fov[starfish.FieldOfView.PRIMARY_IMAGES]
 
     # register the raw images
     registration = Registration.FourierShiftRegistration(
@@ -68,7 +68,12 @@ def process_experiment(experiment: starfish.Experiment):
 
 # run the script
 if test:
-    exp = starfish.Experiment.from_json("https://dmf0bdeheu4zf.cloudfront.net/browse/formatted/20180926/iss_breast/experiment.json", True)
+    # TODO: (ttung) Pending a fix for https://github.com/spacetx/starfish/issues/700, it's not
+    # possible to validate the schema for this experiment.
+    exp = starfish.Experiment.from_json(
+        "https://d2nhj9g34unfro.cloudfront.net/browse/formatted/20180926/iss_breast/experiment.json",
+        False,
+    )
 else:
     exp = starfish.Experiment.from_json("iss/formatted/experiment.json")
 decoded_intensities, regions = process_experiment(exp)
diff --git a/examples/get_cli_test_data.py b/examples/get_cli_test_data.py
@@ -5,7 +5,7 @@
 
 import requests
 
-from starfish import Experiment
+from starfish import Experiment, FieldOfView
 from starfish.util.argparse import FsExistsType
 
 
@@ -21,8 +21,10 @@
     for fov in exp.fovs():
         fov_dir = pathlib.Path(args.output_dir, fov.name)
         fov_dir.mkdir()
-        fov.primary_image.write(str(fov_dir / "hybridization.json"))
-        for image_type in fov.auxiliary_image_types:
+        fov[FieldOfView.PRIMARY_IMAGES].write(str(fov_dir / "hybridization.json"))
+        for image_type in fov.image_types:
+            if image_type == FieldOfView.PRIMARY_IMAGES:
+                continue
             fov[image_type].write(str(fov_dir / f"{image_type}.json"))
 
     # get codebook from url and save locally to tmp dir

diff --git a/notebooks/DARTFISH_Pipeline_-_Human_Occipital_Cortex_-_1_FOV.ipynb b/notebooks/DARTFISH_Pipeline_-_Human_Occipital_Cortex_-_1_FOV.ipynb
@@ -23,7 +23,7 @@
     "import seaborn as sns\n",
     "import os\n",
     "\n",
-    "from starfish import data\n",
+    "from starfish import data, FieldOfView\n",
     "from starfish.types import Features, Indices\n",
     "\n",
     "from starfish import IntensityTable\n",
@@ -53,7 +53,7 @@
     "use_test_data = os.getenv(\"USE_TEST_DATA\") is not None\n",
     "exp = data.DARTFISH(use_test_data=use_test_data)\n",
     "\n",
-    "stack = exp.fov().primary_image"
+    "stack = exp.fov()[FieldOfView.PRIMARY_IMAGES]"
    ]
   },
   {
@@ -103,7 +103,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "cnts_benchmark = pd.read_csv('https://dmf0bdeheu4zf.cloudfront.net/20180926/DARTFISH/fov_001/counts.csv')\n",
+    "cnts_benchmark = pd.read_csv('https://d2nhj9g34unfro.cloudfront.net/20181005/DARTFISH/fov_001/counts.csv')\n",
     "cnts_benchmark.head()"
    ]
   },

diff --git a/notebooks/ISS_Pipeline_-_Breast_-_1_FOV.ipynb b/notebooks/ISS_Pipeline_-_Breast_-_1_FOV.ipynb
@@ -28,7 +28,7 @@
     "from showit import image\n",
     "import pprint\n",
     "\n",
-    "from starfish import data\n",
+    "from starfish import data, FieldOfView\n",
     "from starfish.types import Features, Indices"
    ]
   },
@@ -78,7 +78,7 @@
    "outputs": [],
    "source": [
     "fov = experiment.fov()\n",
-    "primary_image = fov.primary_image\n",
+    "primary_image = fov[FieldOfView.PRIMARY_IMAGES]\n",
     "dots = fov['dots']\n",
     "nuclei = fov['nuclei']\n",
     "images = [primary_image, nuclei, dots]"

diff --git a/notebooks/MERFISH_Pipeline_-_U2O2_Cell_Culture_-_1_FOV.ipynb b/notebooks/MERFISH_Pipeline_-_U2O2_Cell_Culture_-_1_FOV.ipynb
@@ -35,7 +35,7 @@
     "import matplotlib.pyplot as plt\n",
     "\n",
     "from showit import image as show_image\n",
-    "from starfish import data\n",
+    "from starfish import data, FieldOfView\n",
     "from starfish.types import Features, Indices"
    ]
   },
@@ -63,7 +63,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "primary_image = experiment.fov().primary_image"
+    "primary_image = experiment.fov()[FieldOfView.PRIMARY_IMAGES]"
    ]
   },
   {
@@ -305,7 +305,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "bench = pd.read_csv('https://dmf0bdeheu4zf.cloudfront.net/MERFISH/benchmark_results.csv',\n",
+    "bench = pd.read_csv('https://d2nhj9g34unfro.cloudfront.net/MERFISH/benchmark_results.csv',\n",
     "                    dtype = {'barcode':object})\n",
     "\n",
     "benchmark_counts = bench.groupby('gene')['gene'].count()\n",

diff --git a/notebooks/allen_smFISH.ipynb b/notebooks/allen_smFISH.ipynb
@@ -21,7 +21,7 @@
     "import numpy as np\n",
     "import os\n",
     "\n",
-    "from starfish import data\n",
+    "from starfish import data, FieldOfView\n",
     "from starfish.types import Indices"
    ]
   },
@@ -49,7 +49,7 @@
    "source": [
     "use_test_data = os.getenv(\"USE_TEST_DATA\") is not None\n",
     "experiment = data.allen_smFISH(use_test_data=use_test_data)\n",
-    "primary_image = experiment.fov().primary_image"
+    "primary_image = experiment.fov()[FieldOfView.PRIMARY_IMAGES]"
    ]
   },
   {

diff --git a/notebooks/assay_comparison.ipynb b/notebooks/assay_comparison.ipynb
@@ -34,7 +34,7 @@
    "outputs": [],
    "source": [
     "# science\n",
-    "from starfish import IntensityTable, Experiment, ImageStack\n",
+    "from starfish import IntensityTable, Experiment, FieldOfView, ImageStack\n",
     "from starfish.plot import histogram, compare_copy_number\n",
     "from starfish.plot.decoded_spots import decoded_spots\n",
     "from starfish.types import Features, Indices\n",
@@ -59,7 +59,7 @@
    "source": [
     "# IntensityTable can't download from directories without list privileges\n",
     "\n",
-    "data_root = \"https://dmf0bdeheu4zf.cloudfront.net/assay_comparison/\"\n",
+    "data_root = \"https://d2nhj9g34unfro.cloudfront.net/assay_comparison/\"\n",
     "iss_link = os.path.join(data_root, \"iss.nc\")\n",
     "merfish_link = os.path.join(data_root, \"merfish.nc\")\n",
     "dartfish_link = os.path.join(data_root, \"dartfish.nc\")\n",
@@ -140,7 +140,7 @@
     "\n",
     "# merfish doesn't have a dots image, and some of the channels are stronger than others.\n",
     "# We can use the scale factors to get the right levels\n",
-    "merfish_background = experiment.fov().primary_image.max_proj(Indices.CH, Indices.ROUND)\n",
+    "merfish_background = experiment.fov()[FieldOfView.PRIMARY_IMAGES].max_proj(Indices.CH, Indices.ROUND)\n",
     "merfish_background = np.reshape(merfish_background, (1, 1, *merfish_background.shape))\n",
     "merfish_background = ImageStack.from_numpy_array(merfish_background)\n",
     "\n",
@@ -349,7 +349,7 @@
    "outputs": [],
    "source": [
     "dartfish_copy_number = pd.read_csv(\n",
-    "    'https://dmf0bdeheu4zf.cloudfront.net/20180926/DARTFISH/fov_001/counts.csv',\n",
+    "    'https://d2nhj9g34unfro.cloudfront.net/20181005/DARTFISH/fov_001/counts.csv',\n",
     "    index_col=0,\n",
     "    squeeze=True\n",
     ")\n",
@@ -479,4 +479,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}
diff --git a/notebooks/osmFISH.ipynb b/notebooks/osmFISH.ipynb
@@ -13,7 +13,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from starfish import data\n",
+    "from starfish import data, FieldOfView\n",
     "from starfish.types import Indices\n",
     "import os"
    ]
@@ -48,7 +48,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "image = experiment.fov().primary_image\n",
+    "image = experiment.fov()[FieldOfView.PRIMARY_IMAGES]\n",
     "image.show_stack({Indices.CH: 0})"
    ]
   }

diff --git a/notebooks/py/DARTFISH_Pipeline_-_Human_Occipital_Cortex_-_1_FOV.py b/notebooks/py/DARTFISH_Pipeline_-_Human_Occipital_Cortex_-_1_FOV.py
@@ -18,7 +18,7 @@
 import seaborn as sns
 import os
 
-from starfish import data
+from starfish import data, FieldOfView
 from starfish.types import Features, Indices
 
 from starfish import IntensityTable
@@ -40,7 +40,7 @@
 use_test_data = os.getenv("USE_TEST_DATA") is not None
 exp = data.DARTFISH(use_test_data=use_test_data)
 
-stack = exp.fov().primary_image
+stack = exp.fov()[FieldOfView.PRIMARY_IMAGES]
 # EPY: END code
 
 # EPY: START code
@@ -64,7 +64,7 @@
 # EPY: END markdown
 
 # EPY: START code
-cnts_benchmark = pd.read_csv('https://dmf0bdeheu4zf.cloudfront.net/20180926/DARTFISH/fov_001/counts.csv')
+cnts_benchmark = pd.read_csv('https://d2nhj9g34unfro.cloudfront.net/20181005/DARTFISH/fov_001/counts.csv')
 cnts_benchmark.head()
 # EPY: END code
 

diff --git a/notebooks/py/ISS_Pipeline_-_Breast_-_1_FOV.py b/notebooks/py/ISS_Pipeline_-_Breast_-_1_FOV.py
@@ -23,7 +23,7 @@
 from showit import image
 import pprint
 
-from starfish import data
+from starfish import data, FieldOfView
 from starfish.types import Features, Indices
 # EPY: END code
 
@@ -52,7 +52,7 @@
 
 # EPY: START code
 fov = experiment.fov()
-primary_image = fov.primary_image
+primary_image = fov[FieldOfView.PRIMARY_IMAGES]
 dots = fov['dots']
 nuclei = fov['nuclei']
 images = [primary_image, nuclei, dots]

diff --git a/notebooks/py/MERFISH_Pipeline_-_U2O2_Cell_Culture_-_1_FOV.py b/notebooks/py/MERFISH_Pipeline_-_U2O2_Cell_Culture_-_1_FOV.py
@@ -25,7 +25,7 @@
 import matplotlib.pyplot as plt
 
 from showit import image as show_image
-from starfish import data
+from starfish import data, FieldOfView
 from starfish.types import Features, Indices
 # EPY: END code
 
@@ -40,7 +40,7 @@
 # EPY: END markdown
 
 # EPY: START code
-primary_image = experiment.fov().primary_image
+primary_image = experiment.fov()[FieldOfView.PRIMARY_IMAGES]
 # EPY: END code
 
 # EPY: START code
@@ -199,7 +199,7 @@
 # EPY: END markdown
 
 # EPY: START code
-bench = pd.read_csv('https://dmf0bdeheu4zf.cloudfront.net/MERFISH/benchmark_results.csv',
+bench = pd.read_csv('https://d2nhj9g34unfro.cloudfront.net/MERFISH/benchmark_results.csv',
                     dtype = {'barcode':object})
 
 benchmark_counts = bench.groupby('gene')['gene'].count()

diff --git a/notebooks/py/allen_smFISH.py b/notebooks/py/allen_smFISH.py
@@ -16,7 +16,7 @@
 import numpy as np
 import os
 
-from starfish import data
+from starfish import data, FieldOfView
 from starfish.types import Indices
 # EPY: END code
 
@@ -33,7 +33,7 @@
 # EPY: START code
 use_test_data = os.getenv("USE_TEST_DATA") is not None
 experiment = data.allen_smFISH(use_test_data=use_test_data)
-primary_image = experiment.fov().primary_image
+primary_image = experiment.fov()[FieldOfView.PRIMARY_IMAGES]
 # EPY: END code
 
 # EPY: START code

diff --git a/notebooks/py/assay_comparison.py b/notebooks/py/assay_comparison.py
@@ -22,7 +22,7 @@
 
 # EPY: START code
 # science
-from starfish import IntensityTable, Experiment, ImageStack
+from starfish import IntensityTable, Experiment, FieldOfView, ImageStack
 from starfish.plot import histogram, compare_copy_number
 from starfish.plot.decoded_spots import decoded_spots
 from starfish.types import Features, Indices
@@ -39,7 +39,7 @@
 # EPY: START code
 # IntensityTable can't download from directories without list privileges
 
-data_root = "https://dmf0bdeheu4zf.cloudfront.net/assay_comparison/"
+data_root = "https://d2nhj9g34unfro.cloudfront.net/assay_comparison/"
 iss_link = os.path.join(data_root, "iss.nc")
 merfish_link = os.path.join(data_root, "merfish.nc")
 dartfish_link = os.path.join(data_root, "dartfish.nc")
@@ -97,7 +97,7 @@ def curl(dest_path, link):
 
 # merfish doesn't have a dots image, and some of the channels are stronger than others.
 # We can use the scale factors to get the right levels
-merfish_background = experiment.fov().primary_image.max_proj(Indices.CH, Indices.ROUND)
+merfish_background = experiment.fov()[FieldOfView.PRIMARY_IMAGES].max_proj(Indices.CH, Indices.ROUND)
 merfish_background = np.reshape(merfish_background, (1, 1, *merfish_background.shape))
 merfish_background = ImageStack.from_numpy_array(merfish_background)
 
@@ -260,7 +260,7 @@ def curl(dest_path, link):
 
 # EPY: START code
 dartfish_copy_number = pd.read_csv(
-    'https://dmf0bdeheu4zf.cloudfront.net/20180926/DARTFISH/fov_001/counts.csv',
+    'https://d2nhj9g34unfro.cloudfront.net/20181005/DARTFISH/fov_001/counts.csv',
     index_col=0,
     squeeze=True
 )

diff --git a/notebooks/py/osmFISH.py b/notebooks/py/osmFISH.py
@@ -8,7 +8,7 @@
 # EPY: END markdown
 
 # EPY: START code
-from starfish import data
+from starfish import data, FieldOfView
 from starfish.types import Indices
 import os
 # EPY: END code
@@ -27,6 +27,6 @@
 # EPY: END markdown
 
 # EPY: START code
-image = experiment.fov().primary_image
+image = experiment.fov()[FieldOfView.PRIMARY_IMAGES]
 image.show_stack({Indices.CH: 0})
 # EPY: END code