Release beta 27

Makefile

@@ -1,6 +1,6 @@
 SHELL=/bin/bash
 
-ifneq ($(shell python -c "import sys; print(hasattr(sys, 'real_prefix'))"),True)
+ifndef VIRTUAL_ENV
 $(error Looks like no virtualenv is active)
 endif
 

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name="hca-metadata-api",
-    version="1.0b26",
+    version="1.0b27",
     license='MIT',
     install_requires=[
         "dataclasses >= 0.6;python_version<'3.7'"

src/humancellatlas/data/metadata/api.py

@@ -16,6 +16,7 @@
     Type,
     TypeVar,
     Union,
+    Dict,
 )
 from uuid import UUID
 import warnings
@@ -35,16 +36,17 @@
 # A few helpful type aliases
 #
 UUID4 = UUID
-AnyJSON2 = Union[str, int, float, bool, None, Mapping[str, Any], List[Any]]
-AnyJSON1 = Union[str, int, float, bool, None, Mapping[str, AnyJSON2], List[AnyJSON2]]
-AnyJSON = Union[str, int, float, bool, None, Mapping[str, AnyJSON1], List[AnyJSON1]]
-JSON = Mapping[str, AnyJSON]
+AnyJSON2 = Union[str, int, float, bool, None, Dict[str, Any], List[Any]]
+AnyJSON1 = Union[str, int, float, bool, None, Dict[str, AnyJSON2], List[AnyJSON2]]
+AnyJSON = Union[str, int, float, bool, None, Dict[str, AnyJSON1], List[AnyJSON1]]
+JSON = Dict[str, AnyJSON]
 
 
 @dataclass(init=False)
 class Entity:
     json: JSON = field(repr=False)
     document_id: UUID4
+    submitter_id: Optional[str]
 
     @classmethod
     def from_json(cls, json: JSON, **kwargs):
@@ -62,6 +64,7 @@ def __init__(self, json: JSON) -> None:
         self.json = json
         provenance = json.get('hca_ingest') or json['provenance']
         self.document_id = UUID4(provenance['document_id'])
+        self.submitter_id = provenance.get('submitter_id')
 
     @property
     def address(self):
@@ -258,6 +261,7 @@ class DonorOrganism(Biomaterial):
     organism_age: str
     organism_age_unit: str
     sex: str
+    development_stage: Optional[str]
 
     def __init__(self, json: JSON):
         super().__init__(json)
@@ -267,6 +271,7 @@ def __init__(self, json: JSON):
         self.organism_age = content.get('organism_age')
         self.organism_age_unit = ontology_label(content.get('organism_age_unit'), default=None)
         self.sex = lookup(content, 'sex', 'biological_sex')
+        self.development_stage = ontology_label(content.get('development_stage'), default=None)
 
     @property
     def organism_age_in_seconds(self) -> Optional[AgeRange]:
@@ -507,12 +512,14 @@ def _connect_to(self, other: Entity, forward: bool) -> None:
 @dataclass(init=False)
 class LibraryPreparationProtocol(Protocol):
     library_construction_method: str
+    nucleic_acid_source: Optional[str]
 
     def __init__(self, json: JSON) -> None:
         super().__init__(json)
         content = json.get('content', json)
         temp = lookup(content, 'library_construction_method', 'library_construction_approach')
         self.library_construction_method = ontology_label(temp) if isinstance(temp, dict) else temp
+        self.nucleic_acid_source = content.get('nucleic_acid_source')
 
     @property
     def library_construction_approach(self) -> str:
@@ -583,30 +590,57 @@ class ImagingPreparationProtocol(Protocol):
     pass
 
 
+def is_optional(t):
+    """
+    https://stackoverflow.com/a/62641842/4171119
+
+    >>> is_optional(str)
+    False
+    >>> is_optional(Optional[str])
+    True
+    >>> is_optional(Union[str, None])
+    True
+    >>> is_optional(Union[None, str])
+    True
+    >>> is_optional(Union[str, None, int])
+    True
+    >>> is_optional(Union[str, int])
+    False
+    """
+    return t == Optional[t]
+
+
 @dataclass(init=False)
 class ManifestEntry:
     json: JSON = field(init=False, repr=False)
     content_type: str = field(init=False)
     crc32c: str
     indexed: bool
     name: str
-    s3_etag: str
-    sha1: str
+    s3_etag: Optional[str]
+    sha1: Optional[str]
     sha256: str
     size: int
     # only populated if bundle was requested with `directurls` or `directurls` set
-    url: Optional[str] = field(init=False)
+    url: Optional[str]
     uuid: UUID4 = field(init=False)
     version: str
 
     def __init__(self, json: JSON):
+        # '/' was once forbidden in file paths and was encoded with '!'. Now
+        # '/' is allowed and we force it in the metadata so that backwards
+        # compatibility is simplified downstream.
+        json['name'] = json['name'].replace('!', '/')
         self.json = json
         self.content_type = json['content-type']
-        self.url = json.get('url')
         self.uuid = UUID4(json['uuid'])
         for f in fields(self):
             if f.init:
-                setattr(self, f.name, json[f.name])
+                value = json.get(f.name)
+                if value is None and not is_optional(f.type):
+                    raise TypeError('Property cannot be absent or None', f.name)
+                else:
+                    setattr(self, f.name, value)
 
 
 @dataclass(init=False)
@@ -620,7 +654,11 @@ class File(LinkedEntity):
     def __init__(self, json: JSON, manifest: Mapping[str, ManifestEntry]):
         super().__init__(json)
         content = json.get('content', json)
+        # '/' was once forbidden in file paths and was encoded with '!'. Now
+        # '/' is allowed and we force it in the metadata so that backwards
+        # compatibility is simplified downstream.
         core = content['file_core']
+        core['file_name'] = core['file_name'].replace('!', '/')
         self.format = lookup(core, 'format', 'file_format')
         self.manifest_entry = manifest[core['file_name']]
         self.content_description = {ontology_label(cd) for cd in core.get('content_description', [])}

test/cans/prod/6b498499-c5b4-452f-9ff9-2318dbb86000/2019-01-03T163633.780215Z/manifest.json

@@ -4,7 +4,6 @@
         "crc32c": "afa286ee",
         "indexed": true,
         "name": "cell_suspension_0.json",
-        "s3_etag": "2ff58a0bf5b921ff1c370c2d643c628f",
         "sha1": "9300432ea2bf3550791904b26f824dbc547178dc",
         "sha256": "1ae2592f34e00d6c322a660f5cde9a943304bc4f59fc824c0fb2789e0dd4886f",
         "size": 1032,
@@ -17,7 +16,6 @@
         "indexed": true,
         "name": "organoid_0.json",
         "s3_etag": "5195c84d0f7054aea3323fd948f5e511",
-        "sha1": "16a46fde2b1ff18b73f11cd8f89de0131b570010",
         "sha256": "21a2f091afc05363f42cf1a7c08a121c934487873c34d8fc56bb47d20aa1b7cb",
         "size": 1168,
         "uuid": "a690df06-cc52-4346-a926-fe5e5b3a7547",
@@ -28,8 +26,6 @@
         "crc32c": "85ccc604",
         "indexed": true,
         "name": "organoid_1.json",
-        "s3_etag": "7fce3d564fba0973a6796ddea88745c1",
-        "sha1": "b3ab6c1dafea184d5a609ff5d36b434c7a451541",
         "sha256": "7ddccdaaf573866f4209286dd3c33e537ec2ef721ff8f9c0d09b1d7bcc696d9a",
         "size": 1168,
         "uuid": "f0b636ce-505c-4339-a69c-2305352b6796",

test/cans/prod/86e7b58e-b9f0-4020-8b34-c61d6da02d44/2019-09-20T103932.395795Z/metadata.json

@@ -220,6 +220,7 @@
             "document_id": "2e64e936-85ec-49a0-8365-578b5621bcb1",
             "submission_date": "2019-09-20T08:29:52.277Z",
             "update_date": "2019-09-20T08:38:58.167Z",
+            "submitter_id": "67a720af-4482-4619-81d7-3693b2d3cc4c",
             "schema_major_version": 2,
             "schema_minor_version": 2
         }
@@ -236,6 +237,7 @@
             "document_id": "720e6453-71ff-4742-ba6a-e5a3a5d3cee0",
             "submission_date": "2019-09-20T08:29:52.285Z",
             "update_date": "2019-09-20T08:38:58.165Z",
+            "submitter_id": "67a720af-4482-4619-81d7-3693b2d3cc4c",
             "schema_major_version": 2,
             "schema_minor_version": 2
         }
@@ -252,6 +254,7 @@
             "document_id": "09917cf3-76de-4b85-8944-03664cd0d954",
             "submission_date": "2019-09-20T08:29:52.292Z",
             "update_date": "2019-09-20T08:38:58.165Z",
+            "submitter_id": "67a720af-4482-4619-81d7-3693b2d3cc4c",
             "schema_major_version": 2,
             "schema_minor_version": 2
         }
@@ -268,6 +271,7 @@
             "document_id": "8fb964a1-fd66-43b6-9d13-57db6bd185f2",
             "submission_date": "2019-09-20T08:29:52.299Z",
             "update_date": "2019-09-20T08:38:58.167Z",
+            "submitter_id": "67a720af-4482-4619-81d7-3693b2d3cc4c",
             "schema_major_version": 2,
             "schema_minor_version": 2
         }