Update Python library (#99)

* add kfmd-0.1.7.tar.gz

* remove none properties

* update notebook

* add test for demo notebook

* address comments

* fix python import

* remove nbconvert.ipynb

Makefile

@@ -34,7 +34,8 @@ swagger-py-client:
     java -jar /tmp/swagger/swagger-codegen-cli.jar generate \
     	-i api/service.swagger.json \
     	-g python \
+		--package-name kfmd.openapi_client \
     	-o /tmp/swagger && \
     rm -rf sdk/python/kfmd/swagger_client && \
-    cp -r /tmp/swagger/openapi_client sdk/python/kfmd/ && \
+    cp -r /tmp/swagger/kfmd/openapi_client sdk/python/kfmd/ && \
     rm -rf /tmp/swagger

sdk/python/demo.ipynb

@@ -25,7 +25,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import metadata\n",
+    "from kfmd import metadata\n",
     "import pandas\n",
     "from datetime import datetime"
    ]
@@ -466,12 +466,12 @@
     "print(\"model id is %s\\n\" % model.id)\n",
     "    \n",
     "# Find the execution that produces this model.\n",
-    "output_events = ws1._client.search_events2(model.id).events\n",
+    "output_events = ws1.client.list_events2(model.id).events\n",
     "assert len(output_events) == 1\n",
     "execution_id = output_events[0].execution_id\n",
     "\n",
     "# Find all events related to that execution.\n",
-    "all_events = ws1._client.search_events(execution_id).events\n",
+    "all_events = ws1.client.list_events(execution_id).events\n",
     "assert len(all_events) == 3\n",
     "\n",
     "print(\"\\nAll events related to this model:\")\n",

sdk/python/kfmd/metadata.py

@@ -14,8 +14,8 @@
 
 import datetime
 import json
-import openapi_client
-from openapi_client import Configuration, ApiClient, MetadataServiceApi
+from kfmd import openapi_client
+from kfmd.openapi_client import Configuration, ApiClient, MetadataServiceApi
 
 """
 This module conatins Python API for logging metadata of machine learning
@@ -162,10 +162,9 @@ def serialized(self):
         "create_time":
             openapi_client.MlMetadataValue(string_value=self.create_time),
         "description":
-            openapi_client.MlMetadataValue(string_value=self.description),
+            _mlMetadataStringValue(self.description),
     })
-    if self.description is None:
-      del execution.properties["description"]
+    _del_none_properties(execution.properties)
 
     execution.custom_properties = {}
     if self.workspace is not None:
@@ -305,16 +304,17 @@ def serialization(self):
             "create_time":
                 openapi_client.MlMetadataValue(string_value=self.create_time),
             "description":
-                openapi_client.MlMetadataValue(string_value=self.description),
+                _mlMetadataStringValue(self.description),
             "query":
-                openapi_client.MlMetadataValue(string_value=self.query),
+                _mlMetadataStringValue(self.query),
             "version":
-                openapi_client.MlMetadataValue(string_value=self.version),
+                _mlMetadataStringValue(self.version),
             "owner":
-                openapi_client.MlMetadataValue(string_value=self.owner),
+                _mlMetadataStringValue(self.owner),
             ALL_META_PROPERTY_NAME:
-                openapi_client.MlMetadataValue(string_value=json.dumps(self.__dict__)),
+                _mlMetadataStringValue(json.dumps(self.__dict__)),
         })
+    _del_none_properties(data_set_artifact.properties)
     return data_set_artifact
 
 class Model(object):
@@ -372,16 +372,17 @@ def serialization(self):
             "create_time":
                 openapi_client.MlMetadataValue(string_value=self.create_time),
             "description":
-                openapi_client.MlMetadataValue(string_value=self.description),
+                _mlMetadataStringValue(self.description),
             "model_type":
-                openapi_client.MlMetadataValue(string_value=self.model_type),
+                _mlMetadataStringValue(self.model_type),
             "version":
-                openapi_client.MlMetadataValue(string_value=self.version),
+                _mlMetadataStringValue(self.version),
             "owner":
-                openapi_client.MlMetadataValue(string_value=self.owner),
+                _mlMetadataStringValue(self.owner),
             ALL_META_PROPERTY_NAME:
-                openapi_client.MlMetadataValue(string_value=json.dumps(self.__dict__)),
+                _mlMetadataStringValue(json.dumps(self.__dict__)),
         })
+    _del_none_properties(model_artifact.properties)
     return model_artifact
 
 
@@ -438,27 +439,39 @@ def __init__(self,
     self.create_time = get_rfc3339_time()
 
   def serialization(self):
-    model_artifact = openapi_client.MlMetadataArtifact(
+    metrics_artifact = openapi_client.MlMetadataArtifact(
         uri=self.uri,
         properties={
             "name":
                 openapi_client.MlMetadataValue(string_value=self.name),
             "create_time":
                 openapi_client.MlMetadataValue(string_value=self.create_time),
             "description":
-                openapi_client.MlMetadataValue(string_value=self.description),
+                _mlMetadataStringValue(self.description),
             "metrics_type":
-                openapi_client.MlMetadataValue(string_value=self.metrics_type),
+                _mlMetadataStringValue(self.metrics_type),
             "data_set_id":
-                openapi_client.MlMetadataValue(string_value=self.data_set_id),
+                _mlMetadataStringValue(self.data_set_id),
             "model_id":
-                openapi_client.MlMetadataValue(string_value=self.model_id),
+                _mlMetadataStringValue(self.model_id),
             "owner":
-                openapi_client.MlMetadataValue(string_value=self.owner),
+                _mlMetadataStringValue(self.owner),
             ALL_META_PROPERTY_NAME:
-                openapi_client.MlMetadataValue(string_value=json.dumps(self.__dict__)),
+                _mlMetadataStringValue(json.dumps(self.__dict__)),
         })
-    return model_artifact
+    _del_none_properties(metrics_artifact.properties)
+    return metrics_artifact
 
 def get_rfc3339_time():
       return datetime.datetime.utcnow().isoformat("T") + "Z"
+
+def _mlMetadataStringValue(str):
+      if str is None:
+            return None
+      return openapi_client.MlMetadataValue(string_value=str)
+
+def _del_none_properties(dict):
+      keys = [k for k in dict.keys()]
+      for k in keys:
+            if dict[k] is None:
+                  del(dict[k])

sdk/python/setup.py

@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from setuptools import setup
+from setuptools import setup, find_packages
 
 NAME = 'kfmd'
-VERSION = '0.1.6'
+VERSION = '0.1.8'
 
 REQUIRES = [
 ]
@@ -26,10 +26,7 @@
     description='KubeFlow Metadata SDK',
     author='google',
     install_requires=REQUIRES,
-    packages=[
-        'kfmd',
-        'kfmd.openapi_client',
-    ],
+    packages=find_packages(),
     classifiers=[
         'Intended Audience :: Developers',
         'Intended Audience :: Education',

sdk/python/tests/test.py

@@ -24,7 +24,7 @@ def test_log_metadata_successfully(self):
         run=r,
         description="an execution",
     )
-    assert e.id
+    self.assertIsNotNone(e.id)
 
     data_set = e.log_input(
         metadata.DataSet(
@@ -34,7 +34,7 @@ def test_log_metadata_successfully(self):
             uri="file://path/to/dataset",
             version="v1.0.0",
             query="SELECT * FROM mytable"))
-    assert data_set.id
+    self.assertIsNotNone(data_set.id)
 
     metrics = e.log_output(
         metadata.Metrics(
@@ -47,7 +47,7 @@ def test_log_metadata_successfully(self):
             metrics_type=metadata.Metrics.VALIDATION,
             values={"accuracy": 0.95},
             labels={"mylabel": "l1"}))
-    assert metrics.id
+    self.assertIsNotNone(metrics.id)
 
     model = e.log_output(
         metadata.Model(
@@ -67,7 +67,7 @@ def test_log_metadata_successfully(self):
             },
             version="v0.0.1",
             labels={"mylabel": "l1"}))
-    assert model.id
+    self.assertIsNotNone(model.id)
 
     # Test listing artifacts in a workspace
     self.assertTrue(len(ws1.list()) > 0)
@@ -107,6 +107,27 @@ def test_log_invalid_artifacts_should_fail(self):
     ))
     self.assertRaises(ValueError, e.log_output, artifact2)
 
+  def test_log_metadata_successfully_with_minimum_information(self):
+    ws1 = metadata.Workspace(backend_url_prefix="127.0.0.1:8080", name="ws_1")
+
+    r = metadata.Run(workspace=ws1, name="first run")
+
+    e = metadata.Execution(name="test execution", workspace=ws1, run=r)
+    self.assertIsNotNone(e.id)
+
+    data_set = e.log_input(
+        metadata.DataSet(name="mytable-dump", uri="file://path/to/dataset"))
+    self.assertIsNotNone(data_set.id)
+
+    metrics = e.log_output(
+        metadata.Metrics(name="MNIST-evaluation",
+            uri="gcs://my-bucket/mnist-eval.csv"))
+    self.assertIsNotNone(metrics.id)
+
+    model = e.log_output(
+        metadata.Model(name="MNIST", uri="gcs://my-bucket/mnist"))
+    self.assertIsNotNone(model.id)
+
 class ArtifactFixture(object):
   ARTIFACT_TYPE_NAME = "artifact_types/kubeflow.org/alpha/artifact_fixture"
 

test/scripts/setup-services-run-tests.sh

@@ -130,5 +130,11 @@ cd "${SRC_DIR}/test/e2e" && bash make_requests.sh
 # Run Python tests
 pip install pandas
 cd "${SRC_DIR}/sdk/python" && bash tests/run_tests.sh
+# Test demo notebook
+pip install jupyterlab
+pip install nbconvert
+cd "${SRC_DIR}/sdk/python" && \
+  sed -i -e "s@metadata-service.kubeflow:8080@127.0.0.1:8080@" demo.ipynb && \
+  python3 -m nbconvert --to notebook --execute demo.ipynb
 
 exit 0