From 00964c2e0c4c30070bfc31faae0f939f23204d8a Mon Sep 17 00:00:00 2001 From: Bago Amirbekian Date: Tue, 4 Dec 2018 13:50:37 -0800 Subject: [PATCH] Fix warnings in doc build. (#177) We were using `_list_files_with_extension` to find the assembly jar in spark-deep-learning, and simply taking the last jar in the directory where we expect the assembly file to be. During build and release this directory can also have `sources*.jar` which can come after `assembly*.jar`. I switched to using glob so we could check not only the extension but also part of the file name. Since `_list_files_with_extension` seems to be strictly a subset of glob, went ahead and replaced all usages of it. There was another issue in our doc build as well, the indentation of `index.rst` was causing some of the doc files to be missed. These files were being left out of the doc build with this warning: ``` /mnt/sparkdl/python/docs/index.rst:15: WARNING: toctree contains reference to nonexisting document u' sparkdl.graph' /mnt/sparkdl/python/docs/index.rst:15: WARNING: toctree contains reference to nonexisting document u' sparkdl.image' /mnt/sparkdl/python/docs/index.rst:15: WARNING: toctree contains reference to nonexisting document u' sparkdl.transformers' /mnt/sparkdl/python/docs/index.rst:15: WARNING: toctree contains reference to nonexisting document u' sparkdl.udf' /mnt/sparkdl/python/docs/index.rst:15: WARNING: toctree contains reference to nonexisting document u' sparkdl.utils' checking consistency... /mnt/sparkdl/python/docs/sparkdl.graph.rst: WARNING: document isn't included in any toctree /mnt/sparkdl/python/docs/sparkdl.image.rst: WARNING: document isn't included in any toctree /mnt/sparkdl/python/docs/sparkdl.transformers.rst: WARNING: document isn't included in any toctree /mnt/sparkdl/python/docs/sparkdl.udf.rst: WARNING: document isn't included in any toctree /mnt/sparkdl/python/docs/sparkdl.utils.rst: WARNING: document isn't included in any toctree ``` --- dev/run.py | 11 ++++------- python/docs/index.rst | 10 +++++----- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/dev/run.py b/dev/run.py index bbbb96d0..96ba4c68 100755 --- a/dev/run.py +++ b/dev/run.py @@ -5,6 +5,7 @@ from __future__ import print_function +from glob import glob import os import subprocess import sys @@ -32,10 +33,6 @@ def call_subprocess(cmd, add_env={}, verbose=True): return subprocess.call(cmd, env=env) -def _list_files_with_extension(dir_name, ext): - return [os.path.join(dir_name, f) for f in os.listdir(dir_name) if f.endswith(ext)] - - def _get_configured_env(_required_env): spark_home = _required_env["SPARK_HOME"] scala_version = _required_env["SCALA_VERSION"] @@ -48,17 +45,17 @@ def _get_configured_env(_required_env): configured_env = {} spark_lib_path = os.path.join(spark_home, "python/lib") - configured_env["LIBS"] = ":".join(_list_files_with_extension(spark_lib_path, ".zip")) + configured_env["LIBS"] = ":".join(glob(spark_lib_path + "/*.zip")) scala_version_major_minor = ".".join(scala_version.split(".")[0:2]) assembly_path = os.path.join(PROJECT_DIR, "target/scala-" + scala_version_major_minor) - jar_path = ":".join(_list_files_with_extension(assembly_path, ".jar")) + jar_path = ":".join(glob(assembly_path + "/*.jar")) configured_env["JAR_PATH"] = jar_path python_paths = [ os.getenv("PYTHONPATH", ""), os.path.join(PROJECT_DIR, "python"), - _list_files_with_extension(assembly_path, ".jar")[-1], + glob(assembly_path + "/*assembly*.jar")[-1], os.path.join(spark_home, "python"), configured_env["LIBS"] ] diff --git a/python/docs/index.rst b/python/docs/index.rst index f85724ce..c244282a 100644 --- a/python/docs/index.rst +++ b/python/docs/index.rst @@ -15,11 +15,11 @@ Contents: .. toctree:: :maxdepth: 2 - sparkdl.graph - sparkdl.image - sparkdl.transformers - sparkdl.udf - sparkdl.utils + sparkdl.graph + sparkdl.image + sparkdl.transformers + sparkdl.udf + sparkdl.utils Module contents ---------------