[ML-5700] Update release script to build and push docs. (#176)

This PR adds doc building to the release script. The script will checking the release tag and build the docs in docker. It will then push the docs branch, eg "gh-pages-1.4.0" upstream & replace the "gh-pages" branch.

I opted for replacing the gh-pages branch instead of creating a PR because this type of PR will inherently have conflicts.

.gitignore

@@ -3,6 +3,7 @@
 *.class
 *.log
 *.pyc
+python/.setup.sh
 build/*.jar
 
 docs/_site

dev/build-docs.sh

@@ -10,4 +10,5 @@ docker build -t databricks/sparkdl-docs docs/
 build/sbt assembly
 
 # build the API docs
-docker run --rm -v "$(pwd):/mnt/sparkdl" databricks/sparkdl-docs dev/build-docs-in-docker.sh
+docker run --rm -v "$(pwd):/mnt/sparkdl" databricks/sparkdl-docs \
+    bash -i -c dev/build-docs-in-docker.sh

dev/release.py

@@ -1,12 +1,18 @@
 #!/usr/bin/env python
 import click
+from datetime import datetime
 from six.moves import input
 from subprocess import check_call, check_output
 import sys
 
-WORKING_BRANCH = "PREP_RELEASE_%s"
 DATABRICKS_REMOTE = "git@github.com:databricks/spark-deep-learning.git"
-PUBLISH_MODES = {"local": "publishLocal", "m2": "publishM2", "spark-package": "spDist"}
+PUBLISH_MODES = {"local": "publishLocal", "m2": "publishM2", "spark-package-publish": "spPublish"}
+
+WORKING_BRANCH = "WORKING_BRANCH_RELEASE_%s_@%s"
+# lower case "z" puts the branch at the end of the github UI.
+WORKING_DOCS_BRANCH = "zWORKING_BRANCH_DOCS_%s_@%s"
+RELEASE_TAG = "v%s"
+
 
 def prominentPrint(x):
     x = str(x)
@@ -30,9 +36,12 @@ def verify(prompt, interactive):
 @click.option("--publish-to", default="local",
               help="Where to publish artifact, one of: %s" % list(PUBLISH_MODES.keys()))
 @click.option("--no-prompt", is_flag=True, help="Automated mode with no user prompts.")
-def main(release_version, next_version, publish_to, no_prompt):
+@click.option("--git-remote", default=DATABRICKS_REMOTE,
+              help="Push current branch and docs to this git remote.")
+def main(release_version, next_version, publish_to, no_prompt, git_remote):
     interactive = not no_prompt
 
+    time = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
     if publish_to not in PUBLISH_MODES:
         modes = list(PUBLISH_MODES.keys())
         print("Unknown publish target, --publish-to should be one of: %s." % modes)
@@ -47,22 +56,32 @@ def main(release_version, next_version, publish_to, no_prompt):
         sys.exit(1)
 
     current_branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip()
-    if current_branch != "master":
+    if current_branch != b"master":
         if not verify("You're not on the master branch do you want to continue? (y/n)",
                       interactive):
             sys.exit(1)
 
     uncommitted_changes = check_output(["git", "diff", "--stat"])
-    if uncommitted_changes != "":
+    if uncommitted_changes != b"":
         print(uncommitted_changes)
         print("There seem to be uncommitted changes on your current branch. Please commit or "
               "stash them and try again.")
         sys.exit(1)
 
-    working_branch = WORKING_BRANCH % release_version
-    if working_branch in check_output(["git", "branch"]):
-        prominentPrint(
-            "Working branch %s already exists, please delete it and try again." % working_branch)
+    working_branch = WORKING_BRANCH % (release_version, time)
+    gh_pages_branch = WORKING_DOCS_BRANCH % (release_version, time)
+
+    release_tag = RELEASE_TAG % release_version
+    target_tags = [release_tag]
+
+    existing_tags = check_output(["git", "tag"]).decode().split()
+    conflict_tags = list(filter(lambda a: a in existing_tags, target_tags))
+    if conflict_tags:
+        msg = ("The following tags already exist:\n"
+               "    %s\n"
+               "Please delete them and try.")
+        msg = msg % "\n    ".join(conflict_tags)
+        prominentPrint(msg)
         sys.exit(1)
 
     prominentPrint("Creating working branch for this release.")
@@ -73,7 +92,7 @@ def main(release_version, next_version, publish_to, no_prompt):
     check_call(["./build/sbt", update_version])
 
     prominentPrint("Building and testing with sbt.")
-    check_call(["git", "checkout", "v%s" % release_version])
+    check_call(["git", "checkout", release_tag])
 
     publish_target = PUBLISH_MODES[publish_to]
     check_call(["./build/sbt", "clean", publish_target])
@@ -84,8 +103,31 @@ def main(release_version, next_version, publish_to, no_prompt):
     check_call(["git", "branch", "-d", working_branch])
 
     prominentPrint("Local branch updated")
-    if verify("Would you like to push local branch to databricks remote? (y/n)", interactive):
-        check_call(["git", "push", DATABRICKS_REMOTE, current_branch])
+    if verify("Would you like to push local branch & version tag to remote: %s? (y/n)" % git_remote,
+              interactive):
+        check_call(["git", "push", git_remote, current_branch])
+        check_call(["git", "push", git_remote, release_tag])
+
+    prominentPrint("Building release docs")
+
+    if not verify("Would you like to build release docs? (y/n)", interactive):
+        # All done, exit happy
+        sys.exit(0)
+
+    check_call(["git", "checkout", "-b", gh_pages_branch, release_tag])
+    check_call(["./dev/build-docs.sh"])
+
+    commit_message = "Build docs for release %s." % release_version
+    check_call(["git", "add", "-f", "docs/_site"])
+    check_call(["git", "commit", "-m", commit_message])
+    msg = "Would you like to push docs branch to %s and update gh-pages branch? (y/n)"
+    msg %= git_remote
+    if verify(msg, interactive):
+        check_call(["git", "push", git_remote, gh_pages_branch])
+        check_call(["git", "push", "-f", git_remote, gh_pages_branch+":gh-pages"])
+
+    check_call(["git", "checkout", current_branch])
+    check_call(["git", "branch", "-D", gh_pages_branch])
 
 
 if __name__ == "__main__":