From c17651607c8a8cffef37abd50faf468e36e004e6 Mon Sep 17 00:00:00 2001 From: Bago Amirbekian Date: Fri, 30 Nov 2018 22:03:55 -0800 Subject: [PATCH] [ML-5700] Update release script to build and push docs. (#176) This PR adds doc building to the release script. The script will checking the release tag and build the docs in docker. It will then push the docs branch, eg "gh-pages-1.4.0" upstream & replace the "gh-pages" branch. I opted for replacing the gh-pages branch instead of creating a PR because this type of PR will inherently have conflicts. --- .gitignore | 1 + dev/build-docs.sh | 3 ++- dev/release.py | 66 ++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 57 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index b95b0bde..6ba92c1b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.class *.log *.pyc +python/.setup.sh build/*.jar docs/_site diff --git a/dev/build-docs.sh b/dev/build-docs.sh index 587e284d..b9dd3b66 100755 --- a/dev/build-docs.sh +++ b/dev/build-docs.sh @@ -10,4 +10,5 @@ docker build -t databricks/sparkdl-docs docs/ build/sbt assembly # build the API docs -docker run --rm -v "$(pwd):/mnt/sparkdl" databricks/sparkdl-docs dev/build-docs-in-docker.sh +docker run --rm -v "$(pwd):/mnt/sparkdl" databricks/sparkdl-docs \ + bash -i -c dev/build-docs-in-docker.sh diff --git a/dev/release.py b/dev/release.py index 90d4b187..2d7d7721 100755 --- a/dev/release.py +++ b/dev/release.py @@ -1,12 +1,18 @@ #!/usr/bin/env python import click +from datetime import datetime from six.moves import input from subprocess import check_call, check_output import sys -WORKING_BRANCH = "PREP_RELEASE_%s" DATABRICKS_REMOTE = "git@github.com:databricks/spark-deep-learning.git" -PUBLISH_MODES = {"local": "publishLocal", "m2": "publishM2", "spark-package": "spDist"} +PUBLISH_MODES = {"local": "publishLocal", "m2": "publishM2", "spark-package-publish": "spPublish"} + +WORKING_BRANCH = "WORKING_BRANCH_RELEASE_%s_@%s" +# lower case "z" puts the branch at the end of the github UI. +WORKING_DOCS_BRANCH = "zWORKING_BRANCH_DOCS_%s_@%s" +RELEASE_TAG = "v%s" + def prominentPrint(x): x = str(x) @@ -30,9 +36,12 @@ def verify(prompt, interactive): @click.option("--publish-to", default="local", help="Where to publish artifact, one of: %s" % list(PUBLISH_MODES.keys())) @click.option("--no-prompt", is_flag=True, help="Automated mode with no user prompts.") -def main(release_version, next_version, publish_to, no_prompt): +@click.option("--git-remote", default=DATABRICKS_REMOTE, + help="Push current branch and docs to this git remote.") +def main(release_version, next_version, publish_to, no_prompt, git_remote): interactive = not no_prompt + time = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") if publish_to not in PUBLISH_MODES: modes = list(PUBLISH_MODES.keys()) print("Unknown publish target, --publish-to should be one of: %s." % modes) @@ -47,22 +56,32 @@ def main(release_version, next_version, publish_to, no_prompt): sys.exit(1) current_branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip() - if current_branch != "master": + if current_branch != b"master": if not verify("You're not on the master branch do you want to continue? (y/n)", interactive): sys.exit(1) uncommitted_changes = check_output(["git", "diff", "--stat"]) - if uncommitted_changes != "": + if uncommitted_changes != b"": print(uncommitted_changes) print("There seem to be uncommitted changes on your current branch. Please commit or " "stash them and try again.") sys.exit(1) - working_branch = WORKING_BRANCH % release_version - if working_branch in check_output(["git", "branch"]): - prominentPrint( - "Working branch %s already exists, please delete it and try again." % working_branch) + working_branch = WORKING_BRANCH % (release_version, time) + gh_pages_branch = WORKING_DOCS_BRANCH % (release_version, time) + + release_tag = RELEASE_TAG % release_version + target_tags = [release_tag] + + existing_tags = check_output(["git", "tag"]).decode().split() + conflict_tags = list(filter(lambda a: a in existing_tags, target_tags)) + if conflict_tags: + msg = ("The following tags already exist:\n" + " %s\n" + "Please delete them and try.") + msg = msg % "\n ".join(conflict_tags) + prominentPrint(msg) sys.exit(1) prominentPrint("Creating working branch for this release.") @@ -73,7 +92,7 @@ def main(release_version, next_version, publish_to, no_prompt): check_call(["./build/sbt", update_version]) prominentPrint("Building and testing with sbt.") - check_call(["git", "checkout", "v%s" % release_version]) + check_call(["git", "checkout", release_tag]) publish_target = PUBLISH_MODES[publish_to] check_call(["./build/sbt", "clean", publish_target]) @@ -84,8 +103,31 @@ def main(release_version, next_version, publish_to, no_prompt): check_call(["git", "branch", "-d", working_branch]) prominentPrint("Local branch updated") - if verify("Would you like to push local branch to databricks remote? (y/n)", interactive): - check_call(["git", "push", DATABRICKS_REMOTE, current_branch]) + if verify("Would you like to push local branch & version tag to remote: %s? (y/n)" % git_remote, + interactive): + check_call(["git", "push", git_remote, current_branch]) + check_call(["git", "push", git_remote, release_tag]) + + prominentPrint("Building release docs") + + if not verify("Would you like to build release docs? (y/n)", interactive): + # All done, exit happy + sys.exit(0) + + check_call(["git", "checkout", "-b", gh_pages_branch, release_tag]) + check_call(["./dev/build-docs.sh"]) + + commit_message = "Build docs for release %s." % release_version + check_call(["git", "add", "-f", "docs/_site"]) + check_call(["git", "commit", "-m", commit_message]) + msg = "Would you like to push docs branch to %s and update gh-pages branch? (y/n)" + msg %= git_remote + if verify(msg, interactive): + check_call(["git", "push", git_remote, gh_pages_branch]) + check_call(["git", "push", "-f", git_remote, gh_pages_branch+":gh-pages"]) + + check_call(["git", "checkout", current_branch]) + check_call(["git", "branch", "-D", gh_pages_branch]) if __name__ == "__main__":