From 544d91719ad6bcc465cf87df977b1854c45def34 Mon Sep 17 00:00:00 2001 From: "Keith W. Campbell" Date: Mon, 12 Feb 2024 19:54:20 -0500 Subject: [PATCH] Merge scripts to fetch or update source repositories * eliminate repetitive option processing * ignore '--parallel' option * handle special openssl options and treat like other repos * remove old, unused scripts Signed-off-by: Keith W. Campbell --- closed/get_j9_source.sh | 275 --------------------------- closed/get_openssl_source.sh | 96 ---------- get_source.sh | 355 ++++++++++++++++++++++++++--------- 3 files changed, 268 insertions(+), 458 deletions(-) delete mode 100644 closed/get_j9_source.sh delete mode 100644 closed/get_openssl_source.sh diff --git a/closed/get_j9_source.sh b/closed/get_j9_source.sh deleted file mode 100644 index 325b4c3348..0000000000 --- a/closed/get_j9_source.sh +++ /dev/null @@ -1,275 +0,0 @@ -#!/bin/bash - -# =========================================================================== -# (c) Copyright IBM Corp. 2017, 2024 All Rights Reserved -# =========================================================================== -# -# This code is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License version 2 only, as -# published by the Free Software Foundation. -# -# IBM designates this particular file as subject to the "Classpath" exception -# as provided by IBM in the LICENSE file that accompanied this code. -# -# This code is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# version 2 for more details (a copy is included in the LICENSE file that -# accompanied this code). -# -# You should have received a copy of the GNU General Public License version -# 2 along with this work; if not, see . -# -# =========================================================================== - -# exit immediately if any unexpected error occurs -set -e - -usage() { - echo "Usage: $0 [-h|--help] [-openj9-repo=] [-openj9-branch=] [-openj9-sha=] [... other OpenJ9 repositories and branches options] [-parallel=]" - echo "where:" - echo " -h|--help print this help, then exit" - echo " -openj9-repo the OpenJ9 repository url: https://github.com/eclipse-openj9/openj9.git" - echo " or git@github.com:/openj9.git" - echo " -openj9-branch the OpenJ9 git branch: master" - echo " -openj9-sha a commit SHA for the OpenJ9 repository" - echo " -openj9-reference a local repo to use as a clone reference" - echo " -omr-repo the OpenJ9/omr repository url: https://github.com/eclipse-openj9/openj9-omr.git" - echo " or git@github.com:/openj9-omr.git" - echo " -omr-branch the OpenJ9/omr git branch: openj9" - echo " -omr-sha a commit SHA for the omr repository" - echo " -omr-reference a local repo to use as a clone reference" - echo " -openjceplus-repo the OpenJCEPlus repository url" - echo " -openjceplus-branch the OpenJCEPlus git branch" - echo " -openjceplus-sha a commit SHA for the OpenJCEPlus repository" - echo " -openjceplus-reference a local repo to use as a clone reference" - echo " -gskit-bin the GSKit binary url" - echo " -gskit-sdk-bin the GSKIT SDK binary url" - echo " -gskit-credential the credential for downloading the GSKit binaries" - echo " -parallel (boolean) if 'true' then the clone j9 repository commands run in parallel, default is false" - echo "" - exit 1 -} - -# require bash 4.0 or later to support associative arrays -if [ "0${BASH_VERSINFO[0]}" -lt 4 ] ; then - echo "Bash version 4.0 or later is required!" - exit 1 -fi - -declare -A branches -declare -A commands -declare -A git_urls -declare -A shas -declare -A references - -git_urls[openj9]=https://github.com/eclipse-openj9/openj9 -git_urls[omr]=https://github.com/eclipse-openj9/openj9-omr - -currentbranch=$(git rev-parse --abbrev-ref HEAD) -if [[ "$currentbranch" =~ v[0-9]+\.[0-9]+(\.[0-9]+)?-release ]] ; then - branches[openj9]=$currentbranch - branches[omr]=$currentbranch -else - branches[openj9]=master - branches[omr]=openj9 -fi - -pflag=false - -for i in "$@" ; do - case $i in - -h | --help ) - usage - ;; - - -openj9-repo=* ) - git_urls[openj9]="${i#*=}" - ;; - - -openj9-branch=* ) - branches[openj9]="${i#*=}" - ;; - - -openj9-sha=* ) - shas[openj9]="${i#*=}" - ;; - - -openj9-reference=* ) - references[openj9]="${i#*=}" - ;; - - -omr-repo=* ) - git_urls[omr]="${i#*=}" - ;; - - -omr-branch=* ) - branches[omr]="${i#*=}" - ;; - - -omr-sha=* ) - shas[omr]="${i#*=}" - ;; - - -omr-reference=* ) - references[omr]="${i#*=}" - ;; - - -openjceplus-repo=* ) - git_urls[OpenJCEPlus]="${i#*=}" - ;; - - -openjceplus-branch=* ) - branches[OpenJCEPlus]="${i#*=}" - ;; - - -openjceplus-sha=* ) - shas[OpenJCEPlus]="${i#*=}" - ;; - - -openjceplus-reference=* ) - references[OpenJCEPlus]="${i#*=}" - ;; - - -gskit-bin=* ) - gskit_bin="${i#*=}" - ;; - - -gskit-sdk-bin=* ) - gskit_sdk_bin="${i#*=}" - ;; - - -gskit-credential=* ) - gskit_credential="${i#*=}" - ;; - - -parallel=* ) - pflag="${i#*=}" - ;; - - '--' ) # no more options - break - ;; - - -*) # bad option - usage - ;; - - *) # bad option - usage - ;; - esac -done - -# clone OpenJ9 repos -date '+[%F %T] Get OpenJ9 sources' -START_TIME=$(date +%s) - -for i in "${!git_urls[@]}" ; do - branch=${branches[$i]} - - if [ -d ${i} ] ; then - echo - echo "Update ${i} source" - echo - - cd ${i} - git pull --rebase origin ${branch} - - if [ -f .gitmodules ] ; then - git pull --rebase --recurse-submodules=yes - git submodule update --rebase --recursive - fi - cd - > /dev/null - else - if [ -n "${references[$i]+_}" ] ; then - reference=" --reference ${references[$i]}" - else - reference="" - fi - git_clone_command="git clone${reference} --recursive -b ${branch} ${git_urls[$i]} ${i}" - commands[$i]=$git_clone_command - - echo - echo "Clone repository: ${i}" - echo - - if [ ${pflag} = true ] ; then - # run git clone in parallel - ( if $git_clone_command ; then echo 0 ; else echo $? ; fi ) > /tmp/${i}.pid.rc 2>&1 & - else - $git_clone_command - fi - fi -done - -if [ ${pflag} = true ] ; then - # wait for all subprocesses to complete - wait -fi - -# Download OCK binaries and create Java module folder. -openjceplus_source=OpenJCEPlus -if [ -n "${git_urls[$openjceplus_source]}" ] ; then - - echo - echo "$openjceplus_source exists, download OCK binaries" - echo - - cd $openjceplus_source - mkdir -p ock/jgsk_sdk/lib64 - - if [ -n "$gskit_credential" ] ; then - curl -u "$gskit_credential" $gskit_bin > ock/jgsk_crypto.tar - curl -u "$gskit_credential" $gskit_sdk_bin > ock/jgsk_crypto_sdk.tar - else - echo - echo "GSKit binaries are needed for compiling $openjceplus_source" - echo "Please set -gskit-bin, -gskit-sdk-bin, and -gskit-credential" - exit 1 - fi - - tar -xf ock/jgsk_crypto_sdk.tar -C ock - tar -xf ock/jgsk_crypto.tar -C ock/jgsk_sdk/lib64 - - # Create OpenJCEPlus Java module folder. - mkdir -p src/main/openjceplus/share/classes - cp -r src/main/java/* src/main/openjceplus/share/classes/ - - cd .. -fi - -END_TIME=$(date +%s) -date "+[%F %T] OpenJ9 clone repositories finished in $(($END_TIME - $START_TIME)) seconds" - -for i in "${!git_urls[@]}" ; do - if [ -e /tmp/${i}.pid.rc ] ; then - # check if the git clone repository command failed - rc=$(cat /tmp/${i}.pid.rc | tr -d ' \n\r') - - if [ "$rc" != 0 ] ; then - echo "ERROR: repository ${i} exited abnormally!" - cat /tmp/${i}.pid.rc - echo "Re-run: ${commands[$i]}" - - # clean up sources - if [ -d ${i} ] ; then - rm -fdr ${i} - fi - - # clean up pid file - rm -f /tmp/${i}.pid.rc - exit 1 - fi - fi - - if [ "x${shas[$i]}" != x ] ; then - echo - echo "Update ${i} to commit ID: ${shas[$i]}" - echo - - cd ${i} - git checkout -B ${branches[$i]} ${shas[$i]} - cd - > /dev/null - fi -done diff --git a/closed/get_openssl_source.sh b/closed/get_openssl_source.sh deleted file mode 100644 index 3718c748e2..0000000000 --- a/closed/get_openssl_source.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/sh - -# =========================================================================== -# (c) Copyright IBM Corp. 2018, 2023 All Rights Reserved -# =========================================================================== -# -# This code is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License version 2 only, as -# published by the Free Software Foundation. -# -# IBM designates this particular file as subject to the "Classpath" exception -# as provided by IBM in the LICENSE file that accompanied this code. -# -# This code is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# version 2 for more details (a copy is included in the LICENSE file that -# accompanied this code). -# -# You should have received a copy of the GNU General Public License version -# 2 along with this work; if not, see . -# -# =========================================================================== - -usage() { - echo "Usage: $0 [-h|--help] [--openssl-repo=] [--openssl-version=]" - echo "where:" - echo " -h|--help print this help, then exit" - echo " --openssl-repo OpenSSL repository. By default, https://github.com/openssl/openssl.git" - echo " --openssl-version OpenSSL version to download. For example, 1.1.1" - echo "" - exit 1 -} - -OPENSSL_VERSION="" -OPENSSL_URL="https://github.com/openssl/openssl.git" - -for i in "$@" -do - case $i in - -h | --help ) - usage - ;; - - --openssl-repo=* ) - OPENSSL_URL="${i#*=}" - ;; - - --openssl-version=* ) - OPENSSL_VERSION="${i#*=}" - ;; - - '--' ) # no more options - usage - ;; - - -*) # bad option - usage - ;; - - *) # bad option - usage - ;; - esac -done - -case "$OPENSSL_VERSION" in - 1.0.2* | 1.1.*) - OPENSSL_SOURCE_TAG=$(echo "OpenSSL.$OPENSSL_VERSION" | sed -e 's/\./_/g') - ;; - 3.*) - OPENSSL_SOURCE_TAG="openssl-$OPENSSL_VERSION" - ;; - *) - OPENSSL_SOURCE_TAG=$OPENSSL_VERSION - ;; -esac - -if [ -f "openssl/openssl_version.txt" ]; then - DOWNLOADED_VERSION=$(cat openssl/openssl_version.txt) - if [ $OPENSSL_SOURCE_TAG = $DOWNLOADED_VERSION ]; then - echo "" - echo "OpenSSL version $OPENSSL_VERSION is already downloaded" - exit 0 - else - echo "" - echo "Cleaning up OpenSSL source code as version already downloaded is different" - rm -rf openssl - fi -fi - -echo "" -echo "Cloning OpenSSL version $OPENSSL_VERSION from $OPENSSL_URL" -git clone --depth=1 -b $OPENSSL_SOURCE_TAG $OPENSSL_URL - -echo $OPENSSL_SOURCE_TAG > openssl/openssl_version.txt diff --git a/get_source.sh b/get_source.sh index aa2daba874..3d39994548 100644 --- a/get_source.sh +++ b/get_source.sh @@ -2,7 +2,6 @@ # =========================================================================== # (c) Copyright IBM Corp. 2017, 2024 All Rights Reserved # =========================================================================== -# # This code is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License version 2 only, as # published by the Free Software Foundation. @@ -18,99 +17,281 @@ # # You should have received a copy of the GNU General Public License version # 2 along with this work; if not, see . -# # =========================================================================== -# # exit immediately if any unexpected error occurs set -e +openssloptions="" +DOWNLOAD_OPENSSL=false + +# These maps are keyed by the prefix of option names (e.g. openj9, omr). +declare -A source_branch # branch or tag +declare -A source_folder # local working directory +declare -A source_options # extra clone options +declare -A source_ref # local reference repository +declare -A source_sha # commit SHA +declare -A source_url # URL + +# Print a message to stderr and exit. +# +fail() { + echo "$*" >&2 + exit 1 +} + +# Define a single possible source repository. +# $1 - The local working directory. The value converted to lowercase is used +# to form the related option names. For example, the key "OpenJCEPlus" +# means options "-openjceplus-repo=", "-openjceplus-branch=", +# "-openjceplus-sha=", and "-openjceplus-reference=" will be recognized. +# $2 - The source URL; default empty. +# $3 - The source branch or tag; default empty. +# $4 - Extra options for git clone; default empty. +# $5 - The local reference repository; default empty. +# +add_source() { + if [ $# -lt 1 ] ; then + fail "add_source requires at least one argument" + fi + + local key="${1,,}" + local folder="$1" + local url="${2:-}" + local branch="${3:-}" + local options="${4:-}" + local sha="${5:-}" + + source_folder[$key]="$folder" + source_url[$key]="$url" + source_branch[$key]="$branch" + source_options[$key]="$options" + source_sha[$key]="$sha" + source_ref[$key]="" +} + +# Configure the known source repositories. +# +configure_defaults() { + local current_branch="$(git rev-parse --abbrev-ref HEAD)" + local openj9_branch=master + local omr_branch=openj9 + + # If this repository is on a release branch, use the same branch names + # for OpenJ9 and OMR. + if [[ "$current_branch" =~ (ibm-)?(v[0-9]+\.[0-9]+(\.[0-9]+)?-release) ]] ; then + openj9_branch="${BASH_REMATCH[2]}" + omr_branch="${BASH_REMATCH[2]}" + fi + + # folder URL branch options + # ------ --- ------ ------- + add_source openj9 https://github.com/eclipse-openj9/openj9.git $openj9_branch + add_source omr https://github.com/eclipse-openj9/openj9-omr.git $omr_branch + + add_source OpenJCEPlus https://github.com/ibmruntimes/OpenJCEPlus.git + add_source openssl https://github.com/openssl/openssl.git "" "--depth=1" +} + +# Show the usage of a single option. +# $1 - The option name. +# $2 - The option description. +# $3 - The default value of the option; default none. +# +show_option() { + local option="$1" + local description="$2" + local default="${3:+ [$3]}" + + printf " %-22s %s%s\n" "$option" "$description" "$default" +} + +# Print help for this script and exit. +# usage() { - echo "Usage: $0 [-h|--help] [... other j9 options] [-parallel=] [--openssl-version=]" - echo "where:" - echo " -h|--help print this help, then exit" - echo " " - echo " -openj9-repo the OpenJ9 repository url: https://github.com/eclipse-openj9/openj9.git" - echo " or git@github.com:/openj9.git" - echo " -openj9-branch the OpenJ9 git branch: master" - echo " -openj9-sha a commit SHA for the OpenJ9 repository" - echo " -openj9-reference a local repo to use as a clone reference" - echo " -omr-repo the OpenJ9/omr repository url: https://github.com/eclipse-openj9/openj9-omr.git" - echo " or git@github.com:/openj9-omr.git" - echo " -omr-branch the OpenJ9/omr git branch: openj9" - echo " -omr-sha a commit SHA for the omr repository" - echo " -omr-reference a local repo to use as a clone reference" - echo " -openjceplus-repo the OpenJCEPlus repository url" - echo " -openjceplus-branch the OpenJCEPlus git branch" - echo " -openjceplus-sha a commit SHA for the OpenJCEPlus repository" - echo " -openjceplus-reference a local repo to use as a clone reference" - echo " -gskit-bin the GSKit binary url" - echo " -gskit-sdk-bin the GSKIT SDK binary url" - echo " -gskit-credential the credential for downloading the GSKit and GSKit SDK" - echo " -parallel (boolean) if 'true' then the clone j9 repository commands run in parallel, default is false" - echo " --openssl-repo Specify the OpenSSL repository to download from" - echo " --openssl-version Specify the version of OpenSSL source to download" - echo "" + local key + + echo "Usage: $0 [options ...]" + echo " where:" + show_option "-h|--help" "print this help, then exit" + + for key in "${!source_folder[@]}" ; do + local folder="${source_folder[$key]}" + + show_option "-$key-repo" "the repository URL for $folder" "${source_url[$key]}" + show_option "-$key-branch" "the git branch for $folder" "${source_branch[$key]}" + show_option "-$key-sha" "a commit SHA for the $folder repository" "${source_sha[$key]}" + show_option "-$key-reference" "a local repository to use as a reference" "${source_ref[$key]}" + done + + show_option "-gskit-bin" "the GSKit binary URL" + show_option "-gskit-sdk-bin" "the GSKIT SDK binary URL" + show_option "-gskit-credential" "the credential for downloading the GSKit and GSKit SDK" + show_option "--openssl-repo" "equivalent to -openssl-repo" + show_option "--openssl-version" "specify the version of OpenSSL source to download" + show_option "-parallel" "(ignored)" exit 1 } -j9options="" -openssloptions="" -DOWNLOAD_OPENSSL=false +# Process and validate the command-line arguments. +# +process_options() { + local arg="" + local version="" + + for arg in "$@" ; do + # temporarily handle openssl options that don't follow the general pattern + case "$arg" in + --openssl-repo=*) + # remove leading '-' + arg="${arg/--/-}" + ;; + --openssl-version=*) + # map to -openssl-branch + version="${arg#*=}" + case "$version" in + 1.0.2* | 1.1.*) + version="OpenSSL_${version//./_}" + ;; + 3.*) + version="openssl-$version" + ;; + *) + ;; + esac + arg=-openssl-branch=$version + ;; + *) + ;; + esac + + if [[ "$arg" =~ -([A-Za-z0-9]+)-(branch|reference|repo|sha)=.* ]] ; then + local key="${BASH_REMATCH[1]}" + if [ -z "${source_folder[${key}]}" ] ; then + fail "Unknown option: $arg" + fi + + local value="${arg#*=}" + case "${BASH_REMATCH[2]}" in + branch) source_branch[$key]="$value" ;; + reference) source_ref[$key]="$value" ;; + repo) source_url[$key]="$value" ;; + sha) source_sha[$key]="$value" ;; + esac + else + case "$arg" in + -gskit-bin=*) + gskit_bin="${arg#*=}" + ;; + -gskit-sdk-bin=*) + gskit_sdk_bin="${arg#*=}" + ;; + -gskit-credential=*) + gskit_credential="${arg#*=}" + ;; + -h | --help) + usage + ;; + --) + # end of options + break + ;; + *) + # bad option + usage + ;; + esac + fi + done +} + +# The main body which does the actual cloning or updating of sources. +# +clone_or_update_repos() { + local key + + for key in "${!source_folder[@]}" ; do + local url="${source_url[$key]}" + local branch="${source_branch[$key]}" + + if [ -n "$url" ] && [ -n "$branch" ] ; then + local folder="${source_folder[$key]}" + local sha="${source_sha[$key]}" + local reference="${source_ref[$key]}" + + if [ -d "$folder" ] ; then + echo + echo "Update $folder source" + echo + + cd "$folder" + git pull --rebase origin "$branch" + + if [ -f .gitmodules ] ; then + git pull --rebase --recurse-submodules=yes + git submodule update --rebase --recursive + fi + cd - > /dev/null + else + echo + echo "Clone repository: $folder" + echo + + git clone \ + ${reference:+--reference "$reference"} \ + ${source_options[$key]} \ + -b "$branch" \ + "$url" \ + "$folder" + fi + + if [ -n "$sha" ] ; then + echo + echo "Update $folder to commit ID: $sha" + echo + + cd $folder + git checkout -B "$branch" "$sha" + cd - > /dev/null + fi + fi + done +} + +# If OpenJCEPlus is present and the necessary URLs were provided, download OCK +# binaries and create Java module folder. +# +maybe_get_gskit() { + if [ -d OpenJCEPlus ] && [ ! -d OpenJCEPlus/ock ] && [ -n "$gskit_bin" ] && [ -n "$gskit_sdk_bin" ] ; then + echo + echo "Downloading OCK binaries for OpenJCEPlus" + echo + + cd OpenJCEPlus + mkdir -p ock/jgsk_sdk/lib64 + + if [ -n "$gskit_credential" ] ; then + curl -u "$gskit_credential" "$gskit_bin" > ock/jgsk_crypto.tar + curl -u "$gskit_credential" "$gskit_sdk_bin" > ock/jgsk_crypto_sdk.tar + else + echo "GSKit binaries are needed for compiling OpenJCEPlus." + fail "Please specify the -gskit-credential option." + fi + + tar -xf ock/jgsk_crypto_sdk.tar -C ock + tar -xf ock/jgsk_crypto.tar -C ock/jgsk_sdk/lib64 + + # Create OpenJCEPlus Java module folder. + mkdir -p src/main/openjceplus/share/classes + cp -r src/main/java/* src/main/openjceplus/share/classes/ + + cd - > /dev/null + fi +} + +# =========================================================================== -for i in "$@" ; do - case $i in - -h | --help ) - usage - ;; - - -gskit-bin=* \ - | -gskit-credential=* \ - | -gskit-sdk-bin=* \ - | -omr-branch=* \ - | -omr-reference=* \ - | -omr-repo=* \ - | -omr-sha=* \ - | -openj9-branch=* \ - | -openj9-reference=* \ - | -openj9-repo=* \ - | -openj9-sha=* \ - | -openjceplus-branch=* \ - | -openjceplus-reference=* \ - | -openjceplus-repo=* \ - | -openjceplus-sha=* \ - | -parallel=* \ - ) - j9options="${j9options} ${i}" - ;; - - --openssl-repo=* ) - openssloptions="${openssloptions} ${i}" - ;; - - --openssl-version=* ) - DOWNLOAD_OPENSSL=true - openssloptions="${openssloptions} ${i}" - ;; - - '--' ) # no more options - break - ;; - - -*) # bad option - usage - ;; - - *) # bad option - usage - ;; - esac -done - -# Get clones of OpenJ9 absent repositories -bash closed/get_j9_source.sh ${j9options} - -# Download source of OpenSSL if asked -if $DOWNLOAD_OPENSSL; then - bash closed/get_openssl_source.sh ${openssloptions} -fi +configure_defaults +process_options "$@" +clone_or_update_repos +maybe_get_gskit