add download and sbom verification scripts (#3516)

* add download and sbom verification scripts Signed-off-by: Stewart X Addison <sxa@redhat.com> * Linter updates Signed-off-by: Stewart X Addison <sxa@redhat.com> * Fix VS version for JDK20+ Signed-off-by: Stewart X Addison <sxa@redhat.com> * Shut up linter because I don't agree with it Signed-off-by: Stewart X Addison <sxa@redhat.com> --------- Signed-off-by: Stewart X Addison <sxa@redhat.com>
adoptium · Nov 1, 2023 · 2643998 · 2643998
1 parent 30d1211
commit 2643998
Show file tree

Hide file tree

Showing 2 changed files with 295 additions and 0 deletions.
diff --git a/tooling/release_download_test.sh b/tooling/release_download_test.sh
@@ -0,0 +1,171 @@
+#!/bin/sh
+#
+# Adoptium download and SBOM validation utility
+# Takes a tagged build as a parameter and downloads it from the
+# GitHub temurinXX-binaries and runs validation checks on it
+#
+# Exit codes:
+# 1 - Something fundamentally wrong before we could check anything
+# 2 - GPG signature verification failed
+# 3 - SHA checksum failed
+# 4 - aarch64 detected GCC/GLIBC version not as expected
+# 5 - CylconeDX validation checks failed
+# 6 - SBOM contents did not meet expectations
+# Note that if there are multiple failures the highest will be the exit code
+# If there is a non-zero exit code check the output for "ERROR:"
+# 
+# For future enhancement ideas, see https://github.com/adoptium/temurin-build/issues/3506#issuecomment-1783237963
+#
+STARTDIR="$PWD"
+TAG=${1:-$TAG}
+[ -z "$TAG" ] && echo "Usage: $0 TAG" && exit 1
+[ "$(uname -m)" != "aarch64" ] && echo "This script is hard coded to be run on Linux/aarch64 - aborting" && exit 1
+
+[ "$VERBOSE" = "true" ] && set +x
+if echo "$TAG" | grep jdk8u; then
+ MAJOR_VERSION=8
+elif echo "$TAG" | grep ^jdk-; then
+ MAJOR_VERSION=$(echo "$TAG" | cut -d- -f2 | cut -d. -f1 | cut -d\+ -f1)
+else
+ # Probably a beta with the tag starting jdkXXu
+ MAJOR_VERSION=$(echo "$TAG" | cut -d- -f1 | tr -d jdku)
+fi
+
+echo "$(date +%T) : IVT : I will be checking https://github.com/adoptium/temurin${MAJOR_VERSION}-binaries/releases/tag/$TAG"
+if [ -z "${MAJOR_VERSION}" ] || [ -z "${TAG}" ]; then
+ echo "MAJOR_VERSION or TAG undefined - aborting"
+ exit 1
+fi
+
+if ! curl -sS "https://api.github.com/repos/adoptium/temurin${MAJOR_VERSION}-binaries/releases" > "$WORKSPACE/jdk${MAJOR_VERSION}.txt"; then
+ echo "github API call failed - aborting"
+ exit 2
+fi
+
+[ "$VERBOSE" = "true" ] && echo "$(date +%T) : IVT: Downloading files from release repository"
+
+# Leaving this "if/fi" commented out as it can be useful if doing standalone
+# testing to avoid having to re-download. May be removed in future
+#if [ ! -r "staging/$TAG/OpenJDK21U-debugimage_aarch64_mac_hotspot_ea_21-0-35.tar.gz.json" ]; then
+ rm -rf staging
+ mkdir staging "staging/$TAG"
+ cd "staging/$TAG" || exit 3
+ # Early access versions are currently in a different format
+ if echo "$TAG" | grep ea-beta; then
+ FILTER="ea_${MAJOR_VERSION}"
+ else
+ FILTER=$(echo "$TAG" | sed 's/+/%2B/g')
+ fi
+ # Parse the releases list for the one we want and download everything in it
+ # shellcheck disable=SC2013
+ for URL in $(grep "$FILTER" "$WORKSPACE/jdk${MAJOR_VERSION}.txt" | awk -F'"' '/browser_download_url/{print$4}'); do
+ # shellcheck disable=SC2046
+ [ "$VERBOSE" = "true" ] && echo Downloading $(basename "$URL")
+ curl -LORsS "$URL"
+ done
+
+ ls -l "$WORKSPACE/staging/$TAG"
+#fi
+
+echo "$(date +%T) : IVT : Import Temurin GPG key" 
+cd "$WORKSPACE/staging/$TAG" || exit 3
+umask 022
+export GPGID=3B04D753C9050D9A5D343F39843C48A565F8F04B
+export GNUPGHOME="$WORKSPACE/.gpg-temp"
+rm -rf "$GNUPGHOME"
+mkdir -p "$GNUPGHOME" && chmod og-rwx "$GNUPGHOME"
+gpg -q --keyserver keyserver.ubuntu.com --recv-keys "${GPGID}" || exit 1
+# shellcheck disable=SC3037
+/bin/echo -e "5\ny\nq\n" | gpg -q --batch --command-fd 0 --expert --edit-key "${GPGID}" trust || exit 1
+
+RC=0
+
+echo "$(date +%T): IVT : Testing GPG and sha256 signatures of all tar.gz/json files"
+# Note: This will run into problems if there are no tar.gz files
+# e.g. if only windows has been uploaded to the release
+for A in OpenJDK*.tar.gz OpenJDK*.zip *.msi *.pkg *sbom*[0-9].json; do
+ if ! gpg -q --verify "${A}.sig" "$A"; then
+ echo "ERROR: GPG signature verification failed for ${A}"
+ RC=2
+ fi
+ if ! grep sbom "$A" > /dev/null; then # SBOMs don't have sha256.txt files
+ if ! sha256sum -c "${A}.sha256.txt"; then
+ echo "ERROR: SHA256 signature for ${A} is not valid"
+ RC=3
+ fi
+ fi
+done
+
+echo "$(date +%T): IVT : Verifying that all tarballs are a valid format and counting files within them"
+
+for A in OpenJDK*.tar.gz; do
+ if ! tar tfz "$A" > /dev/null; then
+ echo "ERROR: Failed to verify that $A can be extracted"
+ RC=4
+ fi
+ # NOTE: 40 chosen because the static-libs is in the 40s - maybe switch for different tarballs in the future?
+ if [ "$(tar tfz "$A" | wc -l)" -lt 40 ]; then
+ echo "ERROR: Less than 40 files in $A - that does not seem correct"
+ RC=4
+ fi
+done
+for A in OpenJDK*.zip; do 
+ if ! unzip -t "$A" > /dev/null; then
+ echo "ERROR: Failed to verify that $A can be extracted"
+ RC=4
+ fi
+ if [ "$(unzip -l "$A" | wc -l)" -lt 44 ]; then
+ echo "ERROR: Less than 40 files in $A - that does not seem correct"
+ RC=4
+ fi
+done
+
+echo "$(date +%T): IVT : Running java -version and checking glibc version on Linux/aarch64 tarballs"
+
+rm -rf tarballtest && mkdir tarballtest
+tar -C tarballtest --strip-components=1 -xzpf OpenJDK*-jre_aarch64_linux_hotspot_*.tar.gz && tarballtest/bin/java -version || exit 3
+rm -r tarballtest && mkdir tarballtest
+tar -C tarballtest --strip-components=1 -xzpf OpenJDK*-jdk_aarch64_linux_hotspot_*.tar.gz && tarballtest/bin/java -version || exit 3
+
+strings tarballtest/bin/java | grep ^GLIBC
+if ! strings tarballtest/bin/java | grep ^GLIBC_2.17 > /dev/null; then
+ echo "ERROR: GLIBC version detected in the JDK java executable is not the expected 2.17"
+ RC=4
+fi
+
+# shellcheck disable=SC2166
+[ "${MAJOR_VERSION}" = "8" -o "${MAJOR_VERSION}" = "11" ] && EXPECTED_GCC=7.5
+[ "${MAJOR_VERSION}" = "17" ] && EXPECTED_GCC=10.3
+[ "${MAJOR_VERSION}" -ge 20 ] && EXPECTED_GCC=11.2
+if ! strings tarballtest/bin/java | grep "^GCC:.*${EXPECTED_GCC}"; then
+ echo "ERROR: GCC version detected in the JDK java executable is not the expected $EXPECTED_GCC"
+ RC=4
+fi
+rm -r tarballtest
+
+# Also verify SBOM contant matches the above
+echo "$(date +%T): IVT : Downloading CycloneDX validation tool"
+[ ! -r "cyclonedx-linux-arm64" ] && curl -LOsS https://github.com/CycloneDX/cyclonedx-cli/releases/download/v0.25.0/cyclonedx-linux-arm64
+if [ "$(sha256sum cyclonedx-linux-arm64 | cut -d' ' -f1)" != "eaac307ca4d7f3ee2a10e5fe898d7ff16c4b8054b10cc210abe6f6d703d17852" ]; then
+ echo "ERROR: Cannot verify checksum of cycloneDX CLI binary"
+ exit 1
+fi
+chmod 700 cyclonedx-linux-*
+cd "$STARTDIR" || exit 1
+
+# shellcheck disable=SC2010
+for SBOM in $(ls -1 staging/"$TAG"/OpenJDK*-sbom*json | grep -v metadata); do
+ echo "$(date +%T) : IVT : Validating $SBOM ..."
+ if ! staging/"$TAG"/cyclonedx-linux-arm64 validate --input-file "$SBOM"; then
+ echo "ERROR: Failed CycloneDX validation check"
+ RC=5
+ fi
+ # shellcheck disable=SC2086
+ if ! bash "$(dirname $0)/validateSBOMcontent.sh" "$SBOM" "$MAJOR_VERSION" "$TAG"; then
+ echo "ERROR: Failed checks on $SBOM"
+ RC=6
+ fi
+done
+
+echo "$(date +%T) : IVT : Finished. Return code = $RC"
+exit $RC
diff --git a/tooling/validateSBOMcontent.sh b/tooling/validateSBOMcontent.sh
@@ -0,0 +1,124 @@
+#!/bin/sh
+[ "$VERBOSE" = "true" ] && set -x
+if [ $# -lt 3 ]; then
+ echo "Usage: $0 file.json majorversion fullversion"
+ echo "e.g. $0 OpenJDK17_sbom.json 17 17.0.8"
+ exit 1
+fi
+SBOMFILE="$1"
+MAJORVERSION="$2"
+#FULLVERSION="$3"
+
+GLIBC=$(jq '.metadata.tools[] | select(.name|test("GLIBC")) | .version' "$1" | tr -d \")
+GCC=$(jq '.metadata.tools[] | select(.name|test("GCC")) | .version' "$1" | tr -d \")
+BOOTJDK=$(jq '.metadata.tools[] | select(.name|test("BOOTJDK")) | .version' "$1" | tr -d \")
+ALSA=$(jq '.metadata.tools[] | select(.name|test("ALSA")) | .version' "$1" | tr -d \" | sed -e 's/^.*alsa-lib-//' -e 's/\.tar.bz2//')
+FREETYPE=$(jq '.metadata.tools[] | select(.name|test("FreeType")) | .version' "$1" | tr -d \")
+FREEMARKER=$(jq '.metadata.tools[] | select(.name|test("FreeMarker")) | .version' "$1" | tr -d \")
+COMPILER=$(jq '.components[0].properties[] | select(.name|test("Build Tools Summary")).value' "$SBOMFILE" | sed -e 's/^.*Toolchain: //g' -e 's/\ *\*.*//g')
+
+EXPECTED_COMPILER="gcc (GNU Compiler Collection)"
+EXPECTED_GLIBC=""
+EXPECTED_GCC=""
+# [ "${MAJORVERSION}" = "17" ] && EXPECTED_GCC=10.3.0
+EXPECTED_ALSA=N.A
+#EXPECTED_FREETYPE=N.A # https://github.com/adoptium/temurin-build/issues/3493
+#EXPECTED_FREETYPE=https://github.com/freetype/freetype/commit/86bc8a95056c97a810986434a3f268cbe67f2902
+if echo "$SBOMFILE" | grep _solaris_; then
+ #EXPECTED_FREETYPE=N.A
+ EXPECTED_COMPILER="solstudio (Oracle Solaris Studio)"
+elif echo "$SBOMFILE" | grep _aix_; then
+ EXPECTED_COMPILER="xlc (IBM XL C/C++)"
+elif echo "$SBOMFILE" | grep _alpine-linux_ > /dev/null; then
+ #EXPECTED_FREETYPE=N.A
+ EXPECTED_ALSA=1.1.6
+ EXPECTED_GCC=10.3.1
+elif echo "$SBOMFILE" | grep _linux_; then
+ if [ "$MAJORVERSION" -lt 20 ] && echo "$SBOMFILE" | grep x64 > /dev/null; then
+ EXPECTED_GLIBC=2.12
+ elif echo "$SBOMFILE" | grep _arm_ > /dev/null; then
+ EXPECTED_GLIBC=2.23
+ else
+ EXPECTED_GLIBC=2.17
+ fi
+ [ "${MAJORVERSION}" = "8" ] && EXPECTED_GCC=7.5.0
+ [ "${MAJORVERSION}" = "11" ] && EXPECTED_GCC=7.5.0
+ [ "${MAJORVERSION}" = "17" ] && EXPECTED_GCC=10.3.0
+ [ "${MAJORVERSION}" -ge 20 ] && EXPECTED_GCC=11.2.0
+ EXPECTED_ALSA=1.1.6
+ #EXPECTED_FREETYPE=N.A
+#elif echo $SBOMFILE | grep _mac_; then
+# EXPECTED_COMPILER="clang (clang/LLVM from Xcode 10.3)"
+elif echo "$SBOMFILE" | grep _x64_windows_; then
+ if [ "${MAJORVERSION}" = "8" ]; then
+ EXPECTED_COMPILER="microsoft (Microsoft Visual Studio 2017 - CURRENTLY NOT WORKING)"
+ #EXPECTED_FREETYPE="https://github.com/freetype/freetype/commit/ec8853cd18e1a0c275372769bdad37a79550ed66"
+ elif [ "${MAJORVERSION}" -ge 20 ]; then
+ EXPECTED_COMPILER="microsoft (Microsoft Visual Studio 2022)"
+ fi
+elif echo "$SBOMFILE" | grep _x86-32_windows_; then
+ if [ "${MAJORVERSION}" = "8" ]; then
+ EXPECTED_COMPILER="microsoft (Microsoft Visual Studio 2013)"
+ #EXPECTED_FREETYPE="https://github.com/freetype/freetype/commit/ec8853cd18e1a0c275372769bdad37a79550ed66"
+ elif [ "${MAJORVERSION}" = "11" ]; then
+ EXPECTED_COMPILER="microsoft (Microsoft Visual Studio 2017)"
+ else
+ EXPECTED_COMPILER="microsoft (Microsoft Visual Studio 2019)"
+ fi
+elif echo "$SBOMFILE" | grep _mac_; then
+ # NOTE: mac/x64 native builds >=11 were using "clang (clang/LLVM from Xcode 10.3)"
+ EXPECTED_COMPILER="clang (clang/LLVM from Xcode 12.4)"
+ # shellcheck disable=SC2166
+ if [ "${MAJORVERSION}" = "8" -o "${MAJORVERSION}" = "11" ] && echo "$SBOMFILE" | grep _x64_; then
+ EXPECTED_COMPILER="clang (clang/LLVM)"
+# EXPECTED_FREETYPE="https://github.com/freetype/freetype/commit/ec8853cd18e1a0c275372769bdad37a79550ed66"
+ fi
+fi
+
+EXPECTED_FREEMARKER=N.A
+RC=0
+if echo "$SBOMFILE" | grep 'linux_'; then
+ [ "${GLIBC}" != "$EXPECTED_GLIBC" ] && echo "ERROR: GLIBC version not ${EXPECTED_GLIBC} (SBOM has ${GLIBC})" && RC=1
+ [ "${GCC}" != "$EXPECTED_GCC" ] && echo "ERROR: GCC version not ${EXPECTED_GCC} (SBOM has ${GCC})" && RC=1
+fi
+echo "BOOTJDK is ${BOOTJDK}"
+[ "${COMPILER}" != "$EXPECTED_COMPILER" ] && echo "ERROR: Compiler version not ${EXPECTED_COMPILER} (SBOM has ${COMPILER})" && RC=1
+[ "${ALSA}" != "$EXPECTED_ALSA" ] && echo "ERROR: ALSA version not ${EXPECTED_ALSA} (SBOM has ${ALSA})" && RC=1
+# Freetype versions are inconsistent at present - see build#3484
+#[ "${FREETYPE}" != "$EXPECTED_FREETYPE" ] && echo "ERROR: FreeType version not ${EXPECTED_FREETYPE} (SBOM has ${FREETYPE})" && RC=1
+
+# shellcheck disable=SC2086
+[ -n "$(echo $FREETYPE | tr -d '[0-9]\.')" ] && echo "ERROR: FreeType version not a valid number (SBOM has ${FREETYPE})" && RC=1
+echo "FREETYPE is ${FREETYPE}"
+[ "${FREEMARKER}" != "$EXPECTED_FREEMARKER" ] && echo "ERROR: Freemarker version not ${EXPECTED_FREEMARKER} (SBOM has ${FREEMARKER})" && RC=1
+# shellcheck disable=SC3037
+echo -n "Checking for JDK source SHA validity: "
+GITSHA=$(jq '.components[].properties[] | select(.name|test("OpenJDK Source Commit")) | .value' "$1" | tr -d \")
+GITREPO=$(echo "$GITSHA" | cut -d/ -f1-5)
+GITSHA=$( echo "$GITSHA" | cut -d/ -f7)
+if ! git ls-remote "${GITREPO}" | grep "${GITSHA}"; then
+ echo "ERROR: git sha of source repo not found"
+ RC=1
+fi
+
+# shellcheck disable=SC3037
+echo -n "Checking for temurin-build SHA validity: "
+GITSHA=$(jq '.components[].properties[] | select(.name|test("Temurin Build Ref")) | .value' "$1" | tr -d \")
+GITREPO=$(echo "$GITSHA" | cut -d/ -f1-5)
+GITSHA=$(echo "$GITSHA" | cut -d/ -f7)
+echo "Checking for temurin-build SHA $GITSHA"
+if ! git ls-remote "${GITREPO}" | grep "${GITSHA}"; then
+ echo "WARNING: temurin-build SHA check failed. This can happen if it was not a tagged level"
+ if echo "$1" | grep '[0-9][0-9]-[0-9][0-9]-[0-9][0-9]-[0-9][0-9]' 2>/dev/null; then
+ echo "Ignoring return code as filename looks like a nightly"
+ else
+ echo "This can also happen with a branch being used and not a tag as we do for GAs so not failing"kd
+ echo "Note: As this is a warning message this will not cause a non-zero return code by itself"
+ # RC=1
+ fi
+fi
+
+if [ "$RC" != "0" ]; then
+ echo "ERROR: Overall return code from validateSBOMcontent.sh is non-zero - something failed validation"
+fi
+exit $RC