From a0dfec72088d5dc0d96b82c426c45761a8e2f739 Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Fri, 27 Oct 2023 17:10:45 +0100 Subject: [PATCH 1/4] add download and sbom verification scripts Signed-off-by: Stewart X Addison --- tooling/release_download_test.sh | 166 +++++++++++++++++++++++++++++++ tooling/validateSBOMcontent.sh | 121 ++++++++++++++++++++++ 2 files changed, 287 insertions(+) create mode 100755 tooling/release_download_test.sh create mode 100755 tooling/validateSBOMcontent.sh diff --git a/tooling/release_download_test.sh b/tooling/release_download_test.sh new file mode 100755 index 000000000..ca5655105 --- /dev/null +++ b/tooling/release_download_test.sh @@ -0,0 +1,166 @@ +#!/bin/sh +# +# Adoptium download and SBOM validation utility +# Takes a tagged build as a parameter and downloads it from the +# GitHub temurinXX-binaries and runs validation checks on it +# +# Exit codes: +# 1 - Something fundamentally wrong before we could check anything +# 2 - GPG signature verification failed +# 3 - SHA checksum failed +# 4 - aarch64 detected GCC/GLIBC version not as expected +# 5 - CylconeDX validation checks failed +# 6 - SBOM contents did not meet expectations +# Note that if there are multiple failures the highest will be the exit code +# If there is a non-zero exit code check the output for "ERROR:" +# +# For future enhancement ideas, see https://github.com/adoptium/temurin-build/issues/3506#issuecomment-1783237963 +# +STARTDIR="$PWD" +TAG=${1:-$TAG} +[ -z "$TAG" ] && echo "Usage: $0 TAG" && exit 1 +[ "$(uname -m)" != "aarch64" ] && echo "This script is hard coded to be run on Linux/aarch64 - aborting" && exit 1 + +[ "$VERBOSE" = "true" ] && set +x +if echo "$TAG" | grep jdk8u; then + MAJOR_VERSION=8 +elif echo "$TAG" | grep ^jdk-; then + MAJOR_VERSION=$(echo $TAG | cut -d- -f2 | cut -d. -f1 | cut -d\+ -f1) +else + # Probably a beta with the tag starting jdkXXu + MAJOR_VERSION=$(echo $TAG | cut -d- -f1 | tr -d jdku) +fi + +echo "$(date +%T) : IVT : I will be checking https://github.com/adoptium/temurin${MAJOR_VERSION}-binaries/releases/tag/$TAG" +if [ -z "${MAJOR_VERSION}" -o -z "${TAG}" ]; then + echo "MAJOR_VERSION or TAG undefined - aborting" + exit 1 +fi + +curl -sS https://api.github.com/repos/adoptium/temurin${MAJOR_VERSION}-binaries/releases > "$WORKSPACE/jdk${MAJOR_VERSION}.txt" +if [ $? -ne 0 ]; then + echo "github API call failed - aborting" + exit 2 +fi + +[ "$VERBOSE" = "tru" ] && echo "$(date +%T) : IVT: Downloading files from release repository" + +# Leaving this "if/fi" commented out as it can be useful if doing standalone +# testing to avoid having to re-download. May be removed in future +#if [ ! -r "staging/$TAG/OpenJDK21U-debugimage_aarch64_mac_hotspot_ea_21-0-35.tar.gz.json" ]; then + rm -rf staging + mkdir staging "staging/$TAG" + cd "staging/$TAG" || exit 3 + # Early access versions are currently in a different format + if echo $TAG | grep ea-beta; then + FILTER="ea_${MAJOR_VERSION}" + else + FILTER=$(echo $TAG | sed 's/+/%2B/g') + fi + # Parse the releases list for the one we want and download everything in it + for URL in $(cat "$WORKSPACE/jdk${MAJOR_VERSION}.txt" | grep "$FILTER" | awk -F'"' '/browser_download_url/{print$4}'); do + [ "$VERBOSE" = "true" ] && echo "Downloading $(basename $URL)" + curl -LORsS "$URL" + done + + ls -l "$WORKSPACE/staging/$TAG" +#fi + +echo "$(date +%T) : IVT : Import Temurin GPG key" +cd "$WORKSPACE/staging/$TAG" || exit 3 +umask 022 +export GPGID=3B04D753C9050D9A5D343F39843C48A565F8F04B +export GNUPGHOME="$WORKSPACE/.gpg-temp" +rm -rf "$GNUPGHOME" +mkdir -p "$GNUPGHOME" && chmod og-rwx "$GNUPGHOME" +gpg -q --keyserver keyserver.ubuntu.com --recv-keys "${GPGID}" || exit 1 +/bin/echo -e "5\ny\nq\n" | gpg -q --batch --command-fd 0 --expert --edit-key "${GPGID}" trust || exit 1 + +RC=0 + +echo "$(date +%T): IVT : Testing GPG and sha256 signatures of all tar.gz/json files" +# Note: This will run into problems if there are no tar.gz files +# e.g. if only windows has been uploaded to the release +for A in OpenJDK*.tar.gz OpenJDK*.zip *.msi *.pkg *sbom*[0-9].json; do + if ! gpg -q --verify "${A}.sig" "$A"; then + echo "ERROR: GPG signature verification failed for ${A}" + RC=2 + fi + if ! grep sbom "$A" > /dev/null; then # SBOMs don't have sha256.txt files + if ! sha256sum -c "${A}.sha256.txt"; then + echo "ERROR: SHA256 signature for ${A} is not valid" + RC=3 + fi + fi +done + +echo "$(date +%T): IVT : Verifying that all tarballs are a valid format and counting files within them" + +for A in OpenJDK*.tar.gz; do + if ! tar tfz $A > /dev/null; then + echo "ERROR: Failed to verify that $A can be extracted" + RC=4 + fi + # NOTE: 40 chosen because the static-libs is in the 40s - maybe switch for different tarballs in the future? + if [ "$(tar tfz $A | wc -l)" -lt 40 ]; then + echo "ERROR: Less than 40 files in $A - that does not seem correct" + RC=4 + fi +done +for A in OpenJDK*.zip; do + if ! unzip -t $A > /dev/null; then + echo "ERROR: Failed to verify that $A can be extracted" + RC=4 + fi + if [ "$(unzip -l $A | wc -l)" -lt 44 ]; then + echo "ERROR: Less than 40 files in $A - that does not seem correct" + RC=4 + fi +done + +echo "$(date +%T): IVT : Running java -version and checking glibc version on Linux/aarch64 tarballs" + +rm -rf tarballtest && mkdir tarballtest +tar -C tarballtest --strip-components=1 -xzpf OpenJDK*-jre_aarch64_linux_hotspot_*.tar.gz && tarballtest/bin/java -version || exit 3 +rm -r tarballtest && mkdir tarballtest +tar -C tarballtest --strip-components=1 -xzpf OpenJDK*-jdk_aarch64_linux_hotspot_*.tar.gz && tarballtest/bin/java -version || exit 3 + +strings tarballtest/bin/java | grep ^GLIBC +if ! strings tarballtest/bin/java | grep ^GLIBC_2.17 > /dev/null; then + echo "ERROR: GLIBC version detected in the JDK java executable is not the expected 2.17" + RC=4 +fi + +[ "${MAJOR_VERSION}" = "8" -o "${MAJOR_VERSION}" = "11" ] && EXPECTED_GCC=7.5 +[ "${MAJOR_VERSION}" = "17" ] && EXPECTED_GCC=10.3 +[ "${MAJOR_VERSION}" -ge 20 ] && EXPECTED_GCC=11.2 +if ! strings tarballtest/bin/java | grep "^GCC:.*${EXPECTED_GCC}"; then + echo "ERROR: GCC version detected in the JDK java executable is not the expected $EXPECTED_GCC" + RC=4 +fi +rm -r tarballtest + +# Also verify SBOM contant matches the above +echo "$(date +%T): IVT : Downloading CycloneDX validation tool" +[ ! -r "cyclonedx-linux-arm64" ] && curl -LOsS https://github.com/CycloneDX/cyclonedx-cli/releases/download/v0.25.0/cyclonedx-linux-arm64 +if [ "$(sha256sum cyclonedx-linux-arm64 | cut -d' ' -f1)" != "eaac307ca4d7f3ee2a10e5fe898d7ff16c4b8054b10cc210abe6f6d703d17852" ]; then + echo "ERROR: Cannot verify checksum of cycloneDX CLI binary" + exit 1 +fi +chmod 700 cyclonedx-linux-* +cd "$STARTDIR" + +for SBOM in $(ls -1 staging/$TAG/OpenJDK*-sbom*json | grep -v metadata); do + echo "$(date +%T) : IVT : Validating $SBOM ..." + if ! staging/$TAG/cyclonedx-linux-arm64 validate --input-file "$SBOM"; then + echo "ERROR: Failed CycloneDX validation check" + RC=5 + fi + if ! bash $(dirname $0)/validateSBOMcontent.sh "$SBOM" $MAJOR_VERSION $TAG; then + echo "ERROR: Failed checks on $SBOM" + RC=6 + fi +done + +echo "$(date +%T) : IVT : Finished. Return code = $RC" +exit $RC diff --git a/tooling/validateSBOMcontent.sh b/tooling/validateSBOMcontent.sh new file mode 100755 index 000000000..01fd3a150 --- /dev/null +++ b/tooling/validateSBOMcontent.sh @@ -0,0 +1,121 @@ +#!/bin/sh +[ "$VERBOSE" = "true" ] && set -x +if [ $# -lt 3 ]; then + echo "Usage: $0 file.json majorversion fullversion" + echo "e.g. $0 OpenJDK17_sbom.json 17 17.0.8" + exit 1 +fi +SBOMFILE="$1" +MAJORVERSION="$2" +FULLVERSION="$3" + +GLIBC=$(jq '.metadata.tools[] | select(.name|test("GLIBC")) | .version' $1 | tr -d \") +GCC=$(jq '.metadata.tools[] | select(.name|test("GCC")) | .version' $1 | tr -d \") + +BOOTJDK=$(jq '.metadata.tools[] | select(.name|test("BOOTJDK")) | .version' $1 | tr -d \") +ALSA=$(jq '.metadata.tools[] | select(.name|test("ALSA")) | .version' $1 | tr -d \" | sed -e 's/^.*alsa-lib-//' -e 's/\.tar.bz2//') +FREETYPE=$(jq '.metadata.tools[] | select(.name|test("FreeType")) | .version' $1 | tr -d \") +FREEMARKER=$(jq '.metadata.tools[] | select(.name|test("FreeMarker")) | .version' $1 | tr -d \") +COMPILER=$(jq '.components[0].properties[] | select(.name|test("Build Tools Summary")).value' $SBOMFILE | sed -e 's/^.*Toolchain: //g' -e 's/\ *\*.*//g') + +EXPECTED_COMPILER="gcc (GNU Compiler Collection)" +EXPECTED_GLIBC="" +EXPECTED_GCC="" +# [ "${MAJORVERSION}" = "17" ] && EXPECTED_GCC=10.3.0 +EXPECTED_ALSA=N.A +EXPECTED_FREETYPE=N.A # https://github.com/adoptium/temurin-build/issues/3493 +#EXPECTED_FREETYPE=https://github.com/freetype/freetype/commit/86bc8a95056c97a810986434a3f268cbe67f2902 +if echo "$SBOMFILE" | grep _solaris_; then + EXPECTED_FREETYPE=N.A + EXPECTED_COMPILER="solstudio (Oracle Solaris Studio)" +elif echo "$SBOMFILE" | grep _aix_; then + EXPECTED_COMPILER="xlc (IBM XL C/C++)" +elif echo "$SBOMFILE" | grep _alpine-linux_ > /dev/null; then + EXPECTED_FREETYPE=N.A + EXPECTED_ALSA=1.1.6 + EXPECTED_GCC=10.3.1 +elif echo "$SBOMFILE" | grep _linux_; then + if [ $MAJORVERSION -lt 20 ] && echo "$SBOMFILE" | grep x64 > /dev/null; then + EXPECTED_GLIBC=2.12 + elif echo "$SBOMFILE" | grep _arm_ > /dev/null; then + EXPECTED_GLIBC=2.23 + else + EXPECTED_GLIBC=2.17 + fi + [ "${MAJORVERSION}" = "8" ] && EXPECTED_GCC=7.5.0 + [ "${MAJORVERSION}" = "11" ] && EXPECTED_GCC=7.5.0 + [ "${MAJORVERSION}" = "17" ] && EXPECTED_GCC=10.3.0 + [ "${MAJORVERSION}" -ge 20 ] && EXPECTED_GCC=11.2.0 + EXPECTED_ALSA=1.1.6 + EXPECTED_FREETYPE=N.A +#elif echo $SBOMFILE | grep _mac_; then +# EXPECTED_COMPILER="clang (clang/LLVM from Xcode 10.3)" +elif echo "$SBOMFILE" | grep _x64_windows_; then + if [ "${MAJORVERSION}" = "8" ]; then + EXPECTED_COMPILER="microsoft (Microsoft Visual Studio 2017 - CURRENTLY NOT WORKING)" + EXPECTED_FREETYPE="https://github.com/freetype/freetype/commit/ec8853cd18e1a0c275372769bdad37a79550ed66" + else + EXPECTED_COMPILER="microsoft (Microsoft Visual Studio 2019)" + fi +elif echo "$SBOMFILE" | grep _x86-32_windows_; then + if [ "${MAJORVERSION}" = "8" ]; then + EXPECTED_COMPILER="microsoft (Microsoft Visual Studio 2013)" + EXPECTED_FREETYPE="https://github.com/freetype/freetype/commit/ec8853cd18e1a0c275372769bdad37a79550ed66" + elif [ "${MAJORVERSION}" = "11" ]; then + EXPECTED_COMPILER="microsoft (Microsoft Visual Studio 2017)" + else + EXPECTED_COMPILER="microsoft (Microsoft Visual Studio 2019)" + fi +elif echo "$SBOMFILE" | grep _mac_; then + # NOTE: mac/x64 native builds >=11 were using "clang (clang/LLVM from Xcode 10.3)" + EXPECTED_COMPILER="clang (clang/LLVM from Xcode 12.4)" + if [ "${MAJORVERSION}" = "8" -o "${MAJORVERSION}" = "11" ] && echo "$SBOMFILE" | grep _x64_; then + EXPECTED_COMPILER="clang (clang/LLVM)" + EXPECTED_FREETYPE="https://github.com/freetype/freetype/commit/ec8853cd18e1a0c275372769bdad37a79550ed66" + fi +fi + +EXPECTED_FREEMARKER=N.A +RC=0 +if echo "$SBOMFILE" | egrep 'linux_'; then + [ "${GLIBC}" != "$EXPECTED_GLIBC" ] && echo "ERROR: GLIBC version not ${EXPECTED_GLIBC} (SBOM has ${GLIBC})" && RC=1 + [ "${GCC}" != "$EXPECTED_GCC" ] && echo "ERROR: GCC version not ${EXPECTED_GCC} (SBOM has ${GCC})" && RC=1 +fi +echo "BOOTJDK is ${BOOTJDK}" +[ "${COMPILER}" != "$EXPECTED_COMPILER" ] && echo "ERROR: Compiler version not ${EXPECTED_COMPILER} (SBOM has ${COMPILER})" && RC=1 +[ "${ALSA}" != "$EXPECTED_ALSA" ] && echo "ERROR: ALSA version not ${EXPECTED_ALSA} (SBOM has ${ALSA})" && RC=1 +# Freetype versions are inconsistent at present - see build#3484 +#[ "${FREETYPE}" != "$EXPECTED_FREETYPE" ] && echo "ERROR: FreeType version not ${EXPECTED_FREETYPE} (SBOM has ${FREETYPE})" && RC=1 +[ ! -z "$(echo $FREETYPE | tr -d '[0-9]\.')" ] && echo "ERROR: FreeType version not a valid number (SBOM has ${FREETYPE})" && RC=1 +echo "FREETYPE is ${FREETYPE}" +[ "${FREEMARKER}" != "$EXPECTED_FREEMARKER" ] && echo "ERROR: Freemarker version not ${EXPECTED_FREEMARKER} (SBOM has ${FREEMARKER})" && RC=1 + +echo -n "Checking for JDK source SHA validity: " +GITSHA=$(jq '.components[].properties[] | select(.name|test("OpenJDK Source Commit")) | .value' $1 | tr -d \") +GITREPO=$(echo "$GITSHA" | cut -d/ -f1-5) +GITSHA=$( echo "$GITSHA" | cut -d/ -f7) +if ! git ls-remote ${GITREPO} | grep ${GITSHA}; then + echo "ERROR: git sha of source repo not found" + RC=1 +fi + +echo -n "Checking for temurin-build SHA validity: " +GITSHA=$(jq '.components[].properties[] | select(.name|test("Temurin Build Ref")) | .value' $1 | tr -d \") +GITREPO=$(echo "$GITSHA" | cut -d/ -f1-5) +GITSHA=$(echo "$GITSHA" | cut -d/ -f7) +echo "Checking for temurin-build SHA $GITSHA" +if ! git ls-remote ${GITREPO} | grep ${GITSHA}; then + echo "WARNING: temurin-build SHA check failed. This can happen if it was not a tagged level" + if echo "$1" | grep '[0-9][0-9]-[0-9][0-9]-[0-9][0-9]-[0-9][0-9]' 2>/dev/null; then + echo "Ignoring return code as filename looks like a nightly" + else + echo "This can also happen with a branch being used and not a tag as we do for GAs so not failing"kd + echo "Note: As this is a warning message this will not cause a non-zero return code by itself" + # RC=1 + fi +fi + +if [ "$RC" != "0" ]; then + echo "ERROR: Overall return code from validateSBOMcontent.sh is non-zero - something failed validation" +fi +exit $RC From 0414cd22324e31cdc6a19b245880d61b95b994cb Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Tue, 31 Oct 2023 16:38:11 +0000 Subject: [PATCH 2/4] Linter updates Signed-off-by: Stewart X Addison --- tooling/release_download_test.sh | 32 +++++++++++----------- tooling/validateSBOMcontent.sh | 47 ++++++++++++++++---------------- 2 files changed, 40 insertions(+), 39 deletions(-) diff --git a/tooling/release_download_test.sh b/tooling/release_download_test.sh index ca5655105..9d1033933 100755 --- a/tooling/release_download_test.sh +++ b/tooling/release_download_test.sh @@ -25,10 +25,10 @@ TAG=${1:-$TAG} if echo "$TAG" | grep jdk8u; then MAJOR_VERSION=8 elif echo "$TAG" | grep ^jdk-; then - MAJOR_VERSION=$(echo $TAG | cut -d- -f2 | cut -d. -f1 | cut -d\+ -f1) + MAJOR_VERSION=$(echo "$TAG" | cut -d- -f2 | cut -d. -f1 | cut -d\+ -f1) else # Probably a beta with the tag starting jdkXXu - MAJOR_VERSION=$(echo $TAG | cut -d- -f1 | tr -d jdku) + MAJOR_VERSION=$(echo "$TAG" | cut -d- -f1 | tr -d jdku) fi echo "$(date +%T) : IVT : I will be checking https://github.com/adoptium/temurin${MAJOR_VERSION}-binaries/releases/tag/$TAG" @@ -37,13 +37,12 @@ if [ -z "${MAJOR_VERSION}" -o -z "${TAG}" ]; then exit 1 fi -curl -sS https://api.github.com/repos/adoptium/temurin${MAJOR_VERSION}-binaries/releases > "$WORKSPACE/jdk${MAJOR_VERSION}.txt" -if [ $? -ne 0 ]; then +if ! curl -sS "https://api.github.com/repos/adoptium/temurin${MAJOR_VERSION}-binaries/releases" > "$WORKSPACE/jdk${MAJOR_VERSION}.txt"; then echo "github API call failed - aborting" exit 2 fi -[ "$VERBOSE" = "tru" ] && echo "$(date +%T) : IVT: Downloading files from release repository" +[ "$VERBOSE" = "true" ] && echo "$(date +%T) : IVT: Downloading files from release repository" # Leaving this "if/fi" commented out as it can be useful if doing standalone # testing to avoid having to re-download. May be removed in future @@ -52,14 +51,15 @@ fi mkdir staging "staging/$TAG" cd "staging/$TAG" || exit 3 # Early access versions are currently in a different format - if echo $TAG | grep ea-beta; then + if echo "$TAG" | grep ea-beta; then FILTER="ea_${MAJOR_VERSION}" else - FILTER=$(echo $TAG | sed 's/+/%2B/g') + FILTER=$(echo "$TAG" | sed 's/+/%2B/g') fi # Parse the releases list for the one we want and download everything in it - for URL in $(cat "$WORKSPACE/jdk${MAJOR_VERSION}.txt" | grep "$FILTER" | awk -F'"' '/browser_download_url/{print$4}'); do - [ "$VERBOSE" = "true" ] && echo "Downloading $(basename $URL)" + # shellcheck disable=SC2013 + for URL in $(grep "$FILTER" "$WORKSPACE/jdk${MAJOR_VERSION}.txt" | awk -F'"' '/browser_download_url/{print$4}'); do + [ "$VERBOSE" = "true" ] && echo Downloading $(basename "$URL") curl -LORsS "$URL" done @@ -97,22 +97,22 @@ done echo "$(date +%T): IVT : Verifying that all tarballs are a valid format and counting files within them" for A in OpenJDK*.tar.gz; do - if ! tar tfz $A > /dev/null; then + if ! tar tfz "$A" > /dev/null; then echo "ERROR: Failed to verify that $A can be extracted" RC=4 fi # NOTE: 40 chosen because the static-libs is in the 40s - maybe switch for different tarballs in the future? - if [ "$(tar tfz $A | wc -l)" -lt 40 ]; then + if [ "$(tar tfz "$A" | wc -l)" -lt 40 ]; then echo "ERROR: Less than 40 files in $A - that does not seem correct" RC=4 fi done for A in OpenJDK*.zip; do - if ! unzip -t $A > /dev/null; then + if ! unzip -t "$A" > /dev/null; then echo "ERROR: Failed to verify that $A can be extracted" RC=4 fi - if [ "$(unzip -l $A | wc -l)" -lt 44 ]; then + if [ "$(unzip -l "$A" | wc -l)" -lt 44 ]; then echo "ERROR: Less than 40 files in $A - that does not seem correct" RC=4 fi @@ -150,13 +150,13 @@ fi chmod 700 cyclonedx-linux-* cd "$STARTDIR" -for SBOM in $(ls -1 staging/$TAG/OpenJDK*-sbom*json | grep -v metadata); do +for SBOM in $(ls -1 staging/"$TAG"/OpenJDK*-sbom*json | grep -v metadata); do echo "$(date +%T) : IVT : Validating $SBOM ..." - if ! staging/$TAG/cyclonedx-linux-arm64 validate --input-file "$SBOM"; then + if ! staging/"$TAG"/cyclonedx-linux-arm64 validate --input-file "$SBOM"; then echo "ERROR: Failed CycloneDX validation check" RC=5 fi - if ! bash $(dirname $0)/validateSBOMcontent.sh "$SBOM" $MAJOR_VERSION $TAG; then + if ! bash $(dirname "$0")/validateSBOMcontent.sh "$SBOM" "$MAJOR_VERSION" "$TAG"; then echo "ERROR: Failed checks on $SBOM" RC=6 fi diff --git a/tooling/validateSBOMcontent.sh b/tooling/validateSBOMcontent.sh index 01fd3a150..195024dd6 100755 --- a/tooling/validateSBOMcontent.sh +++ b/tooling/validateSBOMcontent.sh @@ -7,35 +7,34 @@ if [ $# -lt 3 ]; then fi SBOMFILE="$1" MAJORVERSION="$2" -FULLVERSION="$3" +#FULLVERSION="$3" -GLIBC=$(jq '.metadata.tools[] | select(.name|test("GLIBC")) | .version' $1 | tr -d \") -GCC=$(jq '.metadata.tools[] | select(.name|test("GCC")) | .version' $1 | tr -d \") - -BOOTJDK=$(jq '.metadata.tools[] | select(.name|test("BOOTJDK")) | .version' $1 | tr -d \") -ALSA=$(jq '.metadata.tools[] | select(.name|test("ALSA")) | .version' $1 | tr -d \" | sed -e 's/^.*alsa-lib-//' -e 's/\.tar.bz2//') -FREETYPE=$(jq '.metadata.tools[] | select(.name|test("FreeType")) | .version' $1 | tr -d \") -FREEMARKER=$(jq '.metadata.tools[] | select(.name|test("FreeMarker")) | .version' $1 | tr -d \") -COMPILER=$(jq '.components[0].properties[] | select(.name|test("Build Tools Summary")).value' $SBOMFILE | sed -e 's/^.*Toolchain: //g' -e 's/\ *\*.*//g') +GLIBC=$(jq '.metadata.tools[] | select(.name|test("GLIBC")) | .version' "$1" | tr -d \") +GCC=$(jq '.metadata.tools[] | select(.name|test("GCC")) | .version' "$1" | tr -d \") +BOOTJDK=$(jq '.metadata.tools[] | select(.name|test("BOOTJDK")) | .version' "$1" | tr -d \") +ALSA=$(jq '.metadata.tools[] | select(.name|test("ALSA")) | .version' "$1" | tr -d \" | sed -e 's/^.*alsa-lib-//' -e 's/\.tar.bz2//') +FREETYPE=$(jq '.metadata.tools[] | select(.name|test("FreeType")) | .version' "$1" | tr -d \") +FREEMARKER=$(jq '.metadata.tools[] | select(.name|test("FreeMarker")) | .version' "$1" | tr -d \") +COMPILER=$(jq '.components[0].properties[] | select(.name|test("Build Tools Summary")).value' "$SBOMFILE" | sed -e 's/^.*Toolchain: //g' -e 's/\ *\*.*//g') EXPECTED_COMPILER="gcc (GNU Compiler Collection)" EXPECTED_GLIBC="" EXPECTED_GCC="" # [ "${MAJORVERSION}" = "17" ] && EXPECTED_GCC=10.3.0 EXPECTED_ALSA=N.A -EXPECTED_FREETYPE=N.A # https://github.com/adoptium/temurin-build/issues/3493 +#EXPECTED_FREETYPE=N.A # https://github.com/adoptium/temurin-build/issues/3493 #EXPECTED_FREETYPE=https://github.com/freetype/freetype/commit/86bc8a95056c97a810986434a3f268cbe67f2902 if echo "$SBOMFILE" | grep _solaris_; then - EXPECTED_FREETYPE=N.A + #EXPECTED_FREETYPE=N.A EXPECTED_COMPILER="solstudio (Oracle Solaris Studio)" elif echo "$SBOMFILE" | grep _aix_; then EXPECTED_COMPILER="xlc (IBM XL C/C++)" elif echo "$SBOMFILE" | grep _alpine-linux_ > /dev/null; then - EXPECTED_FREETYPE=N.A + #EXPECTED_FREETYPE=N.A EXPECTED_ALSA=1.1.6 EXPECTED_GCC=10.3.1 elif echo "$SBOMFILE" | grep _linux_; then - if [ $MAJORVERSION -lt 20 ] && echo "$SBOMFILE" | grep x64 > /dev/null; then + if [ "$MAJORVERSION" -lt 20 ] && echo "$SBOMFILE" | grep x64 > /dev/null; then EXPECTED_GLIBC=2.12 elif echo "$SBOMFILE" | grep _arm_ > /dev/null; then EXPECTED_GLIBC=2.23 @@ -47,20 +46,20 @@ elif echo "$SBOMFILE" | grep _linux_; then [ "${MAJORVERSION}" = "17" ] && EXPECTED_GCC=10.3.0 [ "${MAJORVERSION}" -ge 20 ] && EXPECTED_GCC=11.2.0 EXPECTED_ALSA=1.1.6 - EXPECTED_FREETYPE=N.A + #EXPECTED_FREETYPE=N.A #elif echo $SBOMFILE | grep _mac_; then # EXPECTED_COMPILER="clang (clang/LLVM from Xcode 10.3)" elif echo "$SBOMFILE" | grep _x64_windows_; then if [ "${MAJORVERSION}" = "8" ]; then EXPECTED_COMPILER="microsoft (Microsoft Visual Studio 2017 - CURRENTLY NOT WORKING)" - EXPECTED_FREETYPE="https://github.com/freetype/freetype/commit/ec8853cd18e1a0c275372769bdad37a79550ed66" + #EXPECTED_FREETYPE="https://github.com/freetype/freetype/commit/ec8853cd18e1a0c275372769bdad37a79550ed66" else EXPECTED_COMPILER="microsoft (Microsoft Visual Studio 2019)" fi elif echo "$SBOMFILE" | grep _x86-32_windows_; then if [ "${MAJORVERSION}" = "8" ]; then EXPECTED_COMPILER="microsoft (Microsoft Visual Studio 2013)" - EXPECTED_FREETYPE="https://github.com/freetype/freetype/commit/ec8853cd18e1a0c275372769bdad37a79550ed66" + #EXPECTED_FREETYPE="https://github.com/freetype/freetype/commit/ec8853cd18e1a0c275372769bdad37a79550ed66" elif [ "${MAJORVERSION}" = "11" ]; then EXPECTED_COMPILER="microsoft (Microsoft Visual Studio 2017)" else @@ -71,13 +70,13 @@ elif echo "$SBOMFILE" | grep _mac_; then EXPECTED_COMPILER="clang (clang/LLVM from Xcode 12.4)" if [ "${MAJORVERSION}" = "8" -o "${MAJORVERSION}" = "11" ] && echo "$SBOMFILE" | grep _x64_; then EXPECTED_COMPILER="clang (clang/LLVM)" - EXPECTED_FREETYPE="https://github.com/freetype/freetype/commit/ec8853cd18e1a0c275372769bdad37a79550ed66" +# EXPECTED_FREETYPE="https://github.com/freetype/freetype/commit/ec8853cd18e1a0c275372769bdad37a79550ed66" fi fi EXPECTED_FREEMARKER=N.A RC=0 -if echo "$SBOMFILE" | egrep 'linux_'; then +if echo "$SBOMFILE" | grep 'linux_'; then [ "${GLIBC}" != "$EXPECTED_GLIBC" ] && echo "ERROR: GLIBC version not ${EXPECTED_GLIBC} (SBOM has ${GLIBC})" && RC=1 [ "${GCC}" != "$EXPECTED_GCC" ] && echo "ERROR: GCC version not ${EXPECTED_GCC} (SBOM has ${GCC})" && RC=1 fi @@ -86,25 +85,27 @@ echo "BOOTJDK is ${BOOTJDK}" [ "${ALSA}" != "$EXPECTED_ALSA" ] && echo "ERROR: ALSA version not ${EXPECTED_ALSA} (SBOM has ${ALSA})" && RC=1 # Freetype versions are inconsistent at present - see build#3484 #[ "${FREETYPE}" != "$EXPECTED_FREETYPE" ] && echo "ERROR: FreeType version not ${EXPECTED_FREETYPE} (SBOM has ${FREETYPE})" && RC=1 -[ ! -z "$(echo $FREETYPE | tr -d '[0-9]\.')" ] && echo "ERROR: FreeType version not a valid number (SBOM has ${FREETYPE})" && RC=1 + + # shellcheck disable=SC2086 +[ -n "$(echo $FREETYPE | tr -d '[0-9]\.')" ] && echo "ERROR: FreeType version not a valid number (SBOM has ${FREETYPE})" && RC=1 echo "FREETYPE is ${FREETYPE}" [ "${FREEMARKER}" != "$EXPECTED_FREEMARKER" ] && echo "ERROR: Freemarker version not ${EXPECTED_FREEMARKER} (SBOM has ${FREEMARKER})" && RC=1 echo -n "Checking for JDK source SHA validity: " -GITSHA=$(jq '.components[].properties[] | select(.name|test("OpenJDK Source Commit")) | .value' $1 | tr -d \") +GITSHA=$(jq '.components[].properties[] | select(.name|test("OpenJDK Source Commit")) | .value' "$1" | tr -d \") GITREPO=$(echo "$GITSHA" | cut -d/ -f1-5) GITSHA=$( echo "$GITSHA" | cut -d/ -f7) -if ! git ls-remote ${GITREPO} | grep ${GITSHA}; then +if ! git ls-remote "${GITREPO}" | grep "${GITSHA}"; then echo "ERROR: git sha of source repo not found" RC=1 fi echo -n "Checking for temurin-build SHA validity: " -GITSHA=$(jq '.components[].properties[] | select(.name|test("Temurin Build Ref")) | .value' $1 | tr -d \") +GITSHA=$(jq '.components[].properties[] | select(.name|test("Temurin Build Ref")) | .value' "$1" | tr -d \") GITREPO=$(echo "$GITSHA" | cut -d/ -f1-5) GITSHA=$(echo "$GITSHA" | cut -d/ -f7) echo "Checking for temurin-build SHA $GITSHA" -if ! git ls-remote ${GITREPO} | grep ${GITSHA}; then +if ! git ls-remote "${GITREPO}" | grep "${GITSHA}"; then echo "WARNING: temurin-build SHA check failed. This can happen if it was not a tagged level" if echo "$1" | grep '[0-9][0-9]-[0-9][0-9]-[0-9][0-9]-[0-9][0-9]' 2>/dev/null; then echo "Ignoring return code as filename looks like a nightly" From 33c384e7afefb5a931e7790ba9753903ff246756 Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Tue, 31 Oct 2023 17:46:26 +0000 Subject: [PATCH 3/4] Fix VS version for JDK20+ Signed-off-by: Stewart X Addison --- tooling/validateSBOMcontent.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tooling/validateSBOMcontent.sh b/tooling/validateSBOMcontent.sh index 195024dd6..f8feb2126 100755 --- a/tooling/validateSBOMcontent.sh +++ b/tooling/validateSBOMcontent.sh @@ -53,8 +53,8 @@ elif echo "$SBOMFILE" | grep _x64_windows_; then if [ "${MAJORVERSION}" = "8" ]; then EXPECTED_COMPILER="microsoft (Microsoft Visual Studio 2017 - CURRENTLY NOT WORKING)" #EXPECTED_FREETYPE="https://github.com/freetype/freetype/commit/ec8853cd18e1a0c275372769bdad37a79550ed66" - else - EXPECTED_COMPILER="microsoft (Microsoft Visual Studio 2019)" + elif [ "${MAJORVERSION}" -ge 20 ]; then + EXPECTED_COMPILER="microsoft (Microsoft Visual Studio 2022)" fi elif echo "$SBOMFILE" | grep _x86-32_windows_; then if [ "${MAJORVERSION}" = "8" ]; then From 569ba0d1ba7005f1379bd1d8e52a5dac9bdf920b Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Tue, 31 Oct 2023 17:56:53 +0000 Subject: [PATCH 4/4] Shut up linter because I don't agree with it Signed-off-by: Stewart X Addison --- tooling/release_download_test.sh | 11 ++++++++--- tooling/validateSBOMcontent.sh | 6 ++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/tooling/release_download_test.sh b/tooling/release_download_test.sh index 9d1033933..3d6e084e6 100755 --- a/tooling/release_download_test.sh +++ b/tooling/release_download_test.sh @@ -32,7 +32,7 @@ else fi echo "$(date +%T) : IVT : I will be checking https://github.com/adoptium/temurin${MAJOR_VERSION}-binaries/releases/tag/$TAG" -if [ -z "${MAJOR_VERSION}" -o -z "${TAG}" ]; then +if [ -z "${MAJOR_VERSION}" ] || [ -z "${TAG}" ]; then echo "MAJOR_VERSION or TAG undefined - aborting" exit 1 fi @@ -59,6 +59,7 @@ fi # Parse the releases list for the one we want and download everything in it # shellcheck disable=SC2013 for URL in $(grep "$FILTER" "$WORKSPACE/jdk${MAJOR_VERSION}.txt" | awk -F'"' '/browser_download_url/{print$4}'); do + # shellcheck disable=SC2046 [ "$VERBOSE" = "true" ] && echo Downloading $(basename "$URL") curl -LORsS "$URL" done @@ -74,6 +75,7 @@ export GNUPGHOME="$WORKSPACE/.gpg-temp" rm -rf "$GNUPGHOME" mkdir -p "$GNUPGHOME" && chmod og-rwx "$GNUPGHOME" gpg -q --keyserver keyserver.ubuntu.com --recv-keys "${GPGID}" || exit 1 +# shellcheck disable=SC3037 /bin/echo -e "5\ny\nq\n" | gpg -q --batch --command-fd 0 --expert --edit-key "${GPGID}" trust || exit 1 RC=0 @@ -131,6 +133,7 @@ if ! strings tarballtest/bin/java | grep ^GLIBC_2.17 > /dev/null; then RC=4 fi +# shellcheck disable=SC2166 [ "${MAJOR_VERSION}" = "8" -o "${MAJOR_VERSION}" = "11" ] && EXPECTED_GCC=7.5 [ "${MAJOR_VERSION}" = "17" ] && EXPECTED_GCC=10.3 [ "${MAJOR_VERSION}" -ge 20 ] && EXPECTED_GCC=11.2 @@ -148,15 +151,17 @@ if [ "$(sha256sum cyclonedx-linux-arm64 | cut -d' ' -f1)" != "eaac307ca4d7f3ee2a exit 1 fi chmod 700 cyclonedx-linux-* -cd "$STARTDIR" +cd "$STARTDIR" || exit 1 +# shellcheck disable=SC2010 for SBOM in $(ls -1 staging/"$TAG"/OpenJDK*-sbom*json | grep -v metadata); do echo "$(date +%T) : IVT : Validating $SBOM ..." if ! staging/"$TAG"/cyclonedx-linux-arm64 validate --input-file "$SBOM"; then echo "ERROR: Failed CycloneDX validation check" RC=5 fi - if ! bash $(dirname "$0")/validateSBOMcontent.sh "$SBOM" "$MAJOR_VERSION" "$TAG"; then + # shellcheck disable=SC2086 + if ! bash "$(dirname $0)/validateSBOMcontent.sh" "$SBOM" "$MAJOR_VERSION" "$TAG"; then echo "ERROR: Failed checks on $SBOM" RC=6 fi diff --git a/tooling/validateSBOMcontent.sh b/tooling/validateSBOMcontent.sh index f8feb2126..1807ae891 100755 --- a/tooling/validateSBOMcontent.sh +++ b/tooling/validateSBOMcontent.sh @@ -68,6 +68,7 @@ elif echo "$SBOMFILE" | grep _x86-32_windows_; then elif echo "$SBOMFILE" | grep _mac_; then # NOTE: mac/x64 native builds >=11 were using "clang (clang/LLVM from Xcode 10.3)" EXPECTED_COMPILER="clang (clang/LLVM from Xcode 12.4)" + # shellcheck disable=SC2166 if [ "${MAJORVERSION}" = "8" -o "${MAJORVERSION}" = "11" ] && echo "$SBOMFILE" | grep _x64_; then EXPECTED_COMPILER="clang (clang/LLVM)" # EXPECTED_FREETYPE="https://github.com/freetype/freetype/commit/ec8853cd18e1a0c275372769bdad37a79550ed66" @@ -86,11 +87,11 @@ echo "BOOTJDK is ${BOOTJDK}" # Freetype versions are inconsistent at present - see build#3484 #[ "${FREETYPE}" != "$EXPECTED_FREETYPE" ] && echo "ERROR: FreeType version not ${EXPECTED_FREETYPE} (SBOM has ${FREETYPE})" && RC=1 - # shellcheck disable=SC2086 +# shellcheck disable=SC2086 [ -n "$(echo $FREETYPE | tr -d '[0-9]\.')" ] && echo "ERROR: FreeType version not a valid number (SBOM has ${FREETYPE})" && RC=1 echo "FREETYPE is ${FREETYPE}" [ "${FREEMARKER}" != "$EXPECTED_FREEMARKER" ] && echo "ERROR: Freemarker version not ${EXPECTED_FREEMARKER} (SBOM has ${FREEMARKER})" && RC=1 - +# shellcheck disable=SC3037 echo -n "Checking for JDK source SHA validity: " GITSHA=$(jq '.components[].properties[] | select(.name|test("OpenJDK Source Commit")) | .value' "$1" | tr -d \") GITREPO=$(echo "$GITSHA" | cut -d/ -f1-5) @@ -100,6 +101,7 @@ if ! git ls-remote "${GITREPO}" | grep "${GITSHA}"; then RC=1 fi +# shellcheck disable=SC3037 echo -n "Checking for temurin-build SHA validity: " GITSHA=$(jq '.components[].properties[] | select(.name|test("Temurin Build Ref")) | .value' "$1" | tr -d \") GITREPO=$(echo "$GITSHA" | cut -d/ -f1-5)