Skip to content

Commit

Permalink
aarch64 mac
Browse files Browse the repository at this point in the history
  • Loading branch information
@sophia-guo
sophia-guo committed Oct 10, 2024
1 parent 8b32983 commit bb9361f
Show file tree
Hide file tree
Showing 7 changed files with 76 additions and 175 deletions.
1 change: 1 addition & 0 deletions test/system/reproducibleCompare/build.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
<copy todir="." failonerror="false">
<fileset dir="temurin-build/tooling/reproducible/" includes="*.sh" />
</copy>
<chmod dir="." perm="755" includes="**/*.sh"/>
<copy todir="." failonerror="false">
<fileset dir="${TEST_ROOT}/../jdkbinary/" >
<include name="*-sbom_*.json"/>
Expand Down
21 changes: 21 additions & 0 deletions test/system/reproducibleCompare/playlist.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,4 +60,25 @@
</versions>
<platformRequirements>os.win</platformRequirements>
</test>
<test>
<testCaseName>Rebuild_Same_JDK_Reproducibility_Test_Mac</testCaseName>
<command>unset SPEC;\
ls -l $(TEST_ROOT)$(D)system$(D)reproducibleCompare$(D);\
$(TEST_ROOT)$(D)system$(D)reproducibleCompare$(D)macos_repro_build_compare.sh $(SBOM_FILE) $(JDK_FILE) $(REPORTDIR); \
$(TEST_STATUS)
</command>
<levels>
<level>dev</level>
</levels>
<groups>
<group>system</group>
</groups>
<vendors>
<vendor>eclipse</vendor>
</vendors>
<versions>
<version>21+</version>
</versions>
<platformRequirements>os.osx,arch.aarch64</platformRequirements>
</test>
</playlist>
1 change: 1 addition & 0 deletions test/system/reproducibleCompare/reproducible.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ ifndef SBOM_FILE
SBOM_FILE := $(TEST_ROOT)/../jdkbinary/$(SBOM_FILE)
endif
ifndef JDK_FILE
JDK_FILE := $(shell find $(TEST_ROOT)/../jdkbinary/ -type f -name '*-jdk_*.tar.gz')
ifneq (,$(findstring win,$(SPEC)))
JDK_FILE := $(shell find $(TEST_ROOT)/../jdkbinary/ -type f -name '*-jdk_*.zip')
endif
Expand Down
194 changes: 36 additions & 158 deletions tooling/reproducible/macos_repro_build_compare.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,16 +46,11 @@ REPORT_DIR="$3"
# The Defaults Below Are Suitable For An Adoptium Mac OS X Build Environment
# Which Has Been Created Via The Ansible Infrastructure Playbooks

WORK_DIR=~/comp-jdk-build
WORK_DIR=$(realpath "$(dirname "$0")")/comp-jdk-build
MAC_COMPILER_BASE=/Applications
MAC_COMPILER_APP_PREFIX=Xcode
MAC_SDK_LOCATION=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk

# These 3 variables dictate which parameters passed to makejdk_any_platform.sh are config arguments, build arguments or should be ignored.
CONFIG_ARGS=("--disable-warnings-as-errors" "--openjdk-target" "--with-sysroot" "--with-extra-cxxflags='" "--enable-dtrace" "--with-version-opt")
NOTUSE_ARGS=("--assemble-exploded-image" "--configure-args")
FINAL_ARG=("--build-variant")

# These variables relate to the pre-requisite ant installation
ANT_VERSION="1.10.5"
ANT_CONTRIB_VERSION="1.0b3"
Expand Down Expand Up @@ -441,118 +436,22 @@ Prepare_Env_For_Build() {
echo "Setting Variables"
export BOOTJDK_HOME=$WORK_DIR/jdk-${bootJDK}/Contents/Home

echo "Parsing Make JDK Any Platform ARGS For Build"
# Split the string into an array of words
IFS=' ' read -ra words <<< "$buildArgs"

# Add The Build Time Stamp In Case It Wasnt In The SBOM ARGS
words+=( " --build-reproducible-date \"$buildStamp\"" )

# Initialize variables
param=""
value=""
params=()

# Loop through the words
for word in "${words[@]}"; do
# Check if the word starts with '--'
if [[ $word == --* ]] || [[ $word == -b* ]]; then
# If a parameter already exists, store it in the params array
if [[ -n $param ]]; then
params+=("$param $value")
fi
# Reset variables for the new parameter
param="$word"
value=""
else
value+=" $word"
fi
done

# Add the last parameter to the array
params+=("$param $value")

# Loop Through The Parameters And Reformat Appropriately
export fixed_param=""
export fixed_value=""
export fixed_params=()
export new_params=""
CONFIG_ARRAY=()
BUILD_ARRAY=()
FINAL_ARRAY=()
for element in "${params[@]}"; do
IFS=' ' read -ra parts <<< "$element"
prepped_part0=${parts[0]}
prepped_part1=${parts[1]}
prepped_part2=${parts[2]}

fixed_param="$prepped_part0"
fixed_value="$prepped_part1 $prepped_part2"

# Handle Special Parameters ( overrides and = seperated )
if [ "$fixed_param" == "--jdk-boot-dir" ]; then fixed_value=" $BOOTJDK_HOME " ; fi
if [[ "$fixed_param" == "--with-sysroot="* ]]; then
IFS='=' read -r split_param split_value <<< "$fixed_param"
fixed_param="$split_param"
fixed_value="=$MAC_SDK_LOCATION " ;
fi
if [[ "$fixed_param" == "--openjdk-target="* ]]; then
IFS='=' read -r split_param split_value <<< "$fixed_param"
fixed_param="$split_param"
fixed_value="=$split_value "
fi
if [[ "$fixed_param" == "--with-version-opt="* ]]; then
IFS='=' read -r split_param split_value <<< "$fixed_param"
fixed_param="$split_param"
fixed_value="=$split_value"
fi
if [ "$fixed_param" == "--tag" ]; then fixed_param="-b" fixed_value=" $fixed_value" ; fi
if [ "$fixed_param" == "--target-file-name" ]; then
target_file="$(echo -e "${fixed_value}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
fixed_value=" $fixed_value"
fi
if [ "$fixed_param" == "--freetype-dir" ]; then fixed_value=" $fixed_value" ; fi
if [ "$fixed_param" == "--user-openjdk-build-root-directory" ]; then fixed_value=" $WORK_DIR/temurin-build/workspace/build/openjdkbuild/ " ; fi
if [ "$fixed_param" == "--build-variant" ]; then fixed_value=" $fixed_value " ; fi
if [ "$fixed_param" == "--build-reproducible-date" ]; then fixed_value=" $fixed_value" ; fi

if containsElement "$fixed_param" "${CONFIG_ARGS[@]}"; then
# Check if fixed_param is in CONFIG_ARGS
STRINGTOADD="$fixed_param$fixed_value"
CONFIG_ARRAY+=("$STRINGTOADD")
elif containsElement "$fixed_param" "${NOTUSE_ARGS[@]}"; then
# Check if fixed_param is in NOTUSE_ARGS
STRINGTOADD="$fixed_param$fixed_value"
IGNORED_ARRAY+=("$STRINGTOADD")
elif containsElement "$fixed_param" "${FINAL_ARG[@]}"; then
# Check if parameter should be at the end, usually just version & variant
STRINGTOADD="$fixed_param$fixed_value"
FINAL_ARRAY+=("$STRINGTOADD")
else
# Neither Config Or Ignore, so Build
STRINGTOADD="$fixed_param$fixed_value"
BUILD_ARRAY+=("$STRINGTOADD")
fi
done

# Construct Final Parameter String
for element in "${BUILD_ARRAY[@]}"; do
build_string+="$element"
done

for element in "${CONFIG_ARRAY[@]}"; do
config_string+="$element"
done

for element in "${FINAL_ARRAY[@]}"; do
final_string+="$element"
done

final_params="$build_string --configure-args \"$config_string\" $final_string"
# set --build-reproducible-date if not yet
if [[ "${buildArgs}" != *"--build-reproducible-date"* ]]; then
buildArgs="--build-reproducible-date \"${buildStamp}\" ${buildArgs}"
fi
# reset --jdk-boot-dir
# shellcheck disable=SC2001
buildArgs="$(echo "$buildArgs" | sed -e "s|--jdk-boot-dir [^ ]*|--jdk-boot-dir ${BOOTJDK_HOME}|")"
buildArgs="$(echo "$buildArgs" | sed -e "s|--with-sysroot=[^ ]*|--with-sysroot=${MAC_SDK_LOCATION}|")"
buildArgs="$(echo "$buildArgs" | sed -e "s|--user-openjdk-build-root-directory [^ ]*|--user-openjdk-build-root-directory ${WORK_DIR}/temurin-build/workspace/build/openjdkbuild/|")"
# remove ingored options
buildArgs=${buildArgs/--assemble-exploded-image /}
buildArgs=${buildArgs/--enable-sbom-strace /}

echo ""
echo ""
echo "Make JDK Any Platform Argument List = "
echo "$final_params"
echo "$buildArgs"
echo ""
echo "Parameters Parsed Successfully"
}
Expand All @@ -562,79 +461,58 @@ Build_JDK() {

# Trigger Build
cd "$WORK_DIR"
echo "cd temurin-build && ./makejdk-any-platform.sh $final_params 2>&1 | tee build.$$.log" | sh
echo "cd temurin-build && ./makejdk-any-platform.sh $buildArgs 2>&1 | tee build.$$.log" | sh
# Copy The Built JDK To The Working Directory
cp $WORK_DIR/temurin-build/workspace/target/"$target_file" $WORK_DIR/reproJDK.tar.gz
cp "$WORK_DIR"/temurin-build/workspace/target/OpenJDK*-jdk_*tar.gz $WORK_DIR/reproJDK.tar.gz
}

Compare_JDK() {
echo "Comparing JDKs"
echo ""
mkdir "$WORK_DIR/compare"
cp $WORK_DIR/src_jdk_dist.tar.gz $WORK_DIR/compare
cp $WORK_DIR/reproJDK.tar.gz $WORK_DIR/compare


# Check The Comparison Scripts Are Present And Copy To The Working Directory
cp "$WORK_DIR"/src_jdk_dist.tar.gz "$WORK_DIR"/compare
cp "$WORK_DIR"/reproJDK.tar.gz "$WORK_DIR"/compare
cp "$(dirname "$0")"/repro_*.sh "$WORK_DIR"/compare/

if [ -f ./repro_common.sh ]; then
cp ./repro_common.sh $WORK_DIR/compare/repro_common.sh
else
wget -O "$WORK_DIR/compare/repro_common.sh" "https://raw.githubusercontent.com/adoptium/temurin-build/master/tooling/reproducible/repro_common.sh"
fi
if [ -f ./repro_compare.sh ]; then
cp ./repro_compare.sh $WORK_DIR/compare/repro_compare.sh
else
wget -O "$WORK_DIR/compare/repro_compare.sh" "https://raw.githubusercontent.com/adoptium/temurin-build/master/tooling/reproducible/repro_compare.sh"
fi
if [ -f ./repro_process.sh ]; then
cp ./repro_process.sh $WORK_DIR/compare/repro_process.sh
else
wget -O "$WORK_DIR/compare/repro_process.sh" "https://raw.githubusercontent.com/adoptium/temurin-build/master/tooling/reproducible/repro_process.sh"
fi

# Set Permissions
chmod +x "$WORK_DIR/compare/"*sh
cd "$WORK_DIR/compare"

# Unzip And Rename The Source JDK
echo "Unzip Source"
tar xvfz src_jdk_dist.tar.gz
tar xfz src_jdk_dist.tar.gz
original_directory_name=$(find . -maxdepth 1 -type d | tail -1)
mv "$original_directory_name" src_jdk

#Unzip And Rename The Target JDK
echo "Unzip Target"
tar xvfz reproJDK.tar.gz
tar xfz reproJDK.tar.gz
original_directory_name=$(find . -maxdepth 1 -type d | grep -v src_jdk | tail -1)
mv "$original_directory_name" tar_jdk

# Ensure Java Home Is Set
export JAVA_HOME=$BOOTJDK_HOME
export PATH=$JAVA_HOME/bin:$PATH
echo "cd $WORK_DIR/compare && ./repro_compare.sh temurin src_jdk/Contents/Home temurin tar_jdk/Contents/Home Darwin 2>&1" | sh &
wait
rc=$?
set -e
rc=0
./repro_compare.sh temurin src_jdk/Contents/Home temurin tar_jdk/Contents/Home Darwin 2>&1 || rc=$?
cd "$WORK_DIR"
# Display The Content Of reprotest.diff
echo ""
echo "---------------------------------------------"
echo "Output From JDK Comparison Script"
echo "---------------------------------------------"
cat $WORK_DIR/compare/reprotest.diff
echo ""
echo "---------------------------------------------"
echo "Copying Output To $(dirname "$0")"
cp $WORK_DIR/compare/reprotest.diff $WORK_DIR/reprotest.diff


if [ $rc -eq 0 ]; then
echo "Compare identical !"
else
echo "Differences found..., logged in: reprotest.diff"
fi

if [ -n "$REPORT_DIR" ]; then
echo "Copying Output To $REPORT_DIR"
cp $WORK_DIR/compare/reprotest.diff "$REPORT_DIR"
cp $WORK_DIR/reproJDK.tar.gz "$REPORT_DIR"
cp "$WORK_DIR"/compare/reprotest.diff "$REPORT_DIR"
cp "$WORK_DIR"/reproJDK.tar.gz "$REPORT_DIR"
fi

exit $rc

}

#
Clean_Up_Everything() {
# Remove Working Directorys
Expand Down
26 changes: 13 additions & 13 deletions tooling/reproducible/repro_common.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,8 @@ function removeSystemModulesHashBuilderParams() {
moduleHashesFunction="// Method jdk/internal/module/ModuleHashes\$Builder.hashForModule:(Ljava/lang/String;[B)Ljdk/internal/module/ModuleHashes\$Builder;"
moduleString="// String "
virtualFunction="invokevirtual"
local JDK_DIR="$1"
systemModules="SystemModules\$0.class SystemModules\$all.class SystemModules\$default.class"
local JDK_DIR="$1"
for systemModule in $systemModules
do
FILES=$(find "${JDK_DIR}" -type f -name "$systemModule")
Expand All @@ -101,11 +101,11 @@ function removeSystemModulesHashBuilderParams() {
else
ff=$f
fi
javap -v -sysinfo -l -p -c -s -constants "$ff" > "$f.javap.tmp"
"${JDK_DIR}"/bin/javap -v -sysinfo -l -p -c -s -constants "$ff" > "$f.javap.tmp"
rm "$f"

# Remove "instruction number:" prefix, so we can just match code
if [[ $(uname) =~ Darwin* ]]; then
if [[ "$OS" =~ Darwin* ]]; then
sed -i "" -E 's/^[[:space:]]+[0-9]+:(.*)/\1/' "$f.javap.tmp"
else
sed -i -E 's/^[[:space:]]+[0-9]+:(.*)/\1/' "$f.javap.tmp"
Expand Down Expand Up @@ -153,7 +153,6 @@ function removeSystemModulesHashBuilderParams() {
# reprohex - A hex UUID to identify the binary version, again generated from binary content
function removeWindowsNonComparableData() {
echo "Removing EXE/DLL timestamps, CRC and debug repro hex from ${JDK_DIR}"

# We need to do this for all executables if patching VS_VERSION_INFO
if [[ "$PATCH_VS_VERSION_INFO" = true ]]; then
FILES=$(find "${JDK_DIR}" -type f -path '*.exe' && find "${JDK_DIR}" -type f -path '*.dll')
Expand Down Expand Up @@ -191,13 +190,13 @@ function removeWindowsNonComparableData() {
hexstr="00000000"
timestamphex=${hexstr:0:-${#timestamp}}$timestamp
timestamphexLE="${timestamphex:6:2}:${timestamphex:4:2}:${timestamphex:2:2}:${timestamphex:0:2}"
if ! java "$TEMURIN_TOOLS_BINREPL" --inFile "$f" --outFile "$f" --hex "${timestamphexLE}-AA:AA:AA:AA"; then
echo " FAILED ==> java $TEMURIN_TOOLS_BINREPL --inFile \"$f\" --outFile \"$f\" --hex \"${timestamphexLE}-AA:AA:AA:AA\""
if ! $java "$TEMURIN_TOOLS_BINREPL" --inFile "$f" --outFile "$f" --hex "${timestamphexLE}-AA:AA:AA:AA"; then
echo " FAILED ==> $work_JAVA $TEMURIN_TOOLS_BINREPL --inFile \"$f\" --outFile \"$f\" --hex \"${timestamphexLE}-AA:AA:AA:AA\""
exit 1
fi
if [ -n "$reprohexhalf" ]; then
if ! java "$TEMURIN_TOOLS_BINREPL" --inFile "$f" --outFile "$f" --hex "${reprohexhalf}-AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA"; then
echo " FAILED ==> java $TEMURIN_TOOLS_BINREPL --inFile \"$f\" --outFile \"$f\" --hex \"${reprohexhalf}-AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA\""
if ! $java "$TEMURIN_TOOLS_BINREPL" --inFile "$f" --outFile "$f" --hex "${reprohexhalf}-AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA"; then
echo " FAILED ==> $work_JAVA $TEMURIN_TOOLS_BINREPL --inFile \"$f\" --outFile \"$f\" --hex \"${reprohexhalf}-AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA\""
exit 1
fi
fi
Expand All @@ -208,7 +207,7 @@ function removeWindowsNonComparableData() {
checksumhex=${hexstr:0:-${#checksum}}$checksum
checksumhexLE="${checksumhex:6:2}:${checksumhex:4:2}:${checksumhex:2:2}:${checksumhex:0:2}"
if ! java "$TEMURIN_TOOLS_BINREPL" --inFile "$f" --outFile "$f" --hex "${checksumhexLE}-AA:AA:AA:AA" --firstOnly --32bitBoundaryOnly; then
echo " FAILED ==> java $TEMURIN_TOOLS_BINREPL --inFile \"$f\" --outFile \"$f\" --hex \"${checksumhexLE}-AA:AA:AA:AA\" --firstOnly --32bitBoundaryOnly"
echo " FAILED ==> $work_JAVA $TEMURIN_TOOLS_BINREPL --inFile \"$f\" --outFile \"$f\" --hex \"${checksumhexLE}-AA:AA:AA:AA\" --firstOnly --32bitBoundaryOnly"
exit 1
fi
done
Expand All @@ -223,7 +222,7 @@ function removeWindowsNonComparableData() {
# See https://github.com/adoptium/temurin-build/issues/2899#issuecomment-1153757419
function removeMacOSNonComparableData() {
echo "Removing MacOS dylib non-comparable UUID from ${JDK_DIR}"

MAC_JDK_ROOT="${JDK_DIR}/../../Contents"
FILES=$(find "${MAC_JDK_ROOT}" \( -type f -and -path '*.dylib' -or -path '*/bin/*' -or -path '*/lib/jspawnhelper' -not -path '*/modules_extracted/*' -or -path '*/jpackageapplauncher*' \))
for f in $FILES
do
Expand All @@ -234,8 +233,8 @@ function removeMacOSNonComparableData() {
else
# Format uuid for BINREPL
uuidhex="${uuid:0:2}:${uuid:2:2}:${uuid:4:2}:${uuid:6:2}:${uuid:8:2}:${uuid:10:2}:${uuid:12:2}:${uuid:14:2}:${uuid:16:2}:${uuid:18:2}:${uuid:20:2}:${uuid:22:2}:${uuid:24:2}:${uuid:26:2}:${uuid:28:2}:${uuid:30:2}"
if ! java "$TEMURIN_TOOLS_BINREPL" --inFile "$f" --outFile "$f" --hex "${uuidhex}-AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA" --firstOnly; then
echo " FAILED ==> java \"$TEMURIN_TOOLS_BINREPL\" --inFile \"$f\" --outFile \"$f\" --hex \"${uuidhex}-AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA\" --firstOnly"
if java "$TEMURIN_TOOLS_BINREPL" --inFile "$f" --outFile "$f" --hex "${uuidhex}-AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA" --firstOnly; then
echo " FAILED ==> $work_JAVA \"$TEMURIN_TOOLS_BINREPL\" --inFile \"$f\" --outFile \"$f\" --hex \"${uuidhex}-AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA\" --firstOnly"
exit 1
fi
fi
Expand All @@ -251,6 +250,7 @@ function removeMacOSNonComparableData() {
# java.base also requires the dependent module "hash:" values to be excluded
# as they differ due to the Signatures
function processModuleInfo() {
echo "process Module Info from ${JDK_DIR}"
if [[ "$OS" =~ CYGWIN* ]] || [[ "$OS" =~ Darwin* ]]; then
echo "Normalizing ModuleAttributes order in module-info.class, converting to javap"

Expand All @@ -264,7 +264,7 @@ function processModuleInfo() {
else
ff=$f
fi
javap -v -sysinfo -l -p -c -s -constants "$ff" > "$f.javap.tmp"
"${JDK_DIR}"/bin/javap -v -sysinfo -l -p -c -s -constants "$ff" > "$f.javap.tmp"
rm "$f"

cc=99
Expand Down
Loading

0 comments on commit bb9361f

Please sign in to comment.