Added system_lib for system prestage jars (adoptium#5400)

- Added system_lib for system prestage jars.
- Check if vendor test repo is SVTTestRepo and if so create directories `systemtest_prereqs` under respective test directories.
- Copy files from `system_lib` to the created directories so they can levarage the use of the system prestage jars.

related: adoptium#4912
related: https://github.ibm.com/runtimes/backlog/issues/1443

Signed-off-by: Anna Babu Palathingal <anna.bp@ibm.com>

buildenv/jenkins/JenkinsfileBase

@@ -106,11 +106,13 @@ def setupEnv() {
 	env.EXIT_SUCCESS = params.EXIT_SUCCESS ? params.EXIT_SUCCESS : false
 	NUM_MACHINES = params.NUM_MACHINES ? params.NUM_MACHINES.toInteger() : 1
 	env.LIB_DIR = JOB_NAME.contains("SmokeTests") ? "${WORKSPACE}/../../../../../testDependency/lib" : "${WORKSPACE}/../../testDependency/lib"
+	env.SYSTEM_LIB_DIR = JOB_NAME.contains("SmokeTests") ? "${WORKSPACE}/../../../../../testDependency/system_lib" : "${WORKSPACE}/../../testDependency/system_lib"
 	env.OPENJCEPLUS_GIT_REPO = params.OPENJCEPLUS_GIT_REPO ?: "https://github.com/ibmruntimes/OpenJCEPlus.git"
 	env.OPENJCEPLUS_GIT_BRANCH = params.OPENJCEPLUS_GIT_BRANCH ?: "semeru-java${params.JDK_VERSION}"
 
 	env.LIGHT_WEIGHT_CHECKOUT = (params.LIGHT_WEIGHT_CHECKOUT == false) ? params.LIGHT_WEIGHT_CHECKOUT : true
 	env.GENERATE_JOBS = params.GENERATE_JOBS ?: false
+	env.isSVTTestRepo=false
 
 	if (JOB_NAME.contains("Grinder")) {
 		def currentDate = new Date()
@@ -211,6 +213,10 @@ def setupParallelEnv() {
 					def customUrl = getCustomUrl()
 					if (PLATFORM.contains("windows")) {
 						env.LIB_DIR = env.LIB_DIR.replaceAll("\\\\", "/")
+						env.SYSTEM_LIB_DIR = env.SYSTEM_LIB_DIR.replaceAll("\\\\", "/")
+					}
+					if (env.BUILD_LIST == 'system') {
+						env.LIB_DIR = env.SYSTEM_LIB_DIR
 					}
 					sh "perl ./aqa-tests/TKG/scripts/getDependencies.pl -path ${env.LIB_DIR} -task default -customUrl ${customUrl}"
 				}
@@ -652,32 +658,24 @@ def buildTest() {
 		}
 
 		try {
-			// check pre-stage test libs on the machine
-			// check for each lib. If lib does not exist, donwload it.
-			// If lib exists, SHA will be checked. Re-download if SHA does not match.
-			timeout(time: 20, unit: 'MINUTES') {
-				def customUrl = getCustomUrl()
-				if (PLATFORM.contains("windows")) {
-					env.LIB_DIR = env.LIB_DIR.replaceAll("\\\\", "/")
-				}
-				sh "perl ./aqa-tests/TKG/scripts/getDependencies.pl -path ${env.LIB_DIR} -task default -customUrl ${customUrl}"
-			}
- 		} catch (Exception e) {
-			echo 'Exception: ' + e.toString()
-			echo "Cannot pre-stage test libs from ${env.LIB_DIR} on the machine. Skipping..."
-		}
-
-		try {
-			if (env.BUILD_LIST.contains('system')) {
-				//get pre-staged test jars from systemtest.getDependency build before system test compilation
-				timeout(time: 60, unit: 'MINUTES') {
-					copyArtifacts fingerprintArtifacts: true, projectName: "systemtest.getDependency", selector: lastSuccessful(), target: 'aqa-tests'
+				// check pre-stage test libs on the machine
+				// check for each lib. If lib does not exist, donwload it.
+				// If lib exists, SHA will be checked. Re-download if SHA does not match.
+				timeout(time: 20, unit: 'MINUTES') {
+					def customUrl = getCustomUrl()
+					if (PLATFORM.contains("windows")) {
+						env.LIB_DIR = env.LIB_DIR.replaceAll("\\\\", "/")
+						env.SYSTEM_LIB_DIR = env.SYSTEM_LIB_DIR.replaceAll("\\\\", "/")
+					}
+					if (env.BUILD_LIST == 'system') {
+						env.LIB_DIR = env.SYSTEM_LIB_DIR
+					}
+					sh "perl ./aqa-tests/TKG/scripts/getDependencies.pl -path ${env.LIB_DIR} -task default -customUrl ${customUrl}"
 				}
+			} catch (Exception e) {
+					echo 'Exception: ' + e.toString()
+					echo "Cannot pre-stage test libs from ${env.LIB_DIR} on the machine. Skipping..."
 			}
-		} catch (Exception e) {
-			echo 'Exception: ' + e.toString()
-			echo 'Cannot run copyArtifacts from test.getDependency/systemtest.getDependency. Skipping copyArtifacts...'
-		}
 
 		if (BUILD_LIST.contains("external")){
 			try {

get.sh

@@ -621,7 +621,7 @@ getVendorTestMaterial() {
 		sha=${vendor_shas_array[$i]}
 		dir=${vendor_dirs_array[$i]}
 		dest="vendor_${i}"
-
+		echo "vendor repo is $repoURL"
 		branchOption=""
 		if [ "$branch" != "" ]; then
 			branchOption="-b $branch"
@@ -638,6 +638,10 @@ getVendorTestMaterial() {
 			fi
 		fi
 
+		if [[ "$repoURL" =~ "SVTTestRepo" ]]; then
+			isSVTTestRepo=true
+		fi
+
 		echo "git clone ${branchOption} $repoURL $dest"
 		git clone -q --depth 1 $branchOption $repoURL $dest
 
@@ -763,7 +767,7 @@ checkOpenJ9RepoSHA()
 
 parseCommandLineArgs "$@"
 if [ "$USE_TESTENV_PROPERTIES" = true ]; then
-    teFile="./testenv/testenv.properties"
+	teFile="./testenv/testenv.properties"
 	if [[ "$PLATFORM" == *"zos"* ]]; then
 		echo "load ./testenv/testenv_zos.properties"
 		source ./testenv/testenv_zos.properties

system/common.xml

@@ -336,9 +336,41 @@
 					<chmod file="${SYSTEMTEST_DEST}/openj9-systemtest/openj9.test.sharedClasses.jvmti/bin/native/**" perm="755"/>
 				</then>
 		</if>
-		<copy todir="${SYSTEMTEST_DEST}/systemtest_prereqs/">
-			<fileset dir="${TEST_ROOT}/systemtest_prereqs/" includes="**" />
-		</copy>
+		<condition property="system_lib_dir_not_set" value="true" else="false">
+			<not>
+				<isset property="env.SYSTEM_LIB_DIR"/>
+			</not>
+		</condition>
+		<if>
+			<equals arg1="${system_lib_dir_not_set}" arg2="true"/>
+			<then>
+				<mkdir dir="${SYSTEMTEST_DEST}/systemtest_prereqs"/>
+				<copy todir="${SYSTEMTEST_DEST}/systemtest_prereqs/">
+					<fileset dir="${TEST_ROOT}/systemtest_prereqs/" includes="**"/>
+				</copy>
+			</then>
+		</if>
+		<condition property="isSVTTestRepo" value="true">
+			<isset property="env.isSVTTestRepo"/>
+		</condition>
+		<condition property="isSVTTestRepo" value="false">
+   		 	<not>
+				<isset property="env.isSVTTestRepo"/>
+			</not>
+		</condition>
+		<if>
+			<equals arg1="${isSVTTestRepo}" arg2="true"/>
+			<then>
+				<mkdir dir="${SYSTEMTEST_DEST}/systemtest_prereqs"/>
+				<mkdir dir="${TEST_ROOT}/systemtest_prereqs"/>
+				<copy todir="${SYSTEMTEST_DEST}/systemtest_prereqs/">
+					<fileset dir="${env.SYSTEM_LIB_DIR}/" includes="**"/>
+				</copy>
+				<copy todir="${TEST_ROOT}/systemtest_prereqs/">
+					<fileset dir="${env.SYSTEM_LIB_DIR}/" includes="**"/>
+				</copy>
+			</then>
+		</if>
 	</target>
 
 	<!-- target to build all projects in the repository.  -->

system/system.mk

@@ -71,15 +71,19 @@ ifdef JVM_OPTIONS
   APPLICATION_OPTIONS := $(APPLICATION_OPTIONS) -jvmArgs $(Q)$(JVM_OPTIONS)$(Q)
 endif
 
+ifndef SYSTEM_LIB_DIR
+  SYSTEM_LIB_DIR=$(SYSTEMTEST_RESROOT)/systemtest_prereqs
+endif
+
 define SYSTEMTEST_CMD_TEMPLATE
 perl $(SYSTEMTEST_RESROOT)$(D)STF$(D)stf.core$(D)scripts$(D)stf.pl \
   -test-root=$(Q)$(SYSTEMTEST_RESROOT)$(D)STF;$(SYSTEMTEST_RESROOT)$(D)aqa-systemtest$(OPENJ9_PRAM)$(Q) \
-  -systemtest-prereqs=$(Q)$(SYSTEMTEST_RESROOT)$(D)systemtest_prereqs$(Q) \
+  -systemtest-prereqs=$(Q)$(SYSTEM_LIB_DIR)$(D)$(Q) \
   -java-args=$(SQ)$(JAVA_ARGS)$(SQ) \
   -results-root=$(REPORTDIR)
 endef
 
-# Default test to be run for system_custom in regular system test builds 
+# Default test to be run for system_custom in regular system test builds
 CUSTOM_TARGET ?= -test=ClassloadingLoadTest
 
 ifneq ($(JDK_VERSION),8)