Support regex in feature tag

.gitignore

@@ -19,6 +19,7 @@ bin
 output_*
 resources/TRSS
 scripts/testBot/__pycache__
+scripts/testTKG/__pycache__
 
 # misc auto generated files
 autoGen.mk

examples/feature/playlist.xml

@@ -0,0 +1,51 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<!--
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+<playlist xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../../TKG/playlist.xsd">
+ <test>
+ <testCaseName>testFeatureFips</testCaseName>
+ <command>echo "match FIPS140_3_20220501 and everything else by default"; \
+ $(TEST_STATUS)</command>
+ <features>
+ <feature>FIPS140_3_20220501:applicable</feature>
+ </features>
+ </test>
+
+ <test>
+ <testCaseName>testFeatureFipsNoApplicable</testCaseName>
+ <command>echo "test excluded on FIPS140_2"; \
+ $(TEST_STATUS)</command>
+ <features>
+ <feature>FIPS140_2:nonapplicable</feature>
+ </features>
+ </test>
+
+ <test>
+ <testCaseName>testFeatureFipsRequired</testCaseName>
+ <command>echo "only run on FIPS140_3_20220101"; \
+ $(TEST_STATUS)</command>
+ <features>
+ <feature>FIPS140_3_20220101:required</feature>
+ </features>
+ </test>
+
+ <test>
+ <testCaseName>testFeatureFipsRegexRequired</testCaseName>
+ <command>echo "only run on FIPS140_3 from 2022 Dec 01 to 2029 Dec 31"; \
+ $(TEST_STATUS)</command>
+ <features>
+ <feature>/FIPS140_3_(2022(12\d(2))|202[3-9]\d{4})/:required</feature>
+ </features>
+ </test>
+</playlist>

scripts/testTKG/test_base.py

@@ -23,12 +23,12 @@ def run_test():
  command = "make _all"
  print(f"\t{command}")
  result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
- rt = checkResult(result, {'testSuccess_0'}, {'testFail_0'}, set(), set())
+ rt &= checkResult(result, {'testSuccess_0'}, {'testFail_0'}, set(), set())
 
  command = "make _testSuccess"
  print(f"\t{command}")
  result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
- rt = checkResult(result, {'testSuccess_0'}, set(), set(), set())
+ rt &= checkResult(result, {'testSuccess_0'}, set(), set(), set())
  return rt
 
 if __name__ == "__main__":

scripts/testTKG/test_feature.py

@@ -0,0 +1,56 @@
+################################################################################
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import subprocess
+from utils import *
+
+def run_test():
+ rt = True
+ printTestheader("feature")
+
+ buildList = "TKG/examples/feature"
+ command = "make _all"
+ print(f"\t{command}")
+ result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
+ rt &= checkResult(result, {'testFeatureFips_0', 'testFeatureFipsNoApplicable_0'}, set(), set(), set())
+
+ buildList = "TKG/examples/feature"
+ command = "make _all"
+ testFlag = "export TEST_FLAG=FIPS140_2"
+ print(f"\t{testFlag}; {command}")
+ result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {testFlag}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
+ rt &= checkResult(result, {'testFeatureFips_0'}, set(), set(), set())
+
+ command = "make _all"
+ testFlag = "export TEST_FLAG=FIPS140_3_20220501"
+ print(f"\t{testFlag}; {command}")
+ result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {testFlag}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
+ rt &= checkResult(result, {'testFeatureFips_0', 'testFeatureFipsNoApplicable_0'}, set(), set(), set())
+
+ command = "make _all"
+ testFlag = "export TEST_FLAG=FIPS140_3_20230511"
+ print(f"\t{testFlag}; {command}")
+ result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {testFlag}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
+ rt &= checkResult(result, {'testFeatureFips_0', 'testFeatureFipsNoApplicable_0', 'testFeatureFipsRegexRequired_0'}, set(), set(), set())
+
+ command = "make _all"
+ testFlag = "export TEST_FLAG=FIPS140_3_20220101"
+ print(f"\t{testFlag}; {command}")
+ result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {testFlag}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
+ rt &= checkResult(result, {'testFeatureFips_0', 'testFeatureFipsNoApplicable_0', 'testFeatureFipsRequired_0'}, set(), set(), set())
+
+ return rt
+
+if __name__ == "__main__":
+ run_test()

scripts/testTKG/test_level.py

@@ -24,27 +24,27 @@ def run_test():
  command = "make _sanity"
  print(f"\t{command}")
  result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
- rt = checkResult(result, {'testSanity_0'}, set(), set(), set())
+ rt &= checkResult(result, {'testSanity_0'}, set(), set(), set())
 
  command = "make _extended"
  print(f"\t{command}")
  result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
- rt = checkResult(result, {'testDefault_0', 'testExtended_0'}, set(), set(), set())
+ rt &= checkResult(result, {'testDefault_0', 'testExtended_0'}, set(), set(), set())
 
  command = "make _dev"
  print(f"\t{command}")
  result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
- rt = checkResult(result, {'testDev_0'}, set(), set(), set())
+ rt &= checkResult(result, {'testDev_0'}, set(), set(), set())
 
  command = "make _special"
  print(f"\t{command}")
  result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
- rt = checkResult(result, {'testSpecial_0'}, set(), set(), set())
+ rt &= checkResult(result, {'testSpecial_0'}, set(), set(), set())
 
  command = "make _all"
  print(f"\t{command}")
  result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
- rt = checkResult(result, {'testSanity_0', 'testDefault_0', 'testExtended_0', 'testDev_0', 'testSpecial_0'}, set(), set(), set())
+ rt &= checkResult(result, {'testSanity_0', 'testDefault_0', 'testExtended_0', 'testDev_0', 'testSpecial_0'}, set(), set(), set())
  return rt
 
 if __name__ == "__main__":

src/org/testKitGen/TestInfoParser.java

@@ -88,31 +88,37 @@ public TestInfo parse() {
  }
  Set<String> testFlags = new HashSet<>(arg.getTestFlag());
  for (Map.Entry<String,String> entry : ti.getFeatures().entrySet()) {
- if (entry.getValue().equalsIgnoreCase("required")) {
- if (!testFlags.contains(entry.getKey())) {
+ String featureOpt = entry.getValue().toLowerCase();
+ if (featureOpt.equals("required")) {
+ if (!isFeatureInTestFlags(testFlags, entry.getKey())) {
  return null;
- } else if (entry.getKey().equalsIgnoreCase("aot")) {
- ti.setAotOptions("$(AOT_OPTIONS) ");
  }
- } else if (entry.getValue().equalsIgnoreCase("applicable")) {
- if (testFlags.contains("aot") && (entry.getKey().equalsIgnoreCase("aot") || entry.getKey().equalsIgnoreCase("all"))) {
- ti.setAotOptions("$(AOT_OPTIONS) ");
- }
- } else if (entry.getValue().equalsIgnoreCase("nonapplicable")) {
+ } else if (featureOpt.equals("nonapplicable")) {
  // Do not generate make target if the test is not applicable for one feature defined in TEST_FLAG
- if (testFlags.contains(entry.getKey())) {
+ if (isFeatureInTestFlags(testFlags, entry.getKey())) {
  return null;
  }
- } else if (entry.getValue().equalsIgnoreCase("explicit")) {
- if (testFlags.contains("aot") && entry.getKey().equalsIgnoreCase("aot")) {
- ti.setAotIterations(1);
- }
+ } else if (featureOpt.equals("applicable") || featureOpt.equals("explicit")) {
+ // Do nothing
  } else {
  System.err.println("Error: Please provide a valid feature parameter in test " + ti.getTestCaseName() + ". The valid string is <feature_name>:[required|applicable|nonapplicable|explicit].");
  System.exit(1);
  }
  }
 
+ if (testFlags.contains("aot")) {
+ for (Map.Entry<String,String> entry : ti.getFeatures().entrySet()) {
+ if (doesFeatureMatchTestFlag("aot", entry.getKey())) {
+ String featureOpt = entry.getValue().toLowerCase();
+ if (featureOpt.equals("required") || featureOpt.equals("applicable")) {
+ ti.setAotOptions("$(AOT_OPTIONS) ");
+ } else if (featureOpt.equals("explicit")) {
+ ti.setAotIterations(1);
+ }
+ }
+ }
+ }
+
  String rerun = getImmediateChildContent(testEle, "rerun");
  if (rerun != null) {
  if (rerun.equals("true")) {
@@ -207,6 +213,30 @@ public TestInfo parse() {
  return ti;
  }
 
+ private boolean isFeatureInTestFlags(Set<String> testFlags, String feature) {
+ for (String testFlag : testFlags) {
+ if (doesFeatureMatchTestFlag(testFlag, feature)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean doesFeatureMatchTestFlag(String testFlag, String feature) {
+ if (feature.equals("all")) {
+ return true;
+ }
+ if (!feature.startsWith("/") || !feature.endsWith("/")) {
+ return testFlag.equalsIgnoreCase(feature);
+ }
+ Pattern pattern = Pattern.compile(feature.substring(1, feature.length() - 1));
+ Matcher matcher = pattern.matcher(testFlag);
+ if (matcher.matches()) {
+ return true;
+ }
+ return false;
+ }
+
  private boolean checkJavaVersion(String version) {
  if (version.equalsIgnoreCase(arg.getJdkVersion())) {
  return true;