diff --git a/.gitignore b/.gitignore index 81e17ac8..fd2b92f6 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ bin output_* resources/TRSS scripts/testBot/__pycache__ +scripts/testTKG/__pycache__ # misc auto generated files autoGen.mk diff --git a/examples/feature/playlist.xml b/examples/feature/playlist.xml new file mode 100644 index 00000000..aa5c72c8 --- /dev/null +++ b/examples/feature/playlist.xml @@ -0,0 +1,51 @@ + + + + + testFeatureFips + echo "match FIPS140_3_20220501 and everything else by default"; \ + $(TEST_STATUS) + + FIPS140_3_20220501:applicable + + + + + testFeatureFipsNoApplicable + echo "test excluded on FIPS140_2"; \ + $(TEST_STATUS) + + FIPS140_2:nonapplicable + + + + + testFeatureFipsRequired + echo "only run on FIPS140_3_20220101"; \ + $(TEST_STATUS) + + FIPS140_3_20220101:required + + + + + testFeatureFipsRegexRequired + echo "only run on FIPS140_3 from 2022 Dec 01 to 2029 Dec 31"; \ + $(TEST_STATUS) + + /FIPS140_3_(2022(12\d(2))|202[3-9]\d{4})/:required + + + diff --git a/scripts/testTKG/test_base.py b/scripts/testTKG/test_base.py index 3f56dca0..036c4f51 100644 --- a/scripts/testTKG/test_base.py +++ b/scripts/testTKG/test_base.py @@ -23,12 +23,12 @@ def run_test(): command = "make _all" print(f"\t{command}") result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) - rt = checkResult(result, {'testSuccess_0'}, {'testFail_0'}, set(), set()) + rt &= checkResult(result, {'testSuccess_0'}, {'testFail_0'}, set(), set()) command = "make _testSuccess" print(f"\t{command}") result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) - rt = checkResult(result, {'testSuccess_0'}, set(), set(), set()) + rt &= checkResult(result, {'testSuccess_0'}, set(), set(), set()) return rt if __name__ == "__main__": diff --git a/scripts/testTKG/test_feature.py b/scripts/testTKG/test_feature.py new file mode 100644 index 00000000..4db652c4 --- /dev/null +++ b/scripts/testTKG/test_feature.py @@ -0,0 +1,56 @@ +################################################################################ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +import subprocess +from utils import * + +def run_test(): + rt = True + printTestheader("feature") + + buildList = "TKG/examples/feature" + command = "make _all" + print(f"\t{command}") + result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) + rt &= checkResult(result, {'testFeatureFips_0', 'testFeatureFipsNoApplicable_0'}, set(), set(), set()) + + buildList = "TKG/examples/feature" + command = "make _all" + testFlag = "export TEST_FLAG=FIPS140_2" + print(f"\t{testFlag}; {command}") + result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {testFlag}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) + rt &= checkResult(result, {'testFeatureFips_0'}, set(), set(), set()) + + command = "make _all" + testFlag = "export TEST_FLAG=FIPS140_3_20220501" + print(f"\t{testFlag}; {command}") + result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {testFlag}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) + rt &= checkResult(result, {'testFeatureFips_0', 'testFeatureFipsNoApplicable_0'}, set(), set(), set()) + + command = "make _all" + testFlag = "export TEST_FLAG=FIPS140_3_20230511" + print(f"\t{testFlag}; {command}") + result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {testFlag}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) + rt &= checkResult(result, {'testFeatureFips_0', 'testFeatureFipsNoApplicable_0', 'testFeatureFipsRegexRequired_0'}, set(), set(), set()) + + command = "make _all" + testFlag = "export TEST_FLAG=FIPS140_3_20220101" + print(f"\t{testFlag}; {command}") + result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {testFlag}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) + rt &= checkResult(result, {'testFeatureFips_0', 'testFeatureFipsNoApplicable_0', 'testFeatureFipsRequired_0'}, set(), set(), set()) + + return rt + +if __name__ == "__main__": + run_test() \ No newline at end of file diff --git a/scripts/testTKG/test_level.py b/scripts/testTKG/test_level.py index d9f7d083..b9e8495a 100644 --- a/scripts/testTKG/test_level.py +++ b/scripts/testTKG/test_level.py @@ -24,27 +24,27 @@ def run_test(): command = "make _sanity" print(f"\t{command}") result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) - rt = checkResult(result, {'testSanity_0'}, set(), set(), set()) + rt &= checkResult(result, {'testSanity_0'}, set(), set(), set()) command = "make _extended" print(f"\t{command}") result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) - rt = checkResult(result, {'testDefault_0', 'testExtended_0'}, set(), set(), set()) + rt &= checkResult(result, {'testDefault_0', 'testExtended_0'}, set(), set(), set()) command = "make _dev" print(f"\t{command}") result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) - rt = checkResult(result, {'testDev_0'}, set(), set(), set()) + rt &= checkResult(result, {'testDev_0'}, set(), set(), set()) command = "make _special" print(f"\t{command}") result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) - rt = checkResult(result, {'testSpecial_0'}, set(), set(), set()) + rt &= checkResult(result, {'testSpecial_0'}, set(), set(), set()) command = "make _all" print(f"\t{command}") result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) - rt = checkResult(result, {'testSanity_0', 'testDefault_0', 'testExtended_0', 'testDev_0', 'testSpecial_0'}, set(), set(), set()) + rt &= checkResult(result, {'testSanity_0', 'testDefault_0', 'testExtended_0', 'testDev_0', 'testSpecial_0'}, set(), set(), set()) return rt if __name__ == "__main__": diff --git a/src/org/testKitGen/TestInfoParser.java b/src/org/testKitGen/TestInfoParser.java index 949941e9..df0b7c67 100644 --- a/src/org/testKitGen/TestInfoParser.java +++ b/src/org/testKitGen/TestInfoParser.java @@ -88,31 +88,37 @@ public TestInfo parse() { } Set testFlags = new HashSet<>(arg.getTestFlag()); for (Map.Entry entry : ti.getFeatures().entrySet()) { - if (entry.getValue().equalsIgnoreCase("required")) { - if (!testFlags.contains(entry.getKey())) { + String featureOpt = entry.getValue().toLowerCase(); + if (featureOpt.equals("required")) { + if (!isFeatureInTestFlags(testFlags, entry.getKey())) { return null; - } else if (entry.getKey().equalsIgnoreCase("aot")) { - ti.setAotOptions("$(AOT_OPTIONS) "); } - } else if (entry.getValue().equalsIgnoreCase("applicable")) { - if (testFlags.contains("aot") && (entry.getKey().equalsIgnoreCase("aot") || entry.getKey().equalsIgnoreCase("all"))) { - ti.setAotOptions("$(AOT_OPTIONS) "); - } - } else if (entry.getValue().equalsIgnoreCase("nonapplicable")) { + } else if (featureOpt.equals("nonapplicable")) { // Do not generate make target if the test is not applicable for one feature defined in TEST_FLAG - if (testFlags.contains(entry.getKey())) { + if (isFeatureInTestFlags(testFlags, entry.getKey())) { return null; } - } else if (entry.getValue().equalsIgnoreCase("explicit")) { - if (testFlags.contains("aot") && entry.getKey().equalsIgnoreCase("aot")) { - ti.setAotIterations(1); - } + } else if (featureOpt.equals("applicable") || featureOpt.equals("explicit")) { + // Do nothing } else { System.err.println("Error: Please provide a valid feature parameter in test " + ti.getTestCaseName() + ". The valid string is :[required|applicable|nonapplicable|explicit]."); System.exit(1); } } + if (testFlags.contains("aot")) { + for (Map.Entry entry : ti.getFeatures().entrySet()) { + if (doesFeatureMatchTestFlag("aot", entry.getKey())) { + String featureOpt = entry.getValue().toLowerCase(); + if (featureOpt.equals("required") || featureOpt.equals("applicable")) { + ti.setAotOptions("$(AOT_OPTIONS) "); + } else if (featureOpt.equals("explicit")) { + ti.setAotIterations(1); + } + } + } + } + String rerun = getImmediateChildContent(testEle, "rerun"); if (rerun != null) { if (rerun.equals("true")) { @@ -207,6 +213,30 @@ public TestInfo parse() { return ti; } + private boolean isFeatureInTestFlags(Set testFlags, String feature) { + for (String testFlag : testFlags) { + if (doesFeatureMatchTestFlag(testFlag, feature)) { + return true; + } + } + return false; + } + + private boolean doesFeatureMatchTestFlag(String testFlag, String feature) { + if (feature.equals("all")) { + return true; + } + if (!feature.startsWith("/") || !feature.endsWith("/")) { + return testFlag.equalsIgnoreCase(feature); + } + Pattern pattern = Pattern.compile(feature.substring(1, feature.length() - 1)); + Matcher matcher = pattern.matcher(testFlag); + if (matcher.matches()) { + return true; + } + return false; + } + private boolean checkJavaVersion(String version) { if (version.equalsIgnoreCase(arg.getJdkVersion())) { return true;