True boolean query parameters are added to rest test case names exper…

…imentation
WebFuzzing · Nov 22, 2024 · e04153a · e04153a
1 parent 614408f
commit e04153a
Show file tree

Hide file tree

Showing 8 changed files with 197 additions and 37 deletions.
diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt
@@ -475,6 +475,10 @@ class EMConfig {
             throw ConfigProblemException("Python output is used only for black-box testing")
         }
 
+        if (namingStrategy == NamingStrategy.NUMBERED && nameWithQueryParameters) {
+            throw ConfigProblemException("Test case naming with Query Parameters is only allowed for Action based naming strategy")
+        }
+
         when (stoppingCriterion) {
             StoppingCriterion.TIME -> if (maxEvaluations != defaultMaxEvaluations) {
                 throw ConfigProblemException("Changing number of max actions, but stopping criterion is time")
@@ -2381,6 +2385,11 @@ class EMConfig {
     @Cfg("Specify the naming strategy for test cases.")
     var namingStrategy = defaultTestCaseNamingStrategy
 
+    @Experimental
+    @Cfg("Specify if true boolean query parameters are included in the test case name." +
+            " Used for test case naming disambiguation. Only valid for Action based naming strategy.")
+    var nameWithQueryParameters = false
+
 
     @Experimental
     @Probability(true)

diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/ActionTestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/ActionTestCaseNamingStrategy.kt
@@ -31,6 +31,7 @@ abstract class ActionTestCaseNamingStrategy(
     protected val with = "with"
     protected val param = "Param"
     protected val queryParam = "query$param"
+    protected val and = "and"
 
     protected fun formatName(nameTokens: List<String>): String {
         return "_${languageConventionFormatter.formatName(nameTokens)}"

diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/RestActionTestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/RestActionTestCaseNamingStrategy.kt
@@ -11,11 +11,13 @@ import org.evomaster.core.search.EvaluatedIndividual
 import org.evomaster.core.search.Solution
 import org.evomaster.core.search.action.Action
 import org.evomaster.core.search.action.EvaluatedAction
+import org.evomaster.core.search.gene.BooleanGene
 import javax.ws.rs.core.MediaType
 
 open class RestActionTestCaseNamingStrategy(
     solution: Solution<*>,
-    languageConventionFormatter: LanguageConventionFormatter
+    languageConventionFormatter: LanguageConventionFormatter,
+    private val nameWithQueryParameters: Boolean,
 ) : ActionTestCaseNamingStrategy(solution, languageConventionFormatter)  {
 
     override fun expandName(individual: EvaluatedIndividual<*>, nameTokens: MutableList<String>, ambiguitySolver: ((Action) -> List<String>)?): String {
@@ -86,7 +88,6 @@ open class RestActionTestCaseNamingStrategy(
             .filter { it.value.size == 1 && !it.key.second }
             .mapNotNull { entry ->
                 val eInd = entry.value[0]
-//                duplicatedIndividuals.remove(eInd)
                 eInd to expandName(eInd, mutableListOf(), ::pathAmbiguitySolver)
             }
             .toMap()
@@ -130,7 +131,6 @@ open class RestActionTestCaseNamingStrategy(
             .filter { it.value.size == 1 && it.key.isNotEmpty()}
             .mapNotNull { entry ->
                 val eInd = entry.value[0]
-//                duplicatedIndividuals.remove(eInd)
                 eInd to expandName(eInd, mutableListOf(), ::queryParamsAmbiguitySolver)
             }
             .toMap()
@@ -147,9 +147,28 @@ open class RestActionTestCaseNamingStrategy(
         val queryParams = restAction.parameters.filterIsInstance<QueryParam>()
         result.add(with)
         result.add(if (queryParams.size > 1) "${queryParam}s" else queryParam)
+        if (nameWithQueryParameters) {
+            addQueryParameterNames(queryParams, result)
+        }
         return result
     }
 
+    private fun addQueryParameterNames(queryParams: List<QueryParam>, result: MutableList<String>) {
+        val booleanQueryParams = getBooleanQueryParams(queryParams)
+        if (booleanQueryParams.isNotEmpty()) {
+            booleanQueryParams.forEachIndexed { index, queryParam ->
+                result.add(queryParam.name)
+                if (index != booleanQueryParams.lastIndex) {
+                    result.add(and)
+                }
+            }
+        }
+    }
+
+    private fun getBooleanQueryParams(queryParams: List<QueryParam>): List<QueryParam> {
+        return queryParams.filter { it.getGeneForQuery() is BooleanGene && (it.getGeneForQuery() as BooleanGene).value }
+    }
+
     private fun removeSolvedDuplicates(duplicatedIndividuals: MutableSet<EvaluatedIndividual<*>>, disambiguatedIndividuals: Set<EvaluatedIndividual<*>>) {
         duplicatedIndividuals.removeAll(disambiguatedIndividuals)
     }

diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt
@@ -11,10 +11,11 @@ import org.slf4j.LoggerFactory
 
 class TestCaseNamingStrategyFactory(
     private val namingStrategy: NamingStrategy,
-    private val languageConventionFormatter: LanguageConventionFormatter
+    private val languageConventionFormatter: LanguageConventionFormatter,
+    private val nameWithQueryParameters: Boolean,
 ) {
 
-    constructor(config: EMConfig): this(config.namingStrategy, LanguageConventionFormatter(config.outputFormat))
+    constructor(config: EMConfig): this(config.namingStrategy, LanguageConventionFormatter(config.outputFormat), config.nameWithQueryParameters)
 
     companion object {
         private val log: Logger = LoggerFactory.getLogger(TestCaseNamingStrategyFactory::class.java)
@@ -31,7 +32,7 @@ class TestCaseNamingStrategyFactory(
     private fun actionBasedNamingStrategy(solution: Solution<*>): NumberedTestCaseNamingStrategy {
         val individuals = solution.individuals
         return when {
-            individuals.any { it.individual is RestIndividual } -> return RestActionTestCaseNamingStrategy(solution, languageConventionFormatter)
+            individuals.any { it.individual is RestIndividual } -> return RestActionTestCaseNamingStrategy(solution, languageConventionFormatter, nameWithQueryParameters)
             individuals.any { it.individual is GraphQLIndividual } -> return GraphQLActionTestCaseNamingStrategy(solution, languageConventionFormatter)
             individuals.any { it.individual is RPCIndividual } -> return RPCActionTestCaseNamingStrategy(solution, languageConventionFormatter)
             individuals.any { it.individual is WebIndividual } -> {

diff --git a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt
@@ -582,7 +582,7 @@ internal class EMConfigTest{
         val parser = EMConfig.getOptionParser()
         val config = EMConfig()
 
-        var options = parser.parse("--outputFormat", "PYTHON_UNITTEST")
+        val options = parser.parse("--outputFormat", "PYTHON_UNITTEST")
         assertThrows(Exception::class.java, {config.updateProperties(options)})
     }
 
@@ -607,4 +607,48 @@ internal class EMConfigTest{
         assertEquals(NamingStrategy.NUMBERED, config.namingStrategy)
     }
 
+    @Test
+    fun testQueryParamsInTestCaseNamesValidForActionStrategy() {
+        val parser = EMConfig.getOptionParser()
+        val config = EMConfig()
+
+        val options = parser.parse("--namingStrategy", "ACTION", "--nameWithQueryParameters", "true")
+        config.updateProperties(options)
+
+        assertEquals(NamingStrategy.ACTION, config.namingStrategy)
+        assertTrue(config.nameWithQueryParameters)
+    }
+
+    @Test
+    fun testQueryParamsInTestCaseNamesFalseTurnsFeatureOff() {
+        val parser = EMConfig.getOptionParser()
+        val config = EMConfig()
+
+        val options = parser.parse("--namingStrategy", "ACTION", "--nameWithQueryParameters", "false")
+        config.updateProperties(options)
+
+        assertEquals(NamingStrategy.ACTION, config.namingStrategy)
+        assertFalse(config.nameWithQueryParameters)
+    }
+
+    @Test
+    fun testQueryParamsInTestCaseNamesIsNotValidForNumberedStrategy() {
+        val parser = EMConfig.getOptionParser()
+        val config = EMConfig()
+
+        val options = parser.parse("--namingStrategy", "NUMBERED", "--nameWithQueryParameters", "true")
+
+        assertThrows(Exception::class.java, {config.updateProperties(options)})
+    }
+
+    @Test
+    fun testQueryParamsInTestCaseNamesIsOffByDefault() {
+        val parser = EMConfig.getOptionParser()
+        val config = EMConfig()
+
+        config.updateProperties(parser.parse())
+
+        assertFalse(config.nameWithQueryParameters)
+    }
+
 }
diff --git a/core/src/test/kotlin/org/evomaster/core/output/naming/RestActionNamingStrategyTest.kt b/core/src/test/kotlin/org/evomaster/core/output/naming/RestActionNamingStrategyTest.kt
@@ -20,6 +20,7 @@ open class RestActionNamingStrategyTest {
     companion object {
         val pythonFormatter = LanguageConventionFormatter(OutputFormat.PYTHON_UNITTEST)
         val javaFormatter = LanguageConventionFormatter(OutputFormat.JAVA_JUNIT_4)
+        const val NO_QUERY_PARAMS_IN_NAME = false
     }
 
     @Test
@@ -29,7 +30,7 @@ open class RestActionNamingStrategyTest {
 
         val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(1, testCases.size)
@@ -42,7 +43,7 @@ open class RestActionNamingStrategyTest {
         val eIndividual = getEvaluatedIndividualWith(restAction)
         val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(1, testCases.size)
@@ -58,7 +59,7 @@ open class RestActionNamingStrategyTest {
         val itemsIndividual = getEvaluatedIndividualWith(itemsAction)
         val solution = Solution(mutableListOf(rootIndividual, itemsIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(2, testCases.size)
@@ -72,7 +73,7 @@ open class RestActionNamingStrategyTest {
         val eIndividual = getEvaluatedIndividualWith(restAction)
         val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(1, testCases.size)
@@ -85,7 +86,7 @@ open class RestActionNamingStrategyTest {
         val eIndividual = getEvaluatedIndividualWith(restAction, 200, "<tag/>", MediaType.TEXT_XML_TYPE)
         val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(1, testCases.size)
@@ -98,7 +99,7 @@ open class RestActionNamingStrategyTest {
         val eIndividual = getEvaluatedIndividualWith(restAction, 200, "[]", MediaType.APPLICATION_JSON_TYPE)
         val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(1, testCases.size)
@@ -111,7 +112,7 @@ open class RestActionNamingStrategyTest {
         val eIndividual = getEvaluatedIndividualWith(restAction, 200, "[1]", MediaType.APPLICATION_JSON_TYPE)
         val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(1, testCases.size)
@@ -124,7 +125,7 @@ open class RestActionNamingStrategyTest {
         val eIndividual = getEvaluatedIndividualWith(restAction, 200, "[1,2,3]", MediaType.APPLICATION_JSON_TYPE)
         val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(1, testCases.size)
@@ -137,7 +138,7 @@ open class RestActionNamingStrategyTest {
         val eIndividual = getEvaluatedIndividualWith(restAction, 200, "{}", MediaType.APPLICATION_JSON_TYPE)
         val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(1, testCases.size)
@@ -150,7 +151,7 @@ open class RestActionNamingStrategyTest {
         val eIndividual = getEvaluatedIndividualWith(restAction, 200, "{\"key\": \"value\"}", MediaType.APPLICATION_JSON_TYPE)
         val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(1, testCases.size)
@@ -163,7 +164,7 @@ open class RestActionNamingStrategyTest {
         val eIndividual = getEvaluatedIndividualWith(restAction, 200, "\"myItem\"", MediaType.APPLICATION_JSON_TYPE)
         val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(1, testCases.size)
@@ -176,7 +177,7 @@ open class RestActionNamingStrategyTest {
         val eIndividual = getEvaluatedIndividualWith(restAction, 401, "[]", MediaType.APPLICATION_JSON_TYPE)
         val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(1, testCases.size)
@@ -189,7 +190,7 @@ open class RestActionNamingStrategyTest {
         val eIndividual = getEvaluatedIndividualWithFaults(restAction, singletonList(DetectedFault(FaultCategory.HTTP_STATUS_500, "items")), 500)
         val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(1, testCases.size)
@@ -203,7 +204,7 @@ open class RestActionNamingStrategyTest {
         val eIndividual = getEvaluatedIndividualWithFaults(restAction, faults, 500)
         val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(1, testCases.size)
@@ -216,7 +217,7 @@ open class RestActionNamingStrategyTest {
         val eIndividual = getEvaluatedIndividualWith(restAction, true)
         val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(1, testCases.size)
@@ -229,7 +230,7 @@ open class RestActionNamingStrategyTest {
         val eIndividual = getEvaluatedIndividualWith(restAction, withMongo = true)
         val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(1, testCases.size)
@@ -242,7 +243,7 @@ open class RestActionNamingStrategyTest {
         val eIndividual = getEvaluatedIndividualWith(restAction, 201, withWireMock = true)
         val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(1, testCases.size)
@@ -255,7 +256,7 @@ open class RestActionNamingStrategyTest {
         val eIndividual = getEvaluatedIndividualWith(restAction, withSql = true, withWireMock = true)
         val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
 
-        val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter)
+        val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME)
 
         val testCases = namingStrategy.getTestCases()
         assertEquals(1, testCases.size)