Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

True boolean query parameters are added to rest test case names experimentation #1138

Merged
merged 2 commits into from
Nov 25, 2024
Merged

True boolean query parameters are added to rest test case names experimentation #1138

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e04153a
True boolean query parameters are added to rest test case names exper…
Pgarrett Nov 22, 2024
2dedef6
Removed check in config for nameWithQueryParameters with numbered str…
Pgarrett Nov 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions core/src/main/kotlin/org/evomaster/core/EMConfig.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -475,6 +475,10 @@ class EMConfig {
throw ConfigProblemException("Python output is used only for black-box testing")
}

if (namingStrategy == NamingStrategy.NUMBERED && nameWithQueryParameters) {
Pgarrett marked this conversation as resolved.
Show resolved Hide resolved
throw ConfigProblemException("Test case naming with Query Parameters is only allowed for Action based naming strategy")
}

when (stoppingCriterion) {
StoppingCriterion.TIME -> if (maxEvaluations != defaultMaxEvaluations) {
throw ConfigProblemException("Changing number of max actions, but stopping criterion is time")
Expand Down Expand Up @@ -2381,6 +2385,11 @@ class EMConfig {
@Cfg("Specify the naming strategy for test cases.")
var namingStrategy = defaultTestCaseNamingStrategy

@Experimental
@Cfg("Specify if true boolean query parameters are included in the test case name." +
" Used for test case naming disambiguation. Only valid for Action based naming strategy.")
var nameWithQueryParameters = false


@Experimental
@Probability(true)
Expand Down
1 change: 1 addition & 0 deletions core/src/main/kotlin/org/evomaster/core/output/naming/ActionTestCaseNamingStrategy.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ abstract class ActionTestCaseNamingStrategy(
protected val with = "with"
protected val param = "Param"
protected val queryParam = "query$param"
protected val and = "and"

protected fun formatName(nameTokens: List<String>): String {
return "_${languageConventionFormatter.formatName(nameTokens)}"
Expand Down
25 changes: 22 additions & 3 deletions core/src/main/kotlin/org/evomaster/core/output/naming/RestActionTestCaseNamingStrategy.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,13 @@ import org.evomaster.core.search.EvaluatedIndividual
import org.evomaster.core.search.Solution
import org.evomaster.core.search.action.Action
import org.evomaster.core.search.action.EvaluatedAction
import org.evomaster.core.search.gene.BooleanGene
import javax.ws.rs.core.MediaType

open class RestActionTestCaseNamingStrategy(
solution: Solution<*>,
languageConventionFormatter: LanguageConventionFormatter
languageConventionFormatter: LanguageConventionFormatter,
private val nameWithQueryParameters: Boolean,
) : ActionTestCaseNamingStrategy(solution, languageConventionFormatter) {

override fun expandName(individual: EvaluatedIndividual<*>, nameTokens: MutableList<String>, ambiguitySolver: ((Action) -> List<String>)?): String {
Expand Down Expand Up @@ -86,7 +88,6 @@ open class RestActionTestCaseNamingStrategy(
.filter { it.value.size == 1 && !it.key.second }
.mapNotNull { entry ->
val eInd = entry.value[0]
// duplicatedIndividuals.remove(eInd)
eInd to expandName(eInd, mutableListOf(), ::pathAmbiguitySolver)
}
.toMap()
Expand Down Expand Up @@ -130,7 +131,6 @@ open class RestActionTestCaseNamingStrategy(
.filter { it.value.size == 1 && it.key.isNotEmpty()}
.mapNotNull { entry ->
val eInd = entry.value[0]
// duplicatedIndividuals.remove(eInd)
eInd to expandName(eInd, mutableListOf(), ::queryParamsAmbiguitySolver)
}
.toMap()
Expand All @@ -147,9 +147,28 @@ open class RestActionTestCaseNamingStrategy(
val queryParams = restAction.parameters.filterIsInstance<QueryParam>()
result.add(with)
result.add(if (queryParams.size > 1) "${queryParam}s" else queryParam)
if (nameWithQueryParameters) {
addQueryParameterNames(queryParams, result)
}
return result
}

private fun addQueryParameterNames(queryParams: List<QueryParam>, result: MutableList<String>) {
val booleanQueryParams = getBooleanQueryParams(queryParams)
if (booleanQueryParams.isNotEmpty()) {
Pgarrett marked this conversation as resolved.
Show resolved Hide resolved
booleanQueryParams.forEachIndexed { index, queryParam ->
result.add(queryParam.name)
if (index != booleanQueryParams.lastIndex) {
result.add(and)
}
}
}
}

private fun getBooleanQueryParams(queryParams: List<QueryParam>): List<QueryParam> {
return queryParams.filter { it.getGeneForQuery() is BooleanGene && (it.getGeneForQuery() as BooleanGene).value }
}

private fun removeSolvedDuplicates(duplicatedIndividuals: MutableSet<EvaluatedIndividual<*>>, disambiguatedIndividuals: Set<EvaluatedIndividual<*>>) {
duplicatedIndividuals.removeAll(disambiguatedIndividuals)
}
Expand Down
7 changes: 4 additions & 3 deletions core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,11 @@ import org.slf4j.LoggerFactory

class TestCaseNamingStrategyFactory(
private val namingStrategy: NamingStrategy,
private val languageConventionFormatter: LanguageConventionFormatter
private val languageConventionFormatter: LanguageConventionFormatter,
private val nameWithQueryParameters: Boolean,
) {

constructor(config: EMConfig): this(config.namingStrategy, LanguageConventionFormatter(config.outputFormat))
constructor(config: EMConfig): this(config.namingStrategy, LanguageConventionFormatter(config.outputFormat), config.nameWithQueryParameters)

companion object {
private val log: Logger = LoggerFactory.getLogger(TestCaseNamingStrategyFactory::class.java)
Expand All @@ -31,7 +32,7 @@ class TestCaseNamingStrategyFactory(
private fun actionBasedNamingStrategy(solution: Solution<*>): NumberedTestCaseNamingStrategy {
val individuals = solution.individuals
return when {
individuals.any { it.individual is RestIndividual } -> return RestActionTestCaseNamingStrategy(solution, languageConventionFormatter)
individuals.any { it.individual is RestIndividual } -> return RestActionTestCaseNamingStrategy(solution, languageConventionFormatter, nameWithQueryParameters)
individuals.any { it.individual is GraphQLIndividual } -> return GraphQLActionTestCaseNamingStrategy(solution, languageConventionFormatter)
individuals.any { it.individual is RPCIndividual } -> return RPCActionTestCaseNamingStrategy(solution, languageConventionFormatter)
individuals.any { it.individual is WebIndividual } -> {
Expand Down
46 changes: 45 additions & 1 deletion core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -582,7 +582,7 @@ internal class EMConfigTest{
val parser = EMConfig.getOptionParser()
val config = EMConfig()

var options = parser.parse("--outputFormat", "PYTHON_UNITTEST")
val options = parser.parse("--outputFormat", "PYTHON_UNITTEST")
assertThrows(Exception::class.java, {config.updateProperties(options)})
}

Expand All @@ -607,4 +607,48 @@ internal class EMConfigTest{
assertEquals(NamingStrategy.NUMBERED, config.namingStrategy)
}

@Test
fun testQueryParamsInTestCaseNamesValidForActionStrategy() {
val parser = EMConfig.getOptionParser()
val config = EMConfig()

val options = parser.parse("--namingStrategy", "ACTION", "--nameWithQueryParameters", "true")
config.updateProperties(options)

assertEquals(NamingStrategy.ACTION, config.namingStrategy)
assertTrue(config.nameWithQueryParameters)
}

@Test
fun testQueryParamsInTestCaseNamesFalseTurnsFeatureOff() {
val parser = EMConfig.getOptionParser()
val config = EMConfig()

val options = parser.parse("--namingStrategy", "ACTION", "--nameWithQueryParameters", "false")
config.updateProperties(options)

assertEquals(NamingStrategy.ACTION, config.namingStrategy)
assertFalse(config.nameWithQueryParameters)
}

@Test
fun testQueryParamsInTestCaseNamesIsNotValidForNumberedStrategy() {
val parser = EMConfig.getOptionParser()
val config = EMConfig()

val options = parser.parse("--namingStrategy", "NUMBERED", "--nameWithQueryParameters", "true")

assertThrows(Exception::class.java, {config.updateProperties(options)})
}

@Test
fun testQueryParamsInTestCaseNamesIsOffByDefault() {
val parser = EMConfig.getOptionParser()
val config = EMConfig()

config.updateProperties(parser.parse())

assertFalse(config.nameWithQueryParameters)
}

}
37 changes: 19 additions & 18 deletions core/src/test/kotlin/org/evomaster/core/output/naming/RestActionNamingStrategyTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ open class RestActionNamingStrategyTest {
companion object {
val pythonFormatter = LanguageConventionFormatter(OutputFormat.PYTHON_UNITTEST)
val javaFormatter = LanguageConventionFormatter(OutputFormat.JAVA_JUNIT_4)
const val NO_QUERY_PARAMS_IN_NAME = false
}

@Test
Expand All @@ -29,7 +30,7 @@ open class RestActionNamingStrategyTest {

val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(1, testCases.size)
Expand All @@ -42,7 +43,7 @@ open class RestActionNamingStrategyTest {
val eIndividual = getEvaluatedIndividualWith(restAction)
val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(1, testCases.size)
Expand All @@ -58,7 +59,7 @@ open class RestActionNamingStrategyTest {
val itemsIndividual = getEvaluatedIndividualWith(itemsAction)
val solution = Solution(mutableListOf(rootIndividual, itemsIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(2, testCases.size)
Expand All @@ -72,7 +73,7 @@ open class RestActionNamingStrategyTest {
val eIndividual = getEvaluatedIndividualWith(restAction)
val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(1, testCases.size)
Expand All @@ -85,7 +86,7 @@ open class RestActionNamingStrategyTest {
val eIndividual = getEvaluatedIndividualWith(restAction, 200, "<tag/>", MediaType.TEXT_XML_TYPE)
val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(1, testCases.size)
Expand All @@ -98,7 +99,7 @@ open class RestActionNamingStrategyTest {
val eIndividual = getEvaluatedIndividualWith(restAction, 200, "[]", MediaType.APPLICATION_JSON_TYPE)
val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(1, testCases.size)
Expand All @@ -111,7 +112,7 @@ open class RestActionNamingStrategyTest {
val eIndividual = getEvaluatedIndividualWith(restAction, 200, "[1]", MediaType.APPLICATION_JSON_TYPE)
val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(1, testCases.size)
Expand All @@ -124,7 +125,7 @@ open class RestActionNamingStrategyTest {
val eIndividual = getEvaluatedIndividualWith(restAction, 200, "[1,2,3]", MediaType.APPLICATION_JSON_TYPE)
val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(1, testCases.size)
Expand All @@ -137,7 +138,7 @@ open class RestActionNamingStrategyTest {
val eIndividual = getEvaluatedIndividualWith(restAction, 200, "{}", MediaType.APPLICATION_JSON_TYPE)
val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(1, testCases.size)
Expand All @@ -150,7 +151,7 @@ open class RestActionNamingStrategyTest {
val eIndividual = getEvaluatedIndividualWith(restAction, 200, "{\"key\": \"value\"}", MediaType.APPLICATION_JSON_TYPE)
val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(1, testCases.size)
Expand All @@ -163,7 +164,7 @@ open class RestActionNamingStrategyTest {
val eIndividual = getEvaluatedIndividualWith(restAction, 200, "\"myItem\"", MediaType.APPLICATION_JSON_TYPE)
val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(1, testCases.size)
Expand All @@ -176,7 +177,7 @@ open class RestActionNamingStrategyTest {
val eIndividual = getEvaluatedIndividualWith(restAction, 401, "[]", MediaType.APPLICATION_JSON_TYPE)
val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(1, testCases.size)
Expand All @@ -189,7 +190,7 @@ open class RestActionNamingStrategyTest {
val eIndividual = getEvaluatedIndividualWithFaults(restAction, singletonList(DetectedFault(FaultCategory.HTTP_STATUS_500, "items")), 500)
val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(1, testCases.size)
Expand All @@ -203,7 +204,7 @@ open class RestActionNamingStrategyTest {
val eIndividual = getEvaluatedIndividualWithFaults(restAction, faults, 500)
val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(1, testCases.size)
Expand All @@ -216,7 +217,7 @@ open class RestActionNamingStrategyTest {
val eIndividual = getEvaluatedIndividualWith(restAction, true)
val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(1, testCases.size)
Expand All @@ -229,7 +230,7 @@ open class RestActionNamingStrategyTest {
val eIndividual = getEvaluatedIndividualWith(restAction, withMongo = true)
val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(1, testCases.size)
Expand All @@ -242,7 +243,7 @@ open class RestActionNamingStrategyTest {
val eIndividual = getEvaluatedIndividualWith(restAction, 201, withWireMock = true)
val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(1, testCases.size)
Expand All @@ -255,7 +256,7 @@ open class RestActionNamingStrategyTest {
val eIndividual = getEvaluatedIndividualWith(restAction, withSql = true, withWireMock = true)
val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())

val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter)
val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME)

val testCases = namingStrategy.getTestCases()
assertEquals(1, testCases.size)
Expand Down
Loading
Loading