-
Notifications
You must be signed in to change notification settings - Fork 0
/
build.gradle
306 lines (270 loc) · 11 KB
/
build.gradle
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
import info.solidsoft.gradle.pitest.PitestTask
defaultTasks 'explain'
group 'org.autotest'
version '1.0-SNAPSHOT'
apply plugin: 'java'
apply plugin: 'jacoco'
apply plugin: 'info.solidsoft.pitest'
sourceCompatibility = 1.8
project.buildDir = 'build/out'
// Properties
ext {
randoopPath = 'dependencies/randoop.jar'
evosuitePath = 'dependencies/evosuite.jar'
coverageReports = "$buildDir/reports/covfefe"
mutationReports = "$buildDir/reports/pitest"
generatedDataPath = 'generated-data.csv'
listClassListPath = 'listPopulationClassList'
mapClassListPath = 'predicatedMapClassList'
classesUnderTest = ['collections.comparators.FixedOrderComparator',
'collections.iterators.FilterIterator',
'collections.map.PredicatedMap', 'math.genetics.ListPopulation']
populationClassList = ['math.genetics.ElitisticListPopulation', classesUnderTest[3]]
predicatedClassList = ['collections.map.DefaultedMap', 'collections.PredicateUtils', classesUnderTest[2]]
amountOfTests = project.hasProperty('testSuites') ? (testSuites as int) : 2
budget = project.hasProperty('budgetTime') ? (budgetTime as int) : (int) (0.12 * 60);
}
buildscript {
repositories { mavenCentral() }
dependencies {
classpath group:'info.solidsoft.gradle.pitest', name:'gradle-pitest-plugin', version:'1.1.11'
}
}
repositories {
mavenCentral()
}
dependencies {
testCompile group: 'junit', name: 'junit', version: '4.11'
compile group:'info.solidsoft.gradle.pitest', name:'gradle-pitest-plugin', version:'1.1.11'
compile files('dependencies/evosuite.jar')
}
jacoco {
toolVersion = "0.7.6.201602180812"
}
task explain {
doFirst{
println '''
To generate all tests, use "gradle generateTests"
To run all generated tests, use "gradle runAllTests"
To create coverage reports for the generated tests, use "gradle allCoverages"
To create mutation reports for the generated tests, use "gradle allMutations"
To create a csv file with coverage information for the tests, use "gradle exportCSV"
To clean up generated files and tests, use "gradle cleanUp"
*Please note that some of these tasks depend on a previous task.
i.e: generateTests must be called before runAllTests
Several convenience methods are available and can be listed by using "gradle tasks"
Available parameters:
-PbudgetTime=X Time spent creating tests
-PtestSuites=X Amount of test suites to generate for each class
Examples:
Generate a csv with information for 10 tests suites, using 2 minutes as a budget
gradle generateTests runAllTests allCoverages allMutations exportCSV -PbudgetTime=120 -PtestSuites=10
Generate and measure only randoop tests
gradle randoop runRandoopTests randoopCoverage randoopMutation exportCSV
'''
}
}
// Delete created resources
task cleanUp(dependsOn: clean){
doLast{
delete listClassListPath
delete mapClassListPath
new File('src/evosuite/').deleteDir()
new File('src/randoop/').deleteDir()
new File(buildDir.toString()).deleteDir()
delete 'evosuite-report/statistics.csv', 'evosuite-report'
delete generatedDataPath
}
}
// Create necessary files and directories
task setUp(dependsOn: [cleanUp, compileJava]) {
doLast {
createFile(listClassListPath, populationClassList)
createFile(mapClassListPath, predicatedClassList)
}
}
def createFile(String path, def contents) {
def file = new File(path)
file.createNewFile()
file.text = contents.join('\n')
}
// Generate $amountOfTests Randoop tests, using $budget time
task randoop(dependsOn: setUp) {
doFirst { println 'Starting randoop' }
amountOfTests.times { i ->
classesUnderTest.each { c ->
doLast {
def className = getClassName(c)
javaexec {
main = "randoop.main.Main"
classpath = files("$randoopPath:$buildDir/classes/main")
def classes = getClassListFor(c)
args = ['gentests', classes, "--timelimit=$budget", '--noprogressdisplay',
"--error-test-basename=Error_$className", "--regression-test-basename=Ok_$className",
"--junit-output-dir=src/randoop/$i/$className", '--testsperfile=9999']
}
println "Created test suite number $i for class $className"
}
}
}
}
String getClassListFor(String className) {
switch(className) {
case classesUnderTest[2]:
return "--classlist=$mapClassListPath"
case classesUnderTest[3]:
return "--classlist=$listClassListPath"
default:
return "--testclass=$className"
}
}
// Get simple class name from qualified name
String getClassName(String qualifiedName) {
return qualifiedName.split(/\./)[-1]
}
// Generate $amountOfTests Evosuite tests, using $budget time
task evosuite(dependsOn: setUp) {
amountOfTests.times { i ->
classesUnderTest.each { c ->
doLast {
javaexec {
main = '-jar'
args = [evosuitePath, '-projectCP', "$buildDir/classes/main", '-class', c,
'-Dno_runtime_dependency=true', "-Dsearch_budget=$budget",
"-Dtest_dir=src/evosuite/$i/${getClassName(c)}"]
}
}
}
}
}
// Conveniency task to avoid calling multiple tasks
task generateTests(dependsOn: [randoop,evosuite])
// Dynamically generate $amountOfTests tasks to run the test suites created by randoop and evosuite
// Dynamically generate $amountOfTests tasks to create coverage reports for said tasks
// In order to avoid classpath conflicts, and have separate coverage for each test suite,
// each test suite is run and reported in a different task
(amountOfTests * 2).times { i ->
String tool = i < amountOfTests ? "randoop" : "evosuite"
if (i >= amountOfTests) i -= amountOfTests
// Create separate sourceSets for each testsuite, so each has its own classpath
sourceSets.create("$tool$i") {
output.classesDir = "$buildDir/classes/$tool/$i/"
output.resourcesDir = "$buildDir/classes/$tool/$i/"
compileClasspath += sourceSets.main.output + sourceSets.test.output
runtimeClasspath += sourceSets.main.output + sourceSets.test.output
java.srcDir "src/$tool/$i"
}
// Define test dependencies for each sourceSet
configurations."$tool${i}Compile".extendsFrom configurations.testCompile
configurations."$tool${i}Runtime".extendsFrom configurations.testRuntime
// Generate a separate task to run each test suite
task "${tool}Test$i"(dependsOn: "$tool${i}Classes", type: Test){
jacoco {
// Configure jacoco output for later report creation
append = false
destinationFile = file("$buildDir/jacoco/${tool}Test${i}.exec")
}
testClassesDir = sourceSets."$tool$i".output.classesDir
classpath = sourceSets."$tool$i".runtimeClasspath
ignoreFailures = true
outputs.upToDateWhen{ false }
doFirst{
println "----Running $tool test suite number: $i"
}
}
// Generate a jacoco task to report coverage for each test suite execution
task "${tool}Coverage$i"(dependsOn: "${tool}Test$i", type: JacocoReport){
reports {
csv.enabled true
xml.enabled false
html.enabled false
csv.destination "$coverageReports/${tool}${i}.csv"
}
executionData file("$buildDir/jacoco/${tool}Test${i}.exec")
sourceSets sourceSets.main
}
// Generate a pitest mutation task for each test suite
task "${tool}Pitest$i"(type: PitestTask) {
pitest { // Specify test suite to run
testSourceSets = [sourceSets."$tool$i"]
}
// Copy settings for default pitest task
def defaultTask = tasks.findByName("pitest")
mutableCodePaths = defaultTask.mutableCodePaths
taskClasspath = defaultTask.taskClasspath
launchClasspath = defaultTask.launchClasspath
defaultFileForHistoryDate = defaultTask.defaultFileForHistoryDate
sourceDirs = defaultTask.sourceDirs
// Specify classes to mutate, as well as tests to run and report output directory
targetTests = ['*']
targetClasses = classesUnderTest
reportDir = new File("$mutationReports/$tool$i")
excludedClasses = ['Error*'] // Ignore randoop tests known to cause exceptions
outputFormats = ['CSV']
timestampedReports = false
}
}
// Convenience task to run all generated test tasks
task runRandoopTests(dependsOn: (0..amountOfTests-1).collect{ "randoopTest$it" })
task runEvosuiteTests(dependsOn: (0..amountOfTests-1).collect{ "evosuiteTest$it" })
task runAllTests(dependsOn: (0..amountOfTests-1).collectMany{["randoopTest$it", "evosuiteTest$it"]})
// Convenience task to run all generated coverage tasks (coverage tasks depend on test tasks)
task randoopCoverage(dependsOn: (0..amountOfTests-1).collect(["runRandoopTests"]){"randoopCoverage$it"})
task evosuiteCoverage(dependsOn: (0..amountOfTests-1).collect(["runEvosuiteTests"]){"evosuiteCoverage$it"})
task allCoverages(dependsOn: [randoopCoverage, evosuiteCoverage])
// Convenience task to run all generated mutation tasks
task randoopMutation(dependsOn: (0..amountOfTests-1).collect{"randoopPitest$it"})
task evosuiteMutation(dependsOn: (0..amountOfTests-1).collect{"evosuitePitest$it"})
task allMutations(dependsOn: [randoopMutation, evosuiteMutation])
// Create a csv file with values for className, Tool, line coverage, branch coverage, mutation score
task exportCSV {
// data matrix of size: testSuites * numberOfTools * numberOfClasses x featuresToMeasure
def data = new String[amountOfTests * 2 * classesUnderTest.size()][5]
doFirst {
[(0..amountOfTests-1), ['randoop','evosuite']].combinations().each{ i, tool ->
def offset = tool == 'evosuite' ? amountOfTests*classesUnderTest.size() : 0
def coverage = new File("$coverageReports/${tool}${i}.csv")
def mutations = new File("$mutationReports/${tool}${i}/mutations.csv")
if (coverage.exists())
getCoverageInformationForFile(data, coverage, offset, i, tool)
if (mutations.exists())
getMutationInformationForFile(data, mutations, offset, i, tool)
}
}
doLast {
delete generatedDataPath
def output = new File(generatedDataPath)
data.each { row ->
if (row[0] != null)
output << row.join(',') << '\n'
}
println "Finished exporting data to: $output"
}
}
def getCoverageInformationForFile(def data, File file,int offset, int i, String tool) {
file.splitEachLine(',') { dataRow ->
// find class position inside $classesUnderTest array
int classIndex = classesUnderTest.collect{getClassName(it)}.findIndexOf{ it == dataRow[2]}
if (classIndex != -1) { // if it's a class under test
def index = offset + amountOfTests*classIndex + i
data[index][0] = classesUnderTest[classIndex] // class name
data[index][1] = tool // tool name
double lineCvg = (dataRow[8] as double) / ((dataRow[7] as int) + (dataRow[8] as int)) ///
data[index][2] = lineCvg.round(2) // Line coverage
double branchCvg = (dataRow[6] as double) / ((dataRow[5] as int) + (dataRow[6] as int)) ///
data[index][3] = branchCvg.round(2) // branch coverage
}
}
}
def getMutationInformationForFile(def data, File file,int offset, int i, String tool) {
def mutationData = [:].withDefault{ [] } // map with empty lists
file.splitEachLine(',') { dataRow ->
mutationData[dataRow[1]] << dataRow[5] // load map with mutant status
}
mutationData.each { className, mutants ->
def classIndex = classesUnderTest.findIndexOf{it == className}
def index = offset + amountOfTests*classIndex + i
def mutationScore = mutants.count("KILLED") / (mutants.size() as double) ///
data[index][4] = mutationScore.round(2) // Mutation score
}
}