Allow analysis to continue even after the first test error

.smalltalk.ston

@@ -9,6 +9,9 @@ SmalltalkCISpec {
     #testing : {
         #coverage : {
             #packages : [ 'MuTalk*' ]
+        },
+        #exclude : {
+            #classes : [ #AuxiliarTestClassForContinuingTestsExecutionAfterFirstFail ]
         }
     }
 }

src/MuTalk-Model/MethodMutation.class.st

@@ -137,6 +137,15 @@ MethodMutation >> runMutant [
 	] ensure: [ self uninstall ]
 ]
 
+{ #category : 'running' }
+MethodMutation >> runMutantWithoutStoppingOnError [
+
+	^ [ 
+		self install.
+		self runTestsWithoutStoppingOnError
+	] ensure: [ self uninstall ]
+]
+
 { #category : 'private' }
 MethodMutation >> runTests [
 
@@ -145,6 +154,14 @@ MethodMutation >> runTests [
 		  runStoppingOnFirstFailOrError
 ]
 
+{ #category : 'private' }
+MethodMutation >> runTestsWithoutStoppingOnError [
+
+	^ (TestSuite named: 'Tests cases for: ' , self printString)
+		  addTests: testCaseReferences;
+		  run
+]
+
 { #category : 'private' }
 MethodMutation >> testCaseReferences [
 	^ testCaseReferences

src/MuTalk-Model/MutantEvaluation.class.st

@@ -85,3 +85,18 @@ MutantEvaluation >> value [
 		results: testResults
 		producedBy: self
 ]
+
+{ #category : 'evaluation' }
+MutantEvaluation >> valueWithoutStoppingOnError [
+
+	| testResults |
+	self initializeCoverageResultIfNil.
+
+	mutation testCaseReferences:
+		(strategy testCasesToEvaluate: mutation for: self).
+	testResults := mutation runMutantWithoutStoppingOnError.
+	^ MutantEvaluationResult
+		  for: mutation
+		  results: testResults
+		  producedBy: self
+]

src/MuTalk-Model/MutationTestingAnalysis.class.st

@@ -12,7 +12,8 @@ Class {
 		'logger',
 		'budget',
 		'mutationsSelectionStrategy',
-		'mutantResults'
+		'mutantResults',
+		'stopOnErrorOrFail'
 	],
 	#category : 'MuTalk-Model',
 	#package : 'MuTalk-Model'
@@ -346,6 +347,12 @@ MutationTestingAnalysis >> coverageAnalysisResult: anObject [
 	coverageAnalysisResult := anObject
 ]
 
+{ #category : 'accessing' }
+MutationTestingAnalysis >> doNotStopOnErrorOrFail [
+
+	stopOnErrorOrFail := false
+]
+
 { #category : 'results' }
 MutationTestingAnalysis >> generalResult [
 	^ MutationTestingGeneralResult
@@ -370,16 +377,21 @@ MutationTestingAnalysis >> generateMutations [
 { #category : 'running' }
 MutationTestingAnalysis >> generateResults [
 
+	| mutantEvaluation |
 	mutantResults := OrderedCollection new.
 	mutations do: [ :aMutation |
-		(budget exceedsBudgetOn: mutantResults fromTotalMutations: mutations) ifTrue: [
-			^ mutantResults ].
+		(budget exceedsBudgetOn: mutantResults fromTotalMutations: mutations)
+			ifTrue: [ ^ mutantResults ].
 		logger logStartEvaluating: aMutation.
-		mutantResults add: (MutantEvaluation
-				 for: aMutation
-				 using: testCases
-				 following: mutantsEvaluationStrategy
-				 andConsidering: self coverageAnalysisResult) value ].
+		mutantEvaluation := MutantEvaluation
+			                    for: aMutation
+			                    using: testCases
+			                    following: mutantsEvaluationStrategy
+			                    andConsidering: self coverageAnalysisResult.
+
+		mutantResults add: (stopOnErrorOrFail
+				 ifTrue: [ mutantEvaluation value ]
+				 ifFalse: [ mutantEvaluation valueWithoutStoppingOnError ]) ].
 	^ mutantResults
 ]
 
@@ -402,7 +414,8 @@ MutationTestingAnalysis >> initializeFor: someTestCasesReferences mutating: some
 	mutantResults := OrderedCollection new.
 	elapsedTime := 0.
 	logger := aLogger.
-	budget := aBudget
+	budget := aBudget.
+	stopOnErrorOrFail := true
 ]
 
 { #category : 'accesing' }
@@ -478,6 +491,12 @@ MutationTestingAnalysis >> run [
 							ex return: false]
 ]
 
+{ #category : 'accessing' }
+MutationTestingAnalysis >> stopOnErrorOrFail: aBoolean [
+
+	stopOnErrorOrFail := aBoolean
+]
+
 { #category : 'accesing' }
 MutationTestingAnalysis >> testBaseClasses [
 

src/MuTalk-TestResources/AuxiliarTestClassForContinuingTestsExecutionAfterFirstFail.class.st

@@ -0,0 +1,54 @@
+Class {
+	#name : 'AuxiliarTestClassForContinuingTestsExecutionAfterFirstFail',
+	#superclass : 'TestCase',
+	#classInstVars : [
+		'counter'
+	],
+	#category : 'MuTalk-TestResources',
+	#package : 'MuTalk-TestResources'
+}
+
+{ #category : 'accessing' }
+AuxiliarTestClassForContinuingTestsExecutionAfterFirstFail class >> counter [
+
+	^ counter
+]
+
+{ #category : 'accessing' }
+AuxiliarTestClassForContinuingTestsExecutionAfterFirstFail class >> counter: anObject [
+
+	counter := anObject
+]
+
+{ #category : 'accessing' }
+AuxiliarTestClassForContinuingTestsExecutionAfterFirstFail class >> increment [
+
+	counter := counter + 1
+]
+
+{ #category : 'accessing' }
+AuxiliarTestClassForContinuingTestsExecutionAfterFirstFail class >> reset [
+
+	counter := 0
+]
+
+{ #category : 'tests' }
+AuxiliarTestClassForContinuingTestsExecutionAfterFirstFail >> testA [
+
+	self class increment.
+	self assert: self class counter < 4
+]
+
+{ #category : 'tests' }
+AuxiliarTestClassForContinuingTestsExecutionAfterFirstFail >> testB [
+
+	self class increment.
+	self assert: self class counter < 4
+]
+
+{ #category : 'tests' }
+AuxiliarTestClassForContinuingTestsExecutionAfterFirstFail >> testC [
+
+	self class increment.
+	self assert: self class counter < 4
+]

src/MuTalk-Tests/MutationTestingAnalysisTest.class.st

@@ -198,3 +198,22 @@ MutationTestingAnalysisTest >> testExecutingTwoMutations [
 	results do: [ :mutationResult | self assert: mutationResult killed ].
 	self assert: generalResult numberOfKilledMutants = 2
 ]
+
+{ #category : 'tests' }
+MutationTestingAnalysisTest >> testRunningAllTests [
+	"This test verify that the test evaluation keeps running even after the first error, if specified"
+
+	| analysis |
+	analysis := MutationTestingAnalysis
+		            testCasesFrom: { AuxiliarTestClassForContinuingTestsExecutionAfterFirstFail }
+		            mutating: { AuxiliarClassForMutationTestingAnalysis }
+		            using: MutantOperator contents.
+
+	analysis doNotStopOnErrorOrFail.
+	"In this class, tests fail after a certain of executions"
+	AuxiliarTestClassForContinuingTestsExecutionAfterFirstFail reset.
+	analysis run.
+	"Counting the number of failed test for a mutant"
+	self assert:
+		analysis generalResult killedMutants first result failures size > 1
+]

src/MuTalk-Tests/TestCasesSelectionStrategyTest.class.st

@@ -14,7 +14,8 @@ TestCasesSelectionStrategyTest >> allTestsFromPackage [
 		with: TestClassForTestingCoverage
 		with: AuxiliarTestClassForTestingStrategies
 		with: AuxiliarClassForMutationTestingAnalysisTest
-		with: AuxiliarTestClassForMTBudget)
+		with: AuxiliarTestClassForMTBudget
+		with: AuxiliarTestClassForContinuingTestsExecutionAfterFirstFail)
 		inject: OrderedCollection new
 		into: [:tests :testClass | 
 			tests addAll: testClass suite tests.