Merge branch 'master' of https://github.com/EMResearch/EvoMaster into…

… bb-improvements-activate

core-it/src/test/kotlin/org/evomaster/core/mutationweight/CompareAvgNumberToMutate.kt

@@ -40,7 +40,7 @@ class CompareAvgNumberToMutate {
 
         config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS
         config.focusedSearchActivationTime = 0.5
-        config.maxActionEvaluations = 10
+        config.maxEvaluations = 10
     }
 
     fun run(){

core-it/src/test/kotlin/org/evomaster/core/mutatortest/OneMaxMutator.kt

@@ -91,7 +91,7 @@ class OneMaxMutator {
         sampler.n = n
         config.showProgress = false
         config.mutationTargetsSelectionStrategy = if (first) EMConfig.MutationTargetsSelectionStrategy.FIRST_NOT_COVERED_TARGET else EMConfig.MutationTargetsSelectionStrategy.EXPANDED_UPDATED_NOT_COVERED_TARGET
-        config.maxActionEvaluations = budget
+        config.maxEvaluations = budget
 
         val expId = listOf("${config.mutationTargetsSelectionStrategy}", "budget-$budget","#targets-$n", "improvingMutator-$improve").joinToString("_")
 

core/src/main/kotlin/org/evomaster/core/EMConfig.kt

@@ -476,13 +476,13 @@ class EMConfig {
         }
 
         when (stoppingCriterion) {
-            StoppingCriterion.TIME -> if (maxActionEvaluations != defaultMaxActionEvaluations) {
+            StoppingCriterion.TIME -> if (maxEvaluations != defaultMaxEvaluations) {
                 throw ConfigProblemException("Changing number of max actions, but stopping criterion is time")
             }
 
-            StoppingCriterion.ACTION_EVALUATIONS -> if (maxTimeInSeconds != defaultMaxTimeInSeconds ||
-                    maxTime != defaultMaxTime) {
-                throw ConfigProblemException("Changing max time, but stopping criterion is based on fitness evaluations")
+            StoppingCriterion.ACTION_EVALUATIONS, StoppingCriterion.INDIVIDUAL_EVALUATIONS ->
+                if (maxTimeInSeconds != defaultMaxTimeInSeconds || maxTime != defaultMaxTime) {
+                throw ConfigProblemException("Changing max time, but stopping criterion is based on evaluations")
             }
         }
 
@@ -1167,24 +1167,24 @@ class EMConfig {
 
     enum class StoppingCriterion {
         TIME,
-        ACTION_EVALUATIONS
+        ACTION_EVALUATIONS,
+        INDIVIDUAL_EVALUATIONS
     }
 
     @Cfg("Stopping criterion for the search")
     var stoppingCriterion = StoppingCriterion.TIME
 
 
-    val defaultMaxActionEvaluations = 1000
+    val defaultMaxEvaluations = 1000
 
-    @Cfg("Maximum number of action evaluations for the search." +
-            " A fitness evaluation can be composed of 1 or more actions," +
+    @Cfg("Maximum number of action or individual evaluations (depending on chosen stopping criterion)" +
+            " for the search. A fitness evaluation can be composed of 1 or more actions," +
             " like for example REST calls or SQL setups." +
             " The more actions are allowed, the better results one can expect." +
             " But then of course the test generation will take longer." +
             " Only applicable depending on the stopping criterion.")
     @Min(1.0)
-    var maxActionEvaluations = defaultMaxActionEvaluations
-
+    var maxEvaluations = defaultMaxEvaluations
 
     val defaultMaxTimeInSeconds = 0
 
@@ -1196,7 +1196,6 @@ class EMConfig {
     @Min(0.0)
     var maxTimeInSeconds = defaultMaxTimeInSeconds
 
-
     @Cfg("Whether or not writing statistics of the search process. " +
             "This is only needed when running experiments with different parameter settings")
     var writeStatistics = false

core/src/main/kotlin/org/evomaster/core/search/service/SearchTimeController.kt

@@ -281,7 +281,10 @@ class SearchTimeController {
 
         return when(configuration.stoppingCriterion){
             EMConfig.StoppingCriterion.ACTION_EVALUATIONS ->
-                evaluatedActions.toDouble() / configuration.maxActionEvaluations.toDouble()
+                evaluatedActions.toDouble() / configuration.maxEvaluations.toDouble()
+
+            EMConfig.StoppingCriterion.INDIVIDUAL_EVALUATIONS ->
+                evaluatedIndividuals.toDouble() / configuration.maxEvaluations.toDouble()
 
             EMConfig.StoppingCriterion.TIME ->
                 (System.currentTimeMillis() - startTime).toDouble() /

core/src/main/kotlin/org/evomaster/core/search/service/monitor/SearchProcessMonitor.kt

@@ -156,7 +156,7 @@ class SearchProcessMonitor: SearchListener {
 
     private fun setOverall(){
         val stp = config.stoppingCriterion.toString()+"_"+
-                (if(config.stoppingCriterion.toString().toLowerCase().contains("time")) config.timeLimitInSeconds().toString() else config.maxActionEvaluations)
+                (if(config.stoppingCriterion.toString().toLowerCase().contains("time")) config.timeLimitInSeconds().toString() else config.maxEvaluations)
         this.overall = SearchOverall(stp, time.evaluatedIndividuals, eval!!.individual, eval!!, archive, idMapper, time.getStartTime())
     }
 
@@ -226,7 +226,7 @@ class SearchProcessMonitor: SearchListener {
     }
 
    private fun getStepName(value: Int, isTargetFile: Boolean): String {
-       val num = String.format("%0${config.maxActionEvaluations.toString().length}d", value)
+       val num = String.format("%0${config.maxEvaluations.toString().length}d", value)
        return when(config.processFormat){
            EMConfig.ProcessDataFormat.JSON_ALL -> "EM_${num}Json"
            EMConfig.ProcessDataFormat.TEST_IND-> "EM_${num}Test"

core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestIndividualTestBase.kt

@@ -181,7 +181,7 @@ abstract class RestIndividualTestBase {
     @MethodSource("getBudgetAndNumOfResourceForSampler")
     fun testSampledIndividual(iteration: Int, numResource: Int){
         initResourceNode(numResource, 5)
-        config.maxActionEvaluations = iteration
+        config.maxEvaluations = iteration
 
         (0 until iteration).forEach { i ->
             val ind = getSampler().sample()
@@ -209,7 +209,7 @@ abstract class RestIndividualTestBase {
     @MethodSource("getBudgetAndNumOfResourceForMutator")
     fun testMutatedIndividual(iteration: Int, numResource: Int){
         initResourceNode(numResource, 5)
-        config.maxActionEvaluations = iteration
+        config.maxEvaluations = iteration
         searchTimeController.startSearch()
 
         val ind = getSampler().sample()

core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestResourceIndividualDisabledHMTest.kt

@@ -124,7 +124,7 @@ class RestResourceIndividualDisabledHMTest : RestIndividualTestBase(){
     fun testIndividualResourceManipulation(iteration: Int, numResource: Int){
         initResourceNode(numResource, 5)
 
-        config.maxActionEvaluations = iteration
+        config.maxEvaluations = iteration
         config.maxTestSize = 20
         (0 until iteration).forEach { _ ->
             val dbSize = randomness.nextInt(1, 15)

core/src/test/kotlin/org/evomaster/core/search/StoppingCriterionTest.kt

@@ -0,0 +1,58 @@
+package org.evomaster.core.search
+
+import com.google.inject.Injector
+import com.google.inject.Module
+import com.netflix.governator.guice.LifecycleInjector
+import org.evomaster.core.BaseModule
+import org.evomaster.core.EMConfig
+import org.evomaster.core.search.algorithms.onemax.OneMaxModule
+import org.evomaster.core.search.service.SearchTimeController
+import org.junit.jupiter.api.Assertions
+import org.junit.jupiter.api.Test
+
+
+class StoppingCriterionTest {
+
+    val injector: Injector = LifecycleInjector.builder()
+        .withModules(* arrayOf<Module>(OneMaxModule(), BaseModule()))
+        .build().createInjector()
+
+
+    @Test
+    fun testActionEvaluations(){
+        val config = injector.getInstance(EMConfig::class.java)
+        config.maxEvaluations = 1000
+        config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS
+
+        val stc = injector.getInstance(SearchTimeController::class.java)
+
+        val numberOfActionEvaluation = 5
+
+        stc.startSearch()
+        do {
+            stc.newActionEvaluation(numberOfActionEvaluation)
+            stc.newIndividualEvaluation()
+        } while (stc.shouldContinueSearch())
+        Assertions.assertEquals(config.maxEvaluations, stc.evaluatedActions);
+        Assertions.assertEquals(config.maxEvaluations / numberOfActionEvaluation, stc.evaluatedIndividuals);
+    }
+
+    @Test
+    fun testIndividualEvaluations(){
+        val config = injector.getInstance(EMConfig::class.java)
+        config.maxEvaluations = 1000
+        config.stoppingCriterion = EMConfig.StoppingCriterion.INDIVIDUAL_EVALUATIONS
+
+        val stc = injector.getInstance(SearchTimeController::class.java)
+
+        val numberOfActionEvaluation = 5
+
+        stc.startSearch()
+        do {
+            stc.newActionEvaluation(numberOfActionEvaluation)
+            stc.newIndividualEvaluation()
+        } while (stc.shouldContinueSearch())
+        Assertions.assertEquals(config.maxEvaluations * numberOfActionEvaluation, stc.evaluatedActions);
+        Assertions.assertEquals(config.maxEvaluations, stc.evaluatedIndividuals);
+    }
+}

core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnConstantTest.kt

@@ -32,7 +32,7 @@ class MioAlgorithmOnConstantTest {
         randomness.updateSeed(42)
 
         val config = injector.getInstance(EMConfig::class.java)
-        config.maxActionEvaluations = 200
+        config.maxEvaluations = 200
         config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS
 
         val solution = mio.search()

core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnOneMaxTest.kt

@@ -33,7 +33,7 @@ class MioAlgorithmOnOneMaxTest {
         val sampler = injector.getInstance(OneMaxSampler::class.java)
 
         val config = injector.getInstance(EMConfig::class.java)
-        config.maxActionEvaluations = 30000
+        config.maxEvaluations = 30000
         config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS
 
         val n = 20

core/src/test/kotlin/org/evomaster/core/search/algorithms/RandomSearchTest.kt

@@ -29,7 +29,7 @@ class RandomSearchTest {
                     object : TypeLiteral<RandomAlgorithm<OneMaxIndividual>>() {}))
 
             val config = injector.getInstance(EMConfig::class.java)
-            config.maxActionEvaluations = 3000
+            config.maxEvaluations = 3000
             config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS
 
             val solution = rs.search()

core/src/test/kotlin/org/evomaster/core/search/gene/BigDecimalGeneTest.kt

@@ -27,7 +27,7 @@ class BigDecimalGeneTest {
 
         config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS
         config.focusedSearchActivationTime = 0.5
-        config.maxActionEvaluations = 10
+        config.maxEvaluations = 10
         config.useTimeInFeedbackSampling = false
         config.seed = 42
 

core/src/test/kotlin/org/evomaster/core/search/gene/GeneUtilsGetDeltaTest.kt

@@ -64,7 +64,7 @@ class GeneUtilsGetDeltaTest {
     @ParameterizedTest
     @MethodSource("getBudgetAndRange")
     fun testGetDelta(iteration: Int, range: Long) {
-        config.maxActionEvaluations = iteration
+        config.maxEvaluations = iteration
         (0 until iteration).forEach { _ ->
             fakeOneEvaluation()
             val delta = GeneUtils.getDelta(randomness, apc, range = range)

core/src/test/kotlin/org/evomaster/core/search/impact/genemutationupdate/IntegerGeneMutationUpdateTest.kt

@@ -47,7 +47,7 @@ class IntegerGeneMutationUpdateTest {
                 object : TypeLiteral<MioAlgorithm<PrimitiveTypeMatchIndividual>>() {}))
 
         config = injector.getInstance(EMConfig::class.java)
-        config.maxActionEvaluations = budget
+        config.maxEvaluations = budget
         config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS
         config.probOfRandomSampling = 0.0
 

core/src/test/kotlin/org/evomaster/core/search/impact/genemutationupdate/StringGeneMutationUpdateTest.kt

@@ -51,7 +51,7 @@ class StringGeneMutationUpdateTest {
                 object : TypeLiteral<MioAlgorithm<PrimitiveTypeMatchIndividual>>() {}))
 
         config = injector.getInstance(EMConfig::class.java)
-        config.maxActionEvaluations = budget
+        config.maxEvaluations = budget
         config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS
         config.probOfRandomSampling = 0.0
 

core/src/test/kotlin/org/evomaster/core/search/impact/weightcalculation/ImpactMutationWeightControlTest.kt

@@ -43,7 +43,7 @@ class ImpactMutationWeightControlTest {
 
         config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS
         config.focusedSearchActivationTime = 0.5
-        config.maxActionEvaluations = 10
+        config.maxEvaluations = 10
         config.weightBasedMutationRate = true
     }
 

core/src/test/kotlin/org/evomaster/core/search/mutationweight/MutationWeightControlTest.kt

@@ -44,7 +44,7 @@ class MutationWeightControlTest {
 
         config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS
         config.focusedSearchActivationTime = 0.5
-        config.maxActionEvaluations = 10
+        config.maxEvaluations = 10
         config.useTimeInFeedbackSampling = false
         config.seed = 42
 

core/src/test/kotlin/org/evomaster/core/search/service/AdaptiveParameterControlTest.kt

@@ -46,7 +46,7 @@ internal class AdaptiveParameterControlTest{
     @Test
     fun testEnd(){
 
-        config.maxActionEvaluations = 10
+        config.maxEvaluations = 10
         config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS
 
         fakeEvaluation(5) //kicks in focused search
@@ -66,7 +66,7 @@ internal class AdaptiveParameterControlTest{
 
     @Test
     fun testDuring(){
-        config.maxActionEvaluations = 10
+        config.maxEvaluations = 10
         config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS
 
         val start = 30

core/src/test/kotlin/org/evomaster/core/search/service/mutator/MutatorWithOneMaxTest.kt

@@ -54,7 +54,7 @@ class MutatorWithOneMaxTest {
         injector.getInstance(OneMaxSampler::class.java).n = n
         config.mutationTargetsSelectionStrategy = strategy
 
-        config.maxActionEvaluations = budget
+        config.maxEvaluations = budget
         config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS
 
         config.saveMutationInfo = true

core/src/test/kotlin/org/evomaster/core/search/service/track/MioAlgorithmOnTrackOneMaxTest.kt

@@ -61,7 +61,7 @@ class MioAlgorithmOnTrackOneMaxTest {
                 "false",
                 "--stoppingCriterion",
                 "ACTION_EVALUATIONS",
-                "--maxActionEvaluations",
+                "--maxEvaluations",
                 "10",
                 "--maxLengthOfTraces",
                 "50",
@@ -103,7 +103,7 @@ class MioAlgorithmOnTrackOneMaxTest {
                 "false",
                 "--enableTrackEvaluatedIndividual",
                 "true",
-                "--maxActionEvaluations",
+                "--maxEvaluations",
                 "10",
                 "--maxLengthOfTraces",
                 "50"

core/src/test/kotlin/org/evomaster/core/search/service/track/TraceableElementTest.kt

@@ -67,7 +67,7 @@ class TraceableElementTest {
     fun testOneMaxIndividualWithFT(){
         config.enableTrackIndividual = false
         config.enableTrackEvaluatedIndividual = true
-        config.maxActionEvaluations = 100
+        config.maxEvaluations = 100
         config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS
         config.maxLengthOfTraces = 20
         config.probOfArchiveMutation = 0.0

docs/options.md

@@ -134,8 +134,8 @@ There are 3 types of options:
 |`labelForExperimentConfigs`| __String__. Further label to represent the names of CONFIGS sets in experiment scripts, e.g., exp.py. *Default value*: `-`.|
 |`labelForExperiments`| __String__. When running experiments and statistic files are generated, all configs are saved. So, this one can be used as extra label for classifying the experiment. *Default value*: `-`.|
 |`lastLineEpsilon`| __Double__. The Distance Metric Last Line may use several values for epsilon.During experimentation, it may be useful to adjust these values. Epsilon describes the size of the neighbourhood used for clustering, so may result in different clustering results.Epsilon should be between 0.0 and 1.0. If the value is outside of that range, epsilon will use the default of 0.8. *Constraints*: `min=0.0, max=1.0`. *Default value*: `0.8`.|
-|`maxActionEvaluations`| __Int__. Maximum number of action evaluations for the search. A fitness evaluation can be composed of 1 or more actions, like for example REST calls or SQL setups. The more actions are allowed, the better results one can expect. But then of course the test generation will take longer. Only applicable depending on the stopping criterion. *Constraints*: `min=1.0`. *Default value*: `1000`.|
 |`maxAssertionForDataInCollection`| __Int__. Specify a maximum number of data in a collection to be asserted in the generated tests. Note that zero means that only the size of the collection will be asserted. A negative value means all data in the collection will be asserted (i.e., no limit). *Default value*: `3`.|
+|`maxEvaluations`| __Int__. Maximum number of action or individual evaluations (depending on chosen stopping criterion) for the search. A fitness evaluation can be composed of 1 or more actions, like for example REST calls or SQL setups. The more actions are allowed, the better results one can expect. But then of course the test generation will take longer. Only applicable depending on the stopping criterion. *Constraints*: `min=1.0`. *Default value*: `1000`.|
 |`maxLengthForStrings`| __Int__. The maximum length allowed for evolved strings. Without this limit, strings could in theory be billions of characters long. *Constraints*: `min=0.0, max=20000.0`. *Default value*: `200`.|
 |`maxLengthForStringsAtSamplingTime`| __Int__. Maximum length when sampling a new random string. Such limit can be bypassed when a string is mutated. *Constraints*: `min=0.0`. *Default value*: `16`.|
 |`maxLengthOfTraces`| __Int__. Specify a maxLength of tracking when enableTrackIndividual or enableTrackEvaluatedIndividual is true. Note that the value should be specified with a non-negative number or -1 (for tracking all history). *Constraints*: `min=-1.0`. *Default value*: `10`.|
@@ -190,7 +190,7 @@ There are 3 types of options:
 |`startingPerOfGenesToMutate`| __Double__. Specify a starting percentage of genes of an individual to mutate. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.|
 |`statisticsColumnId`| __String__. An id that will be part as a column of the statistics file (if any is generated). *Default value*: `-`.|
 |`statisticsFile`| __String__. Where the statistics file (if any) is going to be written (in CSV format). *Default value*: `statistics.csv`.|
-|`stoppingCriterion`| __Enum__. Stopping criterion for the search. *Valid values*: `TIME, ACTION_EVALUATIONS`. *Default value*: `TIME`.|
+|`stoppingCriterion`| __Enum__. Stopping criterion for the search. *Valid values*: `TIME, ACTION_EVALUATIONS, INDIVIDUAL_EVALUATIONS`. *Default value*: `TIME`.|
 |`structureMutationProbability`| __Double__. Probability of applying a mutation that can change the structure of a test. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.|
 |`sutControllerHost`| __String__. Host name or IP address of where the SUT REST controller is listening on. *Default value*: `localhost`.|
 |`sutControllerPort`| __Int__. TCP port of where the SUT REST controller is listening on. *Constraints*: `min=0.0, max=65535.0`. *Default value*: `40100`.|

e2e-tests/dropwizard-examples/src/test/java/org/evomaster/e2etests/dw/examples/positiveinteger/PIEMTest.java

@@ -19,7 +19,7 @@ public void testRunEM(){
                 "--createTests", "false",
                 "--seed", "42",
                 "--sutControllerPort", "" + controllerPort,
-                "--maxActionEvaluations", "200",
+                "--maxEvaluations", "200",
                 "--stoppingCriterion", "ACTION_EVALUATIONS"
         };
 
@@ -38,7 +38,7 @@ public void testCreateTest() {
                 "--createTests", "true",
                 "--seed", "42",
                 "--sutControllerPort", "" + controllerPort,
-                "--maxActionEvaluations", "20",
+                "--maxEvaluations", "20",
                 "--stoppingCriterion", "ACTION_EVALUATIONS"
         };
 

e2e-tests/dropwizard-examples/src/test/java/org/evomaster/e2etests/dw/examples/simpleform/SFTest.java

@@ -29,7 +29,7 @@ public void testRunEM() throws Throwable {
                     "--createTests", "false",
                     "--seed", "" + defaultSeed++,
                     "--sutControllerPort", "" + controllerPort,
-                    "--maxActionEvaluations", "500",
+                    "--maxEvaluations", "500",
                     "--stoppingCriterion", "ACTION_EVALUATIONS"
             };
 

e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java

@@ -153,7 +153,7 @@ protected void runAndCheckDeterminism(int iterations, Consumer<List<String>> lam
                 "--showProgress", "false",
                 "--avoidNonDeterministicLogs", "true",
                 "--sutControllerPort", "" + controllerPort,
-                "--maxActionEvaluations", "" + iterations,
+                "--maxEvaluations", "" + iterations,
                 "--stoppingCriterion", "ACTION_EVALUATIONS",
                 "--useTimeInFeedbackSampling" , "false",
                 "--createConfigPathIfMissing", "false"
@@ -416,7 +416,7 @@ protected List<String> getArgsWithCompilation(int iterations, String outputFolde
                 "--seed", "" + defaultSeed,
                 "--useTimeInFeedbackSampling" , "false",
                 "--sutControllerPort", "" + controllerPort,
-                "--maxActionEvaluations", "" + iterations,
+                "--maxEvaluations", "" + iterations,
                 "--stoppingCriterion", "ACTION_EVALUATIONS",
                 "--outputFolder", outputFolderPath(outputFolderName),
                 "--outputFormat", OutputFormat.KOTLIN_JUNIT_5.toString(),