diff --git a/core-it/src/test/kotlin/org/evomaster/core/mutationweight/CompareAvgNumberToMutate.kt b/core-it/src/test/kotlin/org/evomaster/core/mutationweight/CompareAvgNumberToMutate.kt index e1ead79492..92103b9fb2 100644 --- a/core-it/src/test/kotlin/org/evomaster/core/mutationweight/CompareAvgNumberToMutate.kt +++ b/core-it/src/test/kotlin/org/evomaster/core/mutationweight/CompareAvgNumberToMutate.kt @@ -40,7 +40,7 @@ class CompareAvgNumberToMutate { config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS config.focusedSearchActivationTime = 0.5 - config.maxActionEvaluations = 10 + config.maxEvaluations = 10 } fun run(){ diff --git a/core-it/src/test/kotlin/org/evomaster/core/mutatortest/OneMaxMutator.kt b/core-it/src/test/kotlin/org/evomaster/core/mutatortest/OneMaxMutator.kt index e57643bab9..25fafddee3 100644 --- a/core-it/src/test/kotlin/org/evomaster/core/mutatortest/OneMaxMutator.kt +++ b/core-it/src/test/kotlin/org/evomaster/core/mutatortest/OneMaxMutator.kt @@ -91,7 +91,7 @@ class OneMaxMutator { sampler.n = n config.showProgress = false config.mutationTargetsSelectionStrategy = if (first) EMConfig.MutationTargetsSelectionStrategy.FIRST_NOT_COVERED_TARGET else EMConfig.MutationTargetsSelectionStrategy.EXPANDED_UPDATED_NOT_COVERED_TARGET - config.maxActionEvaluations = budget + config.maxEvaluations = budget val expId = listOf("${config.mutationTargetsSelectionStrategy}", "budget-$budget","#targets-$n", "improvingMutator-$improve").joinToString("_") diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 1fc476f58f..5adc2f63f9 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -476,13 +476,13 @@ class EMConfig { } when (stoppingCriterion) { - StoppingCriterion.TIME -> if (maxActionEvaluations != defaultMaxActionEvaluations) { + StoppingCriterion.TIME -> if (maxEvaluations != defaultMaxEvaluations) { throw ConfigProblemException("Changing number of max actions, but stopping criterion is time") } - StoppingCriterion.ACTION_EVALUATIONS -> if (maxTimeInSeconds != defaultMaxTimeInSeconds || - maxTime != defaultMaxTime) { - throw ConfigProblemException("Changing max time, but stopping criterion is based on fitness evaluations") + StoppingCriterion.ACTION_EVALUATIONS, StoppingCriterion.INDIVIDUAL_EVALUATIONS -> + if (maxTimeInSeconds != defaultMaxTimeInSeconds || maxTime != defaultMaxTime) { + throw ConfigProblemException("Changing max time, but stopping criterion is based on evaluations") } } @@ -1167,24 +1167,24 @@ class EMConfig { enum class StoppingCriterion { TIME, - ACTION_EVALUATIONS + ACTION_EVALUATIONS, + INDIVIDUAL_EVALUATIONS } @Cfg("Stopping criterion for the search") var stoppingCriterion = StoppingCriterion.TIME - val defaultMaxActionEvaluations = 1000 + val defaultMaxEvaluations = 1000 - @Cfg("Maximum number of action evaluations for the search." + - " A fitness evaluation can be composed of 1 or more actions," + + @Cfg("Maximum number of action or individual evaluations (depending on chosen stopping criterion)" + + " for the search. A fitness evaluation can be composed of 1 or more actions," + " like for example REST calls or SQL setups." + " The more actions are allowed, the better results one can expect." + " But then of course the test generation will take longer." + " Only applicable depending on the stopping criterion.") @Min(1.0) - var maxActionEvaluations = defaultMaxActionEvaluations - + var maxEvaluations = defaultMaxEvaluations val defaultMaxTimeInSeconds = 0 @@ -1196,7 +1196,6 @@ class EMConfig { @Min(0.0) var maxTimeInSeconds = defaultMaxTimeInSeconds - @Cfg("Whether or not writing statistics of the search process. " + "This is only needed when running experiments with different parameter settings") var writeStatistics = false diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/SearchTimeController.kt b/core/src/main/kotlin/org/evomaster/core/search/service/SearchTimeController.kt index 9b77a83a48..c33ac6cbdc 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/SearchTimeController.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/SearchTimeController.kt @@ -281,7 +281,10 @@ class SearchTimeController { return when(configuration.stoppingCriterion){ EMConfig.StoppingCriterion.ACTION_EVALUATIONS -> - evaluatedActions.toDouble() / configuration.maxActionEvaluations.toDouble() + evaluatedActions.toDouble() / configuration.maxEvaluations.toDouble() + + EMConfig.StoppingCriterion.INDIVIDUAL_EVALUATIONS -> + evaluatedIndividuals.toDouble() / configuration.maxEvaluations.toDouble() EMConfig.StoppingCriterion.TIME -> (System.currentTimeMillis() - startTime).toDouble() / diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/monitor/SearchProcessMonitor.kt b/core/src/main/kotlin/org/evomaster/core/search/service/monitor/SearchProcessMonitor.kt index 59276541ec..d7e6e9b8f7 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/monitor/SearchProcessMonitor.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/monitor/SearchProcessMonitor.kt @@ -156,7 +156,7 @@ class SearchProcessMonitor: SearchListener { private fun setOverall(){ val stp = config.stoppingCriterion.toString()+"_"+ - (if(config.stoppingCriterion.toString().toLowerCase().contains("time")) config.timeLimitInSeconds().toString() else config.maxActionEvaluations) + (if(config.stoppingCriterion.toString().toLowerCase().contains("time")) config.timeLimitInSeconds().toString() else config.maxEvaluations) this.overall = SearchOverall(stp, time.evaluatedIndividuals, eval!!.individual, eval!!, archive, idMapper, time.getStartTime()) } @@ -226,7 +226,7 @@ class SearchProcessMonitor: SearchListener { } private fun getStepName(value: Int, isTargetFile: Boolean): String { - val num = String.format("%0${config.maxActionEvaluations.toString().length}d", value) + val num = String.format("%0${config.maxEvaluations.toString().length}d", value) return when(config.processFormat){ EMConfig.ProcessDataFormat.JSON_ALL -> "EM_${num}Json" EMConfig.ProcessDataFormat.TEST_IND-> "EM_${num}Test" diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestIndividualTestBase.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestIndividualTestBase.kt index 2553c27465..4e3b33653a 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestIndividualTestBase.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestIndividualTestBase.kt @@ -181,7 +181,7 @@ abstract class RestIndividualTestBase { @MethodSource("getBudgetAndNumOfResourceForSampler") fun testSampledIndividual(iteration: Int, numResource: Int){ initResourceNode(numResource, 5) - config.maxActionEvaluations = iteration + config.maxEvaluations = iteration (0 until iteration).forEach { i -> val ind = getSampler().sample() @@ -209,7 +209,7 @@ abstract class RestIndividualTestBase { @MethodSource("getBudgetAndNumOfResourceForMutator") fun testMutatedIndividual(iteration: Int, numResource: Int){ initResourceNode(numResource, 5) - config.maxActionEvaluations = iteration + config.maxEvaluations = iteration searchTimeController.startSearch() val ind = getSampler().sample() diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestResourceIndividualDisabledHMTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestResourceIndividualDisabledHMTest.kt index 11ee5993f7..0192678f46 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestResourceIndividualDisabledHMTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestResourceIndividualDisabledHMTest.kt @@ -124,7 +124,7 @@ class RestResourceIndividualDisabledHMTest : RestIndividualTestBase(){ fun testIndividualResourceManipulation(iteration: Int, numResource: Int){ initResourceNode(numResource, 5) - config.maxActionEvaluations = iteration + config.maxEvaluations = iteration config.maxTestSize = 20 (0 until iteration).forEach { _ -> val dbSize = randomness.nextInt(1, 15) diff --git a/core/src/test/kotlin/org/evomaster/core/search/StoppingCriterionTest.kt b/core/src/test/kotlin/org/evomaster/core/search/StoppingCriterionTest.kt new file mode 100644 index 0000000000..6f682032cb --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/search/StoppingCriterionTest.kt @@ -0,0 +1,58 @@ +package org.evomaster.core.search + +import com.google.inject.Injector +import com.google.inject.Module +import com.netflix.governator.guice.LifecycleInjector +import org.evomaster.core.BaseModule +import org.evomaster.core.EMConfig +import org.evomaster.core.search.algorithms.onemax.OneMaxModule +import org.evomaster.core.search.service.SearchTimeController +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test + + +class StoppingCriterionTest { + + val injector: Injector = LifecycleInjector.builder() + .withModules(* arrayOf(OneMaxModule(), BaseModule())) + .build().createInjector() + + + @Test + fun testActionEvaluations(){ + val config = injector.getInstance(EMConfig::class.java) + config.maxEvaluations = 1000 + config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS + + val stc = injector.getInstance(SearchTimeController::class.java) + + val numberOfActionEvaluation = 5 + + stc.startSearch() + do { + stc.newActionEvaluation(numberOfActionEvaluation) + stc.newIndividualEvaluation() + } while (stc.shouldContinueSearch()) + Assertions.assertEquals(config.maxEvaluations, stc.evaluatedActions); + Assertions.assertEquals(config.maxEvaluations / numberOfActionEvaluation, stc.evaluatedIndividuals); + } + + @Test + fun testIndividualEvaluations(){ + val config = injector.getInstance(EMConfig::class.java) + config.maxEvaluations = 1000 + config.stoppingCriterion = EMConfig.StoppingCriterion.INDIVIDUAL_EVALUATIONS + + val stc = injector.getInstance(SearchTimeController::class.java) + + val numberOfActionEvaluation = 5 + + stc.startSearch() + do { + stc.newActionEvaluation(numberOfActionEvaluation) + stc.newIndividualEvaluation() + } while (stc.shouldContinueSearch()) + Assertions.assertEquals(config.maxEvaluations * numberOfActionEvaluation, stc.evaluatedActions); + Assertions.assertEquals(config.maxEvaluations, stc.evaluatedIndividuals); + } +} \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnConstantTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnConstantTest.kt index 2809da8b14..7951d73e6c 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnConstantTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnConstantTest.kt @@ -32,7 +32,7 @@ class MioAlgorithmOnConstantTest { randomness.updateSeed(42) val config = injector.getInstance(EMConfig::class.java) - config.maxActionEvaluations = 200 + config.maxEvaluations = 200 config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val solution = mio.search() diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnOneMaxTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnOneMaxTest.kt index 0a360125ad..fcac1237f4 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnOneMaxTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnOneMaxTest.kt @@ -33,7 +33,7 @@ class MioAlgorithmOnOneMaxTest { val sampler = injector.getInstance(OneMaxSampler::class.java) val config = injector.getInstance(EMConfig::class.java) - config.maxActionEvaluations = 30000 + config.maxEvaluations = 30000 config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val n = 20 diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/RandomSearchTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/RandomSearchTest.kt index 895bbe65a8..d32bac9fcc 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/RandomSearchTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/RandomSearchTest.kt @@ -29,7 +29,7 @@ class RandomSearchTest { object : TypeLiteral>() {})) val config = injector.getInstance(EMConfig::class.java) - config.maxActionEvaluations = 3000 + config.maxEvaluations = 3000 config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val solution = rs.search() diff --git a/core/src/test/kotlin/org/evomaster/core/search/gene/BigDecimalGeneTest.kt b/core/src/test/kotlin/org/evomaster/core/search/gene/BigDecimalGeneTest.kt index 39b891ddee..0b164b020a 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/gene/BigDecimalGeneTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/gene/BigDecimalGeneTest.kt @@ -27,7 +27,7 @@ class BigDecimalGeneTest { config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS config.focusedSearchActivationTime = 0.5 - config.maxActionEvaluations = 10 + config.maxEvaluations = 10 config.useTimeInFeedbackSampling = false config.seed = 42 diff --git a/core/src/test/kotlin/org/evomaster/core/search/gene/GeneUtilsGetDeltaTest.kt b/core/src/test/kotlin/org/evomaster/core/search/gene/GeneUtilsGetDeltaTest.kt index dd9290635b..e89172b30d 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/gene/GeneUtilsGetDeltaTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/gene/GeneUtilsGetDeltaTest.kt @@ -64,7 +64,7 @@ class GeneUtilsGetDeltaTest { @ParameterizedTest @MethodSource("getBudgetAndRange") fun testGetDelta(iteration: Int, range: Long) { - config.maxActionEvaluations = iteration + config.maxEvaluations = iteration (0 until iteration).forEach { _ -> fakeOneEvaluation() val delta = GeneUtils.getDelta(randomness, apc, range = range) diff --git a/core/src/test/kotlin/org/evomaster/core/search/impact/genemutationupdate/IntegerGeneMutationUpdateTest.kt b/core/src/test/kotlin/org/evomaster/core/search/impact/genemutationupdate/IntegerGeneMutationUpdateTest.kt index 78a071dd3b..fde7f858bf 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/impact/genemutationupdate/IntegerGeneMutationUpdateTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/impact/genemutationupdate/IntegerGeneMutationUpdateTest.kt @@ -47,7 +47,7 @@ class IntegerGeneMutationUpdateTest { object : TypeLiteral>() {})) config = injector.getInstance(EMConfig::class.java) - config.maxActionEvaluations = budget + config.maxEvaluations = budget config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS config.probOfRandomSampling = 0.0 diff --git a/core/src/test/kotlin/org/evomaster/core/search/impact/genemutationupdate/StringGeneMutationUpdateTest.kt b/core/src/test/kotlin/org/evomaster/core/search/impact/genemutationupdate/StringGeneMutationUpdateTest.kt index 019f59053e..d945087cf2 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/impact/genemutationupdate/StringGeneMutationUpdateTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/impact/genemutationupdate/StringGeneMutationUpdateTest.kt @@ -51,7 +51,7 @@ class StringGeneMutationUpdateTest { object : TypeLiteral>() {})) config = injector.getInstance(EMConfig::class.java) - config.maxActionEvaluations = budget + config.maxEvaluations = budget config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS config.probOfRandomSampling = 0.0 diff --git a/core/src/test/kotlin/org/evomaster/core/search/impact/weightcalculation/ImpactMutationWeightControlTest.kt b/core/src/test/kotlin/org/evomaster/core/search/impact/weightcalculation/ImpactMutationWeightControlTest.kt index 336216c154..6ac4ee3aea 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/impact/weightcalculation/ImpactMutationWeightControlTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/impact/weightcalculation/ImpactMutationWeightControlTest.kt @@ -43,7 +43,7 @@ class ImpactMutationWeightControlTest { config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS config.focusedSearchActivationTime = 0.5 - config.maxActionEvaluations = 10 + config.maxEvaluations = 10 config.weightBasedMutationRate = true } diff --git a/core/src/test/kotlin/org/evomaster/core/search/mutationweight/MutationWeightControlTest.kt b/core/src/test/kotlin/org/evomaster/core/search/mutationweight/MutationWeightControlTest.kt index a4ea3d787a..e63c8128a3 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/mutationweight/MutationWeightControlTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/mutationweight/MutationWeightControlTest.kt @@ -44,7 +44,7 @@ class MutationWeightControlTest { config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS config.focusedSearchActivationTime = 0.5 - config.maxActionEvaluations = 10 + config.maxEvaluations = 10 config.useTimeInFeedbackSampling = false config.seed = 42 diff --git a/core/src/test/kotlin/org/evomaster/core/search/service/AdaptiveParameterControlTest.kt b/core/src/test/kotlin/org/evomaster/core/search/service/AdaptiveParameterControlTest.kt index b169a975e7..176d44a396 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/service/AdaptiveParameterControlTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/service/AdaptiveParameterControlTest.kt @@ -46,7 +46,7 @@ internal class AdaptiveParameterControlTest{ @Test fun testEnd(){ - config.maxActionEvaluations = 10 + config.maxEvaluations = 10 config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS fakeEvaluation(5) //kicks in focused search @@ -66,7 +66,7 @@ internal class AdaptiveParameterControlTest{ @Test fun testDuring(){ - config.maxActionEvaluations = 10 + config.maxEvaluations = 10 config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val start = 30 diff --git a/core/src/test/kotlin/org/evomaster/core/search/service/mutator/MutatorWithOneMaxTest.kt b/core/src/test/kotlin/org/evomaster/core/search/service/mutator/MutatorWithOneMaxTest.kt index 0750c2c302..1d05546331 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/service/mutator/MutatorWithOneMaxTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/service/mutator/MutatorWithOneMaxTest.kt @@ -54,7 +54,7 @@ class MutatorWithOneMaxTest { injector.getInstance(OneMaxSampler::class.java).n = n config.mutationTargetsSelectionStrategy = strategy - config.maxActionEvaluations = budget + config.maxEvaluations = budget config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS config.saveMutationInfo = true diff --git a/core/src/test/kotlin/org/evomaster/core/search/service/track/MioAlgorithmOnTrackOneMaxTest.kt b/core/src/test/kotlin/org/evomaster/core/search/service/track/MioAlgorithmOnTrackOneMaxTest.kt index cdd0294349..dc8d574bef 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/service/track/MioAlgorithmOnTrackOneMaxTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/service/track/MioAlgorithmOnTrackOneMaxTest.kt @@ -61,7 +61,7 @@ class MioAlgorithmOnTrackOneMaxTest { "false", "--stoppingCriterion", "ACTION_EVALUATIONS", - "--maxActionEvaluations", + "--maxEvaluations", "10", "--maxLengthOfTraces", "50", @@ -103,7 +103,7 @@ class MioAlgorithmOnTrackOneMaxTest { "false", "--enableTrackEvaluatedIndividual", "true", - "--maxActionEvaluations", + "--maxEvaluations", "10", "--maxLengthOfTraces", "50" diff --git a/core/src/test/kotlin/org/evomaster/core/search/service/track/TraceableElementTest.kt b/core/src/test/kotlin/org/evomaster/core/search/service/track/TraceableElementTest.kt index 6180673ff1..92fc1d0e1c 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/service/track/TraceableElementTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/service/track/TraceableElementTest.kt @@ -67,7 +67,7 @@ class TraceableElementTest { fun testOneMaxIndividualWithFT(){ config.enableTrackIndividual = false config.enableTrackEvaluatedIndividual = true - config.maxActionEvaluations = 100 + config.maxEvaluations = 100 config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS config.maxLengthOfTraces = 20 config.probOfArchiveMutation = 0.0 diff --git a/docs/options.md b/docs/options.md index ef233d2016..a4434904e3 100644 --- a/docs/options.md +++ b/docs/options.md @@ -134,8 +134,8 @@ There are 3 types of options: |`labelForExperimentConfigs`| __String__. Further label to represent the names of CONFIGS sets in experiment scripts, e.g., exp.py. *Default value*: `-`.| |`labelForExperiments`| __String__. When running experiments and statistic files are generated, all configs are saved. So, this one can be used as extra label for classifying the experiment. *Default value*: `-`.| |`lastLineEpsilon`| __Double__. The Distance Metric Last Line may use several values for epsilon.During experimentation, it may be useful to adjust these values. Epsilon describes the size of the neighbourhood used for clustering, so may result in different clustering results.Epsilon should be between 0.0 and 1.0. If the value is outside of that range, epsilon will use the default of 0.8. *Constraints*: `min=0.0, max=1.0`. *Default value*: `0.8`.| -|`maxActionEvaluations`| __Int__. Maximum number of action evaluations for the search. A fitness evaluation can be composed of 1 or more actions, like for example REST calls or SQL setups. The more actions are allowed, the better results one can expect. But then of course the test generation will take longer. Only applicable depending on the stopping criterion. *Constraints*: `min=1.0`. *Default value*: `1000`.| |`maxAssertionForDataInCollection`| __Int__. Specify a maximum number of data in a collection to be asserted in the generated tests. Note that zero means that only the size of the collection will be asserted. A negative value means all data in the collection will be asserted (i.e., no limit). *Default value*: `3`.| +|`maxEvaluations`| __Int__. Maximum number of action or individual evaluations (depending on chosen stopping criterion) for the search. A fitness evaluation can be composed of 1 or more actions, like for example REST calls or SQL setups. The more actions are allowed, the better results one can expect. But then of course the test generation will take longer. Only applicable depending on the stopping criterion. *Constraints*: `min=1.0`. *Default value*: `1000`.| |`maxLengthForStrings`| __Int__. The maximum length allowed for evolved strings. Without this limit, strings could in theory be billions of characters long. *Constraints*: `min=0.0, max=20000.0`. *Default value*: `200`.| |`maxLengthForStringsAtSamplingTime`| __Int__. Maximum length when sampling a new random string. Such limit can be bypassed when a string is mutated. *Constraints*: `min=0.0`. *Default value*: `16`.| |`maxLengthOfTraces`| __Int__. Specify a maxLength of tracking when enableTrackIndividual or enableTrackEvaluatedIndividual is true. Note that the value should be specified with a non-negative number or -1 (for tracking all history). *Constraints*: `min=-1.0`. *Default value*: `10`.| @@ -190,7 +190,7 @@ There are 3 types of options: |`startingPerOfGenesToMutate`| __Double__. Specify a starting percentage of genes of an individual to mutate. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| |`statisticsColumnId`| __String__. An id that will be part as a column of the statistics file (if any is generated). *Default value*: `-`.| |`statisticsFile`| __String__. Where the statistics file (if any) is going to be written (in CSV format). *Default value*: `statistics.csv`.| -|`stoppingCriterion`| __Enum__. Stopping criterion for the search. *Valid values*: `TIME, ACTION_EVALUATIONS`. *Default value*: `TIME`.| +|`stoppingCriterion`| __Enum__. Stopping criterion for the search. *Valid values*: `TIME, ACTION_EVALUATIONS, INDIVIDUAL_EVALUATIONS`. *Default value*: `TIME`.| |`structureMutationProbability`| __Double__. Probability of applying a mutation that can change the structure of a test. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| |`sutControllerHost`| __String__. Host name or IP address of where the SUT REST controller is listening on. *Default value*: `localhost`.| |`sutControllerPort`| __Int__. TCP port of where the SUT REST controller is listening on. *Constraints*: `min=0.0, max=65535.0`. *Default value*: `40100`.| diff --git a/e2e-tests/dropwizard-examples/src/test/java/org/evomaster/e2etests/dw/examples/positiveinteger/PIEMTest.java b/e2e-tests/dropwizard-examples/src/test/java/org/evomaster/e2etests/dw/examples/positiveinteger/PIEMTest.java index a5e97bcda7..8cc24a173e 100644 --- a/e2e-tests/dropwizard-examples/src/test/java/org/evomaster/e2etests/dw/examples/positiveinteger/PIEMTest.java +++ b/e2e-tests/dropwizard-examples/src/test/java/org/evomaster/e2etests/dw/examples/positiveinteger/PIEMTest.java @@ -19,7 +19,7 @@ public void testRunEM(){ "--createTests", "false", "--seed", "42", "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "200", + "--maxEvaluations", "200", "--stoppingCriterion", "ACTION_EVALUATIONS" }; @@ -38,7 +38,7 @@ public void testCreateTest() { "--createTests", "true", "--seed", "42", "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "20", + "--maxEvaluations", "20", "--stoppingCriterion", "ACTION_EVALUATIONS" }; diff --git a/e2e-tests/dropwizard-examples/src/test/java/org/evomaster/e2etests/dw/examples/simpleform/SFTest.java b/e2e-tests/dropwizard-examples/src/test/java/org/evomaster/e2etests/dw/examples/simpleform/SFTest.java index d9c34aabbd..6c614e816f 100644 --- a/e2e-tests/dropwizard-examples/src/test/java/org/evomaster/e2etests/dw/examples/simpleform/SFTest.java +++ b/e2e-tests/dropwizard-examples/src/test/java/org/evomaster/e2etests/dw/examples/simpleform/SFTest.java @@ -29,7 +29,7 @@ public void testRunEM() throws Throwable { "--createTests", "false", "--seed", "" + defaultSeed++, "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "500", + "--maxEvaluations", "500", "--stoppingCriterion", "ACTION_EVALUATIONS" }; diff --git a/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java b/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java index 4263d9ca82..1a92f3be41 100644 --- a/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java +++ b/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java @@ -153,7 +153,7 @@ protected void runAndCheckDeterminism(int iterations, Consumer> lam "--showProgress", "false", "--avoidNonDeterministicLogs", "true", "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "" + iterations, + "--maxEvaluations", "" + iterations, "--stoppingCriterion", "ACTION_EVALUATIONS", "--useTimeInFeedbackSampling" , "false", "--createConfigPathIfMissing", "false" @@ -416,7 +416,7 @@ protected List getArgsWithCompilation(int iterations, String outputFolde "--seed", "" + defaultSeed, "--useTimeInFeedbackSampling" , "false", "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "" + iterations, + "--maxEvaluations", "" + iterations, "--stoppingCriterion", "ACTION_EVALUATIONS", "--outputFolder", outputFolderPath(outputFolderName), "--outputFormat", OutputFormat.KOTLIN_JUNIT_5.toString(), diff --git a/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/RestTestBase.java b/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/RestTestBase.java index 0619ad4e10..c9863f93c6 100644 --- a/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/RestTestBase.java +++ b/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/RestTestBase.java @@ -43,7 +43,7 @@ protected String runAndCheckDeterminism(int iterations, Consumer> l "--showProgress", "false", "--avoidNonDeterministicLogs", "true", "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "" + iterations, + "--maxEvaluations", "" + iterations, "--stoppingCriterion", "ACTION_EVALUATIONS", "--useTimeInFeedbackSampling" , "false", "--createConfigPathIfMissing", "false" diff --git a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/adaptivehypermutation/DeterminismTest.java b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/adaptivehypermutation/DeterminismTest.java index 680ec765b1..bfee0075cd 100644 --- a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/adaptivehypermutation/DeterminismTest.java +++ b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/adaptivehypermutation/DeterminismTest.java @@ -66,7 +66,7 @@ public void testNotDeterminismMIO() { "--showProgress", "false", "--avoidNonDeterministicLogs", "true", "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "" + 4000, + "--maxEvaluations", "" + 4000, "--stoppingCriterion", "ACTION_EVALUATIONS", "--useTimeInFeedbackSampling" , "false" )); diff --git a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java index cfd2ba5b20..bc1b658c74 100644 --- a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java +++ b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java @@ -100,7 +100,7 @@ public void testSteps() { "--createTests", "true", "--seed", "42", "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "1", + "--maxEvaluations", "1", "--stoppingCriterion", "ACTION_EVALUATIONS", "--heuristicsForSQL", "true", "--generateSqlDataWithSearch", "true", diff --git a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/javatypes/JavaTypesEMTest.java b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/javatypes/JavaTypesEMTest.java index 036f80acb3..ff719d7f5b 100644 --- a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/javatypes/JavaTypesEMTest.java +++ b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/javatypes/JavaTypesEMTest.java @@ -43,7 +43,7 @@ public void testRunEMWithSQL() throws Throwable { "--createTests", "true", "--seed", "" + defaultSeed++, "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "3000", + "--maxEvaluations", "3000", "--stoppingCriterion", "ACTION_EVALUATIONS", "--heuristicsForSQL", "true", "--generateSqlDataWithSearch", "true" diff --git a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/resource/ResourceMIOHWTestBase.java b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/resource/ResourceMIOHWTestBase.java index 594ccc97cf..6303a2834e 100644 --- a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/resource/ResourceMIOHWTestBase.java +++ b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/resource/ResourceMIOHWTestBase.java @@ -79,7 +79,7 @@ protected List generalArgs(int budget, int seed){ "--seed", ""+seed, "--useTimeInFeedbackSampling", "false", "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "" + budget, + "--maxEvaluations", "" + budget, "--stoppingCriterion", "ACTION_EVALUATIONS", //there some bugs here "--baseTaintAnalysisProbability", "0.0" diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/adaptivehypermutation/DeterminismTest.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/adaptivehypermutation/DeterminismTest.java index 07d4aea8a5..52967c7e72 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/adaptivehypermutation/DeterminismTest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/adaptivehypermutation/DeterminismTest.java @@ -66,7 +66,7 @@ public void testNotDeterminismMIO() { "--showProgress", "false", "--avoidNonDeterministicLogs", "true", "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "" + 4000, + "--maxEvaluations", "" + 4000, "--stoppingCriterion", "ACTION_EVALUATIONS", "--useTimeInFeedbackSampling" , "false" )); diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java index 9396c47f67..24619b0bba 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java @@ -102,7 +102,7 @@ public void testSteps() { "--createTests", "true", "--seed", "42", "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "1", + "--maxEvaluations", "1", "--stoppingCriterion", "ACTION_EVALUATIONS", "--heuristicsForSQL", "true", "--generateSqlDataWithSearch", "true", diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/javatypes/JavaTypesEMTest.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/javatypes/JavaTypesEMTest.java index 036f80acb3..ff719d7f5b 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/javatypes/JavaTypesEMTest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/javatypes/JavaTypesEMTest.java @@ -43,7 +43,7 @@ public void testRunEMWithSQL() throws Throwable { "--createTests", "true", "--seed", "" + defaultSeed++, "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "3000", + "--maxEvaluations", "3000", "--stoppingCriterion", "ACTION_EVALUATIONS", "--heuristicsForSQL", "true", "--generateSqlDataWithSearch", "true" diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/resource/ResourceMIOHWTestBase.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/resource/ResourceMIOHWTestBase.java index 389870ad5a..e6cb0a6290 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/resource/ResourceMIOHWTestBase.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/resource/ResourceMIOHWTestBase.java @@ -76,7 +76,7 @@ protected List generalArgs(int budget, int seed){ "--seed", ""+seed, "--useTimeInFeedbackSampling", "false", "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "" + budget, + "--maxEvaluations", "" + budget, "--stoppingCriterion", "ACTION_EVALUATIONS", //there some bugs here "--baseTaintAnalysisProbability", "0.0" diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/wiremock/service/ExternalServiceMockingEMTest.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/wiremock/service/ExternalServiceMockingEMTest.java index b680e6f436..02c5f4b926 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/wiremock/service/ExternalServiceMockingEMTest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/wiremock/service/ExternalServiceMockingEMTest.java @@ -37,7 +37,7 @@ public void externalServiceSuccessTest() throws Throwable { "--createTests", "false", "--seed", "42", "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "1", + "--maxEvaluations", "1", "--stoppingCriterion", "ACTION_EVALUATIONS", "--executiveSummary", "false", "--expectationsActive" , "true", diff --git a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/wiremock/hostnameaction/HostnameResolutionActionEMTest.kt b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/wiremock/hostnameaction/HostnameResolutionActionEMTest.kt index 071b00a461..4dec11bc8e 100644 --- a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/wiremock/hostnameaction/HostnameResolutionActionEMTest.kt +++ b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/wiremock/hostnameaction/HostnameResolutionActionEMTest.kt @@ -64,7 +64,7 @@ class HostnameResolutionActionEMTest: SpringTestBase() { "--createTests", "false", "--seed", "42", "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "1", + "--maxEvaluations", "1", "--stoppingCriterion", "ACTION_EVALUATIONS", "--executiveSummary", "false", "--expectationsActive", "true", diff --git a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/wiremock/hostnameaction/HostnameResolutionActionManualTest.kt b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/wiremock/hostnameaction/HostnameResolutionActionManualTest.kt index 9866d8d59d..0ae4df0e8e 100644 --- a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/wiremock/hostnameaction/HostnameResolutionActionManualTest.kt +++ b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/wiremock/hostnameaction/HostnameResolutionActionManualTest.kt @@ -29,7 +29,7 @@ class HostnameResolutionActionManualTest: SpringTestBase() { "--createTests", "false", "--seed", "42", "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "1", + "--maxEvaluations", "1", "--stoppingCriterion", "ACTION_EVALUATIONS", "--executiveSummary", "false", "--expectationsActive", "true", diff --git a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/wiremock/skip/SkipEMTest.kt b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/wiremock/skip/SkipEMTest.kt index 1545fa5cc0..0326e0d7b2 100644 --- a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/wiremock/skip/SkipEMTest.kt +++ b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/wiremock/skip/SkipEMTest.kt @@ -30,7 +30,7 @@ class SkipEMTest : SpringTestBase() { "--createTests", "false", "--seed", "42", "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "1", + "--maxEvaluations", "1", "--stoppingCriterion", "ACTION_EVALUATIONS", "--executiveSummary", "false", "--expectationsActive", "true", diff --git a/e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/numericstring/RPCSchemaHandlerTest.java b/e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/numericstring/RPCSchemaHandlerTest.java index d2de4b6ae1..ebb99a3271 100644 --- a/e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/numericstring/RPCSchemaHandlerTest.java +++ b/e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/numericstring/RPCSchemaHandlerTest.java @@ -30,7 +30,7 @@ public void test(){ "--showProgress", "false", "--avoidNonDeterministicLogs", "true", "--sutControllerPort", "" + controllerPort, - "--maxActionEvaluations", "" + 5, + "--maxEvaluations", "" + 5, "--stoppingCriterion", "ACTION_EVALUATIONS", "--useTimeInFeedbackSampling" , "false" )); diff --git a/scripts/exp.py b/scripts/exp.py index fc9bc1abb3..e097a2eb9e 100644 --- a/scripts/exp.py +++ b/scripts/exp.py @@ -771,7 +771,7 @@ def addJobBody(port, sut, seed, setting, configName): params += " --maxTime=" + BUDGET else: params += " --stoppingCriterion=ACTION_EVALUATIONS" - params += " --maxActionEvaluations=" + BUDGET + params += " --maxEvaluations=" + BUDGET params += " --statisticsColumnId=" + sut.name params += " --seed=" + str(seed)