Merge pull request #1111 from WebFuzzing/bb-improvements-activate

activating new BB-related features

core/src/main/kotlin/org/evomaster/core/EMConfig.kt

@@ -78,7 +78,7 @@ class EMConfig {
 
         private const val externalServiceIPRegex = "$_eip_n$_eip_s$_eip_e"
 
-        private val defaultAlgorithmForBlackBox = Algorithm.RANDOM
+        private val defaultAlgorithmForBlackBox = Algorithm.SMARTS
 
         private val defaultAlgorithmForWhiteBox = Algorithm.MIO
 
@@ -2262,12 +2262,11 @@ class EMConfig {
     @Probability(true)
     var probRestExamples = 0.20
 
-    @Experimental
     @Cfg("In REST, enable the supports of 'links' between resources defined in the OpenAPI schema, if any." +
             " When sampling a test case, if the last call has links, given this probability new calls are" +
             " added for the link.")
     @Probability(true)
-    var probUseRestLinks = 0.0
+    var probUseRestLinks = 0.5
 
     //TODO mark as deprecated once we support proper Robustness Testing
     @Cfg("When generating data, allow in some cases to use invalid values on purpose")
@@ -2291,9 +2290,8 @@ class EMConfig {
     @Cfg("Validate responses against their schema, to check for inconsistencies. Those are treated as faults.")
     var schemaOracles = true
 
-    @Experimental
     @Cfg("Apply more advanced coverage criteria for black-box testing. This can result in larger generated test suites.")
-    var advancedBlackBoxCoverage = false
+    var advancedBlackBoxCoverage = true
 
     fun timeLimitInSeconds(): Int {
         if (maxTimeInSeconds > 0) {
@@ -2341,9 +2339,8 @@ class EMConfig {
     @Min(0.0)
     var thresholdDistanceForDataPool = 2
 
-    @Experimental
     @Cfg("Enable the collection of response data, to feed new individuals based on field names matching.")
-    var useResponseDataPool = false
+    var useResponseDataPool = true
 
     @Experimental
     @Probability(false)

docs/options.md

@@ -62,6 +62,7 @@ There are 3 types of options:
 |`SMdR`| __Double__. Specify a probability to apply SMdR when resource sampling strategy is 'Customized'. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.25`.|
 |`adaptiveGeneSelectionMethod`| __Enum__. Specify a strategy to select genes for mutation adaptively. *Valid values*: `NONE, AWAY_NOIMPACT, APPROACH_IMPACT, APPROACH_LATEST_IMPACT, APPROACH_LATEST_IMPROVEMENT, BALANCE_IMPACT_NOIMPACT, BALANCE_IMPACT_NOIMPACT_WITH_E, ALL_FIXED_RAND`. *Default value*: `APPROACH_IMPACT`.|
 |`addPreDefinedTests`| __Boolean__. Add predefined tests at the end of the search. An example is a test to fetch the schema of RESTful APIs. *Default value*: `true`.|
+|`advancedBlackBoxCoverage`| __Boolean__. Apply more advanced coverage criteria for black-box testing. This can result in larger generated test suites. *Default value*: `true`.|
 |`algorithm`| __Enum__. The algorithm used to generate test cases. The default depends on whether black-box or white-box testing is done. *Valid values*: `DEFAULT, SMARTS, MIO, RANDOM, WTS, MOSA`. *Default value*: `DEFAULT`.|
 |`allowInvalidData`| __Boolean__. When generating data, allow in some cases to use invalid values on purpose. *Default value*: `true`.|
 |`appendToStatisticsFile`| __Boolean__. Whether should add to an existing statistics file, instead of replacing it. *Default value*: `false`.|
@@ -163,6 +164,7 @@ There are 3 types of options:
 |`probOfSmartSampling`| __Double__. When sampling new test cases to evaluate, probability of using some smart strategy instead of plain random. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.95`.|
 |`probRestDefault`| __Double__. In REST, specify probability of using 'default' values, if any is specified in the schema. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.05`.|
 |`probRestExamples`| __Double__. In REST, specify probability of using 'example(s)' values, if any is specified in the schema. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.2`.|
+|`probUseRestLinks`| __Double__. In REST, enable the supports of 'links' between resources defined in the OpenAPI schema, if any. When sampling a test case, if the last call has links, given this probability new calls are added for the link. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.|
 |`problemType`| __Enum__. The type of SUT we want to generate tests for, e.g., a RESTful API. If left to DEFAULT, the type will be inferred from the EM Driver. However, in case of ambiguities (e.g., the driver specifies more than one type), then this field must be set with a specific type. This is also the case for Black-Box testing where there is no EM Driver. In this latter case, the system defaults to handle REST APIs. *Valid values*: `DEFAULT, REST, GRAPHQL`. *Experimental values*: `RPC, WEBFRONTEND`. *Default value*: `DEFAULT`.|
 |`processFiles`| __String__. Specify a folder to save results when a search monitor is enabled. *DEBUG option*. *Default value*: `process_data`.|
 |`processFormat`| __Enum__. Specify a format to save the process data. *DEBUG option*. *Valid values*: `JSON_ALL, TEST_IND, TARGET_TEST_IND`. *Default value*: `JSON_ALL`.|
@@ -204,6 +206,7 @@ There are 3 types of options:
 |`useExtraSqlDbConstraintsProbability`| __Double__. Whether to analyze how SQL databases are accessed to infer extra constraints from the business logic. An example is javax/jakarta annotation constraints defined on JPA entities. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.9`.|
 |`useMethodReplacement`| __Boolean__. Apply method replacement heuristics to smooth the search landscape. Note that the method replacement instrumentations would still be applied, it is just that their testing targets will be ignored in the fitness function if this option is set to false. *Default value*: `true`.|
 |`useNonIntegerReplacement`| __Boolean__. Apply non-integer numeric comparison heuristics to smooth the search landscape. *Default value*: `true`.|
+|`useResponseDataPool`| __Boolean__. Enable the collection of response data, to feed new individuals based on field names matching. *Default value*: `true`.|
 |`useTimeInFeedbackSampling`| __Boolean__. Whether to use timestamp info on the execution time of the tests for sampling (e.g., to reward the quickest ones). *Default value*: `true`.|
 |`weightBasedMutationRate`| __Boolean__. Whether to enable a weight-based mutation rate. *Default value*: `true`.|
 |`writeExtraHeuristicsFile`| __Boolean__. Whether we should collect data on the extra heuristics. Only needed for experiments. *Default value*: `false`.|
@@ -215,7 +218,6 @@ There are 3 types of options:
 |Options|Description|
 |---|---|
 |`abstractInitializationGeneToMutate`| __Boolean__. During mutation, whether to abstract genes for repeated SQL actions. *Default value*: `false`.|
-|`advancedBlackBoxCoverage`| __Boolean__. Apply more advanced coverage criteria for black-box testing. This can result in larger generated test suites. *Default value*: `false`.|
 |`bbProbabilityUseDataPool`| __Double__. Specify the probability of using the data pool when sampling test cases. This is for black-box (bb) mode. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.8`.|
 |`discoveredInfoRewardedInFitness`| __Boolean__. If there is new discovered information from a test execution, reward it in the fitness function. *Default value*: `false`.|
 |`dpcTargetTestSize`| __Int__. Specify a max size of a test to be targeted when either DPC_INCREASING or DPC_DECREASING is enabled. *Default value*: `1`.|
@@ -250,7 +252,6 @@ There are 3 types of options:
 |`probOfMutatingResponsesBasedOnActualResponse`| __Double__. a probability of mutating mocked responses based on actual responses. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.|
 |`probOfPrioritizingSuccessfulHarvestedActualResponses`| __Double__. a probability of prioritizing to employ successful harvested actual responses from external services as seeds (e.g., 2xx from HTTP external service). *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.|
 |`probOfSmartInitStructureMutator`| __Double__. Specify a probability of applying a smart structure mutator for initialization of the individual. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.|
-|`probUseRestLinks`| __Double__. In REST, enable the supports of 'links' between resources defined in the OpenAPI schema, if any. When sampling a test case, if the last call has links, given this probability new calls are added for the link. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.|
 |`saveMockedResponseAsSeparatedFile`| __Boolean__. Whether to save mocked responses as separated files. *Default value*: `false`.|
 |`security`| __Boolean__. Apply a security testing phase after functional test cases have been generated. *Default value*: `false`.|
 |`seedTestCases`| __Boolean__. Whether to seed EvoMaster with some initial test cases. These test cases will be used and evolved throughout the search process. *Default value*: `false`.|
@@ -263,7 +264,6 @@ There are 3 types of options:
 |`thresholdDistanceForDataPool`| __Int__. Threshold of Levenshtein Distance for key-matching in Data Pool. *Constraints*: `min=0.0`. *Default value*: `2`.|
 |`useGlobalTaintInfoProbability`| __Double__. When sampling new individual, check whether to use already existing info on tainted values. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.|
 |`useInsertionForSqlHeuristics`| __Boolean__. Specify whether insertions should be used to calculate SQL heuristics instead of retrieving data from real databases. *Default value*: `false`.|
-|`useResponseDataPool`| __Boolean__. Enable the collection of response data, to feed new individuals based on field names matching. *Default value*: `false`.|
 |`useWeightedSampling`| __Boolean__. When sampling from archive based on targets, decide whether to use weights based on properties of the targets (e.g., a target likely leading to a flag will be sampled less often). *Default value*: `false`.|
 |`wbProbabilityUseDataPool`| __Double__. Specify the probability of using the data pool when sampling test cases. This is for white-box (wb) mode. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.2`.|
 |`writeSnapshotTestsIntervalInSeconds`| __Int__. The size (in seconds) of the interval that the snapshots will be printed, if enabled. *Default value*: `3600`.|

e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/endpointfocusandprefix/EndpointFocusAndPrefixTest.java

@@ -163,6 +163,7 @@ public void testRunBlackboxWithFocusWithParameters() throws Throwable {
                     args.add(baseUrlOfSut + "/v2/api-docs");
                     args.add("--endpointFocus");
                     args.add(endpointFocus);
+                    setOption(args, "advancedBlackBoxCoverage", "false");
 
                     // no endpointFocus or endpointPrefix is provided
                     Solution<RestIndividual> solution = initAndRun(args);
@@ -174,7 +175,7 @@ public void testRunBlackboxWithFocusWithParameters() throws Throwable {
                     assertAllSolutionsHavePathFocusOrPrefixList(solution, pathsToCheck, true);
 
                     // The solution should include 4 solutions, 3 endpoints and 1 failure case
-                    assertEquals(solution.getIndividuals().size(), 4);
+                    assertEquals(4, solution.getIndividuals().size());
 
                     // write test into the output folder
                     compile(outputFolder);
@@ -283,6 +284,7 @@ public void testRunBlackboxWithPrefixWithParameters() throws Throwable {
                     args.add(baseUrlOfSut + "/v2/api-docs");
                     args.add("--endpointPrefix");
                     args.add(endpointPrefix);
+                    setOption(args, "advancedBlackBoxCoverage", "false");
 
                     // no endpointFocus or endpointPrefix is provided
                     Solution<RestIndividual> solution = initAndRun(args);
@@ -294,7 +296,7 @@ public void testRunBlackboxWithPrefixWithParameters() throws Throwable {
                     assertAllSolutionsHavePathFocusOrPrefixList(solution, pathsToCheck, false);
 
                     // The solution should include 5 solutions, 4 endpoints and 1 failure case
-                    assertEquals(solution.getIndividuals().size(), 5);
+                    assertEquals(5, solution.getIndividuals().size());
 
                     // write test into the output folder
                     compile(outputFolder);
@@ -323,6 +325,7 @@ public void testRunBlackboxPrefixNonExistingFocusValidPrefix() throws Throwable
                     args.add(baseUrlOfSut + "/v2/api-docs");
                     args.add("--endpointPrefix");
                     args.add(endpointPrefix);
+                    setOption(args, "advancedBlackBoxCoverage", "false");
 
                     // no endpointFocus or endpointPrefix is provided
                     Solution<RestIndividual> solution = initAndRun(args);
@@ -334,7 +337,7 @@ public void testRunBlackboxPrefixNonExistingFocusValidPrefix() throws Throwable
                     assertAllSolutionsHavePathFocusOrPrefixList(solution, pathsToCheck, false);
 
                     // The solution should include 5 solutions, 4 endpoints and 1 failure case
-                    assertEquals(solution.getIndividuals().size(), 5);
+                    assertEquals(5, solution.getIndividuals().size());
 
                     // write test into the output folder
                     compile(outputFolder);

e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/hypermutation/AdaptiveHypermutationTest.java

@@ -48,7 +48,8 @@ public void testRunAdaptiveHypermutation() throws Throwable {
                     //minimization loses impact info
                     args.add("--minimize");
                     args.add("false");
-
+                    //this had side-effects
+                    setOption(args, "advancedBlackBoxCoverage", "false");
 
                     Solution<RestIndividual> solution = initAndRun(args);
 

e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/bodyundefined/BodyUndefinedEMTest.kt

@@ -25,6 +25,9 @@ class BodyUndefinedEMTest : SpringTestBase() {
                 20
         ) { args: MutableList<String> ->
 
+            //TODO remove once fixed issue
+            setOption(args, "advancedBlackBoxCoverage", "false")
+
             val solution = initAndRun(args)
 
             Assertions.assertTrue(solution.individuals.size >= 1)
@@ -35,7 +38,13 @@ class BodyUndefinedEMTest : SpringTestBase() {
                 There is some weird bug in Jersey that it looks like it transform the GET into a POST ?!?
                 When we upgrade Jersey (once moving ot JDK 11), need to fix AbstractRestFitness and RestActionBuilderV3.
                 Then, we will need to check if this fails, and if so, change
-                into a 400 and 200 instead of 415
+                into a 400 and 200 instead of 415.
+
+                Even more weird, need to deactivate advancedBB coverage, otherwise generated tests
+                add a .body(" {} ") to the GET.
+                in Jersey, this still give a 415, but then a different code in RestAssured when compiled
+                tests are executed... WTF!!!
+                TODO look into this once upgraded Jersey
              */
             assertHasAtLeastOne(solution, HttpVerb.GET, 415, "/api/bodyundefined", null)
         }

release_notes.md

@@ -6,6 +6,9 @@ Under development in `master` branch.
 
 - MongoDB support. For white-box heuristics, can analyze all queries done toward MongoDB databases, as well as being able to insert data directly as part of the generated test cases.
 - improved fault detection for OpenAPI schema faults, in particular regarding the structure of the received responses, which are now validated. 
+- improved coverage criteria for black-box testing for REST APIs.
+- support for exploiting "links" declarations in OpenAPI schemas. 
+- improved re-used of data between endpoints (e.g., data returned from GET requests can be used as input for following requests using fields with similar names). 
 
 ### Bug Fixes