From d41efd010747bd70c5eeb26ee436984a216b522a Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Thu, 24 Oct 2024 21:17:43 +0200 Subject: [PATCH 1/2] activating new BB-related features --- core/src/main/kotlin/org/evomaster/core/EMConfig.kt | 11 ++++------- docs/options.md | 6 +++--- release_notes.md | 3 +++ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 5775793ddf..1fc476f58f 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -78,7 +78,7 @@ class EMConfig { private const val externalServiceIPRegex = "$_eip_n$_eip_s$_eip_e" - private val defaultAlgorithmForBlackBox = Algorithm.RANDOM + private val defaultAlgorithmForBlackBox = Algorithm.SMARTS private val defaultAlgorithmForWhiteBox = Algorithm.MIO @@ -2263,12 +2263,11 @@ class EMConfig { @Probability(true) var probRestExamples = 0.20 - @Experimental @Cfg("In REST, enable the supports of 'links' between resources defined in the OpenAPI schema, if any." + " When sampling a test case, if the last call has links, given this probability new calls are" + " added for the link.") @Probability(true) - var probUseRestLinks = 0.0 + var probUseRestLinks = 0.5 //TODO mark as deprecated once we support proper Robustness Testing @Cfg("When generating data, allow in some cases to use invalid values on purpose") @@ -2292,9 +2291,8 @@ class EMConfig { @Cfg("Validate responses against their schema, to check for inconsistencies. Those are treated as faults.") var schemaOracles = true - @Experimental @Cfg("Apply more advanced coverage criteria for black-box testing. This can result in larger generated test suites.") - var advancedBlackBoxCoverage = false + var advancedBlackBoxCoverage = true fun timeLimitInSeconds(): Int { if (maxTimeInSeconds > 0) { @@ -2342,9 +2340,8 @@ class EMConfig { @Min(0.0) var thresholdDistanceForDataPool = 2 - @Experimental @Cfg("Enable the collection of response data, to feed new individuals based on field names matching.") - var useResponseDataPool = false + var useResponseDataPool = true @Experimental @Probability(false) diff --git a/docs/options.md b/docs/options.md index 7df4213bc1..ef233d2016 100644 --- a/docs/options.md +++ b/docs/options.md @@ -62,6 +62,7 @@ There are 3 types of options: |`SMdR`| __Double__. Specify a probability to apply SMdR when resource sampling strategy is 'Customized'. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.25`.| |`adaptiveGeneSelectionMethod`| __Enum__. Specify a strategy to select genes for mutation adaptively. *Valid values*: `NONE, AWAY_NOIMPACT, APPROACH_IMPACT, APPROACH_LATEST_IMPACT, APPROACH_LATEST_IMPROVEMENT, BALANCE_IMPACT_NOIMPACT, BALANCE_IMPACT_NOIMPACT_WITH_E, ALL_FIXED_RAND`. *Default value*: `APPROACH_IMPACT`.| |`addPreDefinedTests`| __Boolean__. Add predefined tests at the end of the search. An example is a test to fetch the schema of RESTful APIs. *Default value*: `true`.| +|`advancedBlackBoxCoverage`| __Boolean__. Apply more advanced coverage criteria for black-box testing. This can result in larger generated test suites. *Default value*: `true`.| |`algorithm`| __Enum__. The algorithm used to generate test cases. The default depends on whether black-box or white-box testing is done. *Valid values*: `DEFAULT, SMARTS, MIO, RANDOM, WTS, MOSA`. *Default value*: `DEFAULT`.| |`allowInvalidData`| __Boolean__. When generating data, allow in some cases to use invalid values on purpose. *Default value*: `true`.| |`appendToStatisticsFile`| __Boolean__. Whether should add to an existing statistics file, instead of replacing it. *Default value*: `false`.| @@ -163,6 +164,7 @@ There are 3 types of options: |`probOfSmartSampling`| __Double__. When sampling new test cases to evaluate, probability of using some smart strategy instead of plain random. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.95`.| |`probRestDefault`| __Double__. In REST, specify probability of using 'default' values, if any is specified in the schema. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.05`.| |`probRestExamples`| __Double__. In REST, specify probability of using 'example(s)' values, if any is specified in the schema. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.2`.| +|`probUseRestLinks`| __Double__. In REST, enable the supports of 'links' between resources defined in the OpenAPI schema, if any. When sampling a test case, if the last call has links, given this probability new calls are added for the link. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| |`problemType`| __Enum__. The type of SUT we want to generate tests for, e.g., a RESTful API. If left to DEFAULT, the type will be inferred from the EM Driver. However, in case of ambiguities (e.g., the driver specifies more than one type), then this field must be set with a specific type. This is also the case for Black-Box testing where there is no EM Driver. In this latter case, the system defaults to handle REST APIs. *Valid values*: `DEFAULT, REST, GRAPHQL`. *Experimental values*: `RPC, WEBFRONTEND`. *Default value*: `DEFAULT`.| |`processFiles`| __String__. Specify a folder to save results when a search monitor is enabled. *DEBUG option*. *Default value*: `process_data`.| |`processFormat`| __Enum__. Specify a format to save the process data. *DEBUG option*. *Valid values*: `JSON_ALL, TEST_IND, TARGET_TEST_IND`. *Default value*: `JSON_ALL`.| @@ -204,6 +206,7 @@ There are 3 types of options: |`useExtraSqlDbConstraintsProbability`| __Double__. Whether to analyze how SQL databases are accessed to infer extra constraints from the business logic. An example is javax/jakarta annotation constraints defined on JPA entities. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.9`.| |`useMethodReplacement`| __Boolean__. Apply method replacement heuristics to smooth the search landscape. Note that the method replacement instrumentations would still be applied, it is just that their testing targets will be ignored in the fitness function if this option is set to false. *Default value*: `true`.| |`useNonIntegerReplacement`| __Boolean__. Apply non-integer numeric comparison heuristics to smooth the search landscape. *Default value*: `true`.| +|`useResponseDataPool`| __Boolean__. Enable the collection of response data, to feed new individuals based on field names matching. *Default value*: `true`.| |`useTimeInFeedbackSampling`| __Boolean__. Whether to use timestamp info on the execution time of the tests for sampling (e.g., to reward the quickest ones). *Default value*: `true`.| |`weightBasedMutationRate`| __Boolean__. Whether to enable a weight-based mutation rate. *Default value*: `true`.| |`writeExtraHeuristicsFile`| __Boolean__. Whether we should collect data on the extra heuristics. Only needed for experiments. *Default value*: `false`.| @@ -215,7 +218,6 @@ There are 3 types of options: |Options|Description| |---|---| |`abstractInitializationGeneToMutate`| __Boolean__. During mutation, whether to abstract genes for repeated SQL actions. *Default value*: `false`.| -|`advancedBlackBoxCoverage`| __Boolean__. Apply more advanced coverage criteria for black-box testing. This can result in larger generated test suites. *Default value*: `false`.| |`bbProbabilityUseDataPool`| __Double__. Specify the probability of using the data pool when sampling test cases. This is for black-box (bb) mode. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.8`.| |`discoveredInfoRewardedInFitness`| __Boolean__. If there is new discovered information from a test execution, reward it in the fitness function. *Default value*: `false`.| |`dpcTargetTestSize`| __Int__. Specify a max size of a test to be targeted when either DPC_INCREASING or DPC_DECREASING is enabled. *Default value*: `1`.| @@ -250,7 +252,6 @@ There are 3 types of options: |`probOfMutatingResponsesBasedOnActualResponse`| __Double__. a probability of mutating mocked responses based on actual responses. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.| |`probOfPrioritizingSuccessfulHarvestedActualResponses`| __Double__. a probability of prioritizing to employ successful harvested actual responses from external services as seeds (e.g., 2xx from HTTP external service). *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.| |`probOfSmartInitStructureMutator`| __Double__. Specify a probability of applying a smart structure mutator for initialization of the individual. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.| -|`probUseRestLinks`| __Double__. In REST, enable the supports of 'links' between resources defined in the OpenAPI schema, if any. When sampling a test case, if the last call has links, given this probability new calls are added for the link. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.| |`saveMockedResponseAsSeparatedFile`| __Boolean__. Whether to save mocked responses as separated files. *Default value*: `false`.| |`security`| __Boolean__. Apply a security testing phase after functional test cases have been generated. *Default value*: `false`.| |`seedTestCases`| __Boolean__. Whether to seed EvoMaster with some initial test cases. These test cases will be used and evolved throughout the search process. *Default value*: `false`.| @@ -263,7 +264,6 @@ There are 3 types of options: |`thresholdDistanceForDataPool`| __Int__. Threshold of Levenshtein Distance for key-matching in Data Pool. *Constraints*: `min=0.0`. *Default value*: `2`.| |`useGlobalTaintInfoProbability`| __Double__. When sampling new individual, check whether to use already existing info on tainted values. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.| |`useInsertionForSqlHeuristics`| __Boolean__. Specify whether insertions should be used to calculate SQL heuristics instead of retrieving data from real databases. *Default value*: `false`.| -|`useResponseDataPool`| __Boolean__. Enable the collection of response data, to feed new individuals based on field names matching. *Default value*: `false`.| |`useWeightedSampling`| __Boolean__. When sampling from archive based on targets, decide whether to use weights based on properties of the targets (e.g., a target likely leading to a flag will be sampled less often). *Default value*: `false`.| |`wbProbabilityUseDataPool`| __Double__. Specify the probability of using the data pool when sampling test cases. This is for white-box (wb) mode. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.2`.| |`writeSnapshotTestsIntervalInSeconds`| __Int__. The size (in seconds) of the interval that the snapshots will be printed, if enabled. *Default value*: `3600`.| diff --git a/release_notes.md b/release_notes.md index 68069ab281..b27a712b4a 100644 --- a/release_notes.md +++ b/release_notes.md @@ -6,6 +6,9 @@ Under development in `master` branch. - MongoDB support. For white-box heuristics, can analyze all queries done toward MongoDB databases, as well as being able to insert data directly as part of the generated test cases. - improved fault detection for OpenAPI schema faults, in particular regarding the structure of the received responses, which are now validated. +- improved coverage criteria for black-box testing for REST APIs. +- support for exploiting "links" declarations in OpenAPI schemas. +- improved re-used of data between endpoints (e.g., data returned from GET requests can be used as input for following requests using fields with similar names). ### Bug Fixes From c770f52b6502274056abe3f76a19f6dfff9214c6 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Fri, 25 Oct 2024 09:53:19 +0200 Subject: [PATCH 2/2] fixed E2E --- .../EndpointFocusAndPrefixTest.java | 9 ++++++--- .../hypermutation/AdaptiveHypermutationTest.java | 3 ++- .../openapi/v3/bodyundefined/BodyUndefinedEMTest.kt | 11 ++++++++++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/endpointfocusandprefix/EndpointFocusAndPrefixTest.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/endpointfocusandprefix/EndpointFocusAndPrefixTest.java index 464186f423..26fc134b1a 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/endpointfocusandprefix/EndpointFocusAndPrefixTest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/endpointfocusandprefix/EndpointFocusAndPrefixTest.java @@ -163,6 +163,7 @@ public void testRunBlackboxWithFocusWithParameters() throws Throwable { args.add(baseUrlOfSut + "/v2/api-docs"); args.add("--endpointFocus"); args.add(endpointFocus); + setOption(args, "advancedBlackBoxCoverage", "false"); // no endpointFocus or endpointPrefix is provided Solution solution = initAndRun(args); @@ -174,7 +175,7 @@ public void testRunBlackboxWithFocusWithParameters() throws Throwable { assertAllSolutionsHavePathFocusOrPrefixList(solution, pathsToCheck, true); // The solution should include 4 solutions, 3 endpoints and 1 failure case - assertEquals(solution.getIndividuals().size(), 4); + assertEquals(4, solution.getIndividuals().size()); // write test into the output folder compile(outputFolder); @@ -283,6 +284,7 @@ public void testRunBlackboxWithPrefixWithParameters() throws Throwable { args.add(baseUrlOfSut + "/v2/api-docs"); args.add("--endpointPrefix"); args.add(endpointPrefix); + setOption(args, "advancedBlackBoxCoverage", "false"); // no endpointFocus or endpointPrefix is provided Solution solution = initAndRun(args); @@ -294,7 +296,7 @@ public void testRunBlackboxWithPrefixWithParameters() throws Throwable { assertAllSolutionsHavePathFocusOrPrefixList(solution, pathsToCheck, false); // The solution should include 5 solutions, 4 endpoints and 1 failure case - assertEquals(solution.getIndividuals().size(), 5); + assertEquals(5, solution.getIndividuals().size()); // write test into the output folder compile(outputFolder); @@ -323,6 +325,7 @@ public void testRunBlackboxPrefixNonExistingFocusValidPrefix() throws Throwable args.add(baseUrlOfSut + "/v2/api-docs"); args.add("--endpointPrefix"); args.add(endpointPrefix); + setOption(args, "advancedBlackBoxCoverage", "false"); // no endpointFocus or endpointPrefix is provided Solution solution = initAndRun(args); @@ -334,7 +337,7 @@ public void testRunBlackboxPrefixNonExistingFocusValidPrefix() throws Throwable assertAllSolutionsHavePathFocusOrPrefixList(solution, pathsToCheck, false); // The solution should include 5 solutions, 4 endpoints and 1 failure case - assertEquals(solution.getIndividuals().size(), 5); + assertEquals(5, solution.getIndividuals().size()); // write test into the output folder compile(outputFolder); diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/hypermutation/AdaptiveHypermutationTest.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/hypermutation/AdaptiveHypermutationTest.java index 94b8a1daf0..e08e3b4e29 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/hypermutation/AdaptiveHypermutationTest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/hypermutation/AdaptiveHypermutationTest.java @@ -48,7 +48,8 @@ public void testRunAdaptiveHypermutation() throws Throwable { //minimization loses impact info args.add("--minimize"); args.add("false"); - + //this had side-effects + setOption(args, "advancedBlackBoxCoverage", "false"); Solution solution = initAndRun(args); diff --git a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/bodyundefined/BodyUndefinedEMTest.kt b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/bodyundefined/BodyUndefinedEMTest.kt index f8de61f8b0..936787d1ac 100644 --- a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/bodyundefined/BodyUndefinedEMTest.kt +++ b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/bodyundefined/BodyUndefinedEMTest.kt @@ -25,6 +25,9 @@ class BodyUndefinedEMTest : SpringTestBase() { 20 ) { args: MutableList -> + //TODO remove once fixed issue + setOption(args, "advancedBlackBoxCoverage", "false") + val solution = initAndRun(args) Assertions.assertTrue(solution.individuals.size >= 1) @@ -35,7 +38,13 @@ class BodyUndefinedEMTest : SpringTestBase() { There is some weird bug in Jersey that it looks like it transform the GET into a POST ?!? When we upgrade Jersey (once moving ot JDK 11), need to fix AbstractRestFitness and RestActionBuilderV3. Then, we will need to check if this fails, and if so, change - into a 400 and 200 instead of 415 + into a 400 and 200 instead of 415. + + Even more weird, need to deactivate advancedBB coverage, otherwise generated tests + add a .body(" {} ") to the GET. + in Jersey, this still give a 415, but then a different code in RestAssured when compiled + tests are executed... WTF!!! + TODO look into this once upgraded Jersey */ assertHasAtLeastOne(solution, HttpVerb.GET, 415, "/api/bodyundefined", null) }