Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable all testing modes by default, update property mode testing, improve UX, allow for contracts to have starting balances, and fix coverage panic #216

Merged
merged 18 commits into from
Feb 20, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
update deploymentOrder to targetContracts
  • Loading branch information
anishnaik committed Aug 30, 2023
commit a757cbc27f1450a617bda608786341a3517466dd
22 changes: 11 additions & 11 deletions cmd/fuzz_flags.go
Original file line number Diff line number Diff line change
@@ -21,8 +21,8 @@ func addFuzzFlags() error {
// Config file
fuzzCmd.Flags().String("config", "", "path to config file")

// Target
fuzzCmd.Flags().String("target", "", TargetFlagDescription)
// Compilation Target
fuzzCmd.Flags().String("compilation-target", "", TargetFlagDescription)

// Number of workers
fuzzCmd.Flags().Int("workers", 0,
@@ -40,9 +40,9 @@ func addFuzzFlags() error {
fuzzCmd.Flags().Int("seq-len", 0,
fmt.Sprintf("maximum transactions to run in sequence (unless a config file is provided, default is %d)", defaultConfig.Fuzzing.CallSequenceLength))

// Deployment order
fuzzCmd.Flags().StringSlice("deployment-order", []string{},
fmt.Sprintf("order in which to deploy target contracts (unless a config file is provided, default is %v)", defaultConfig.Fuzzing.DeploymentOrder))
// Target contracts
fuzzCmd.Flags().StringSlice("target-contracts", []string{},
fmt.Sprintf("target contracts for fuzz testing (unless a config file is provided, default is %v)", defaultConfig.Fuzzing.TargetContracts))

// Corpus directory
fuzzCmd.Flags().String("corpus-dir", "",
@@ -66,10 +66,10 @@ func addFuzzFlags() error {
func updateProjectConfigWithFuzzFlags(cmd *cobra.Command, projectConfig *config.ProjectConfig) error {
var err error

// If --target was used
if cmd.Flags().Changed("target") {
// If --compilation-target was used
if cmd.Flags().Changed("compilation-target") {
// Get the new target
newTarget, err := cmd.Flags().GetString("target")
newTarget, err := cmd.Flags().GetString("compilation-target")
if err != nil {
return err
}
@@ -112,9 +112,9 @@ func updateProjectConfigWithFuzzFlags(cmd *cobra.Command, projectConfig *config.
}
}

// Update deployment order
if cmd.Flags().Changed("deployment-order") {
projectConfig.Fuzzing.DeploymentOrder, err = cmd.Flags().GetStringSlice("deployment-order")
// Update target contracts
if cmd.Flags().Changed("target-contracts") {
projectConfig.Fuzzing.TargetContracts, err = cmd.Flags().GetStringSlice("target-contracts")
if err != nil {
return err
}
10 changes: 5 additions & 5 deletions cmd/init_flags.go
Original file line number Diff line number Diff line change
@@ -10,18 +10,18 @@ func addInitFlags() error {
// Output path for configuration
initCmd.Flags().String("out", "", "output path for the new project configuration file")

// Target file / directory
initCmd.Flags().String("target", "", TargetFlagDescription)
// Target file / directory for compilation
initCmd.Flags().String("compilation-target", "", TargetFlagDescription)

return nil
}

// updateProjectConfigWithInitFlags will update the given projectConfig with any CLI arguments that were provided to the init command
func updateProjectConfigWithInitFlags(cmd *cobra.Command, projectConfig *config.ProjectConfig) error {
// If --target was used
if cmd.Flags().Changed("target") {
// If --compilation-target was used
if cmd.Flags().Changed("compilation-target") {
// Get the new target
newTarget, err := cmd.Flags().GetString("target")
newTarget, err := cmd.Flags().GetString("compilation-target")
if err != nil {
return err
}
7 changes: 4 additions & 3 deletions fuzzing/config/config.go
Original file line number Diff line number Diff line change
@@ -50,10 +50,11 @@ type FuzzingConfig struct {
// CoverageEnabled describes whether to use coverage-guided fuzzing
CoverageEnabled bool `json:"coverageEnabled"`

// DeploymentOrder determines the order in which the contracts should be deployed
DeploymentOrder []string `json:"deploymentOrder"`
// TargetContracts are the target contracts for fuzz testing
TargetContracts []string `json:"targetContracts"`

// Constructor arguments for contracts deployment. It is available only in init mode
// ConstructorArgs holds the constructor arguments for TargetContracts deployments. It is available via the project
// configuration
ConstructorArgs map[string]map[string]any `json:"constructorArgs"`

// DeployerAddress describe the account address to be used to deploy contracts.
2 changes: 1 addition & 1 deletion fuzzing/config/config_defaults.go
Original file line number Diff line number Diff line change
@@ -37,7 +37,7 @@ func GetDefaultProjectConfig(platform string) (*ProjectConfig, error) {
Timeout: 0,
TestLimit: 0,
CallSequenceLength: 100,
DeploymentOrder: []string{},
TargetContracts: []string{},
ConstructorArgs: map[string]map[string]any{},
CorpusDirectory: "",
CoverageEnabled: true,
20 changes: 10 additions & 10 deletions fuzzing/fuzzer.go
Original file line number Diff line number Diff line change
@@ -327,23 +327,23 @@ func (f *Fuzzer) createTestChain() (*chain.TestChain, error) {

// chainSetupFromCompilations is a TestChainSetupFunc which sets up the base test chain state by deploying
// all compiled contract definitions. This includes any successful compilations as a result of the Fuzzer.config
// definitions, as well as those added by Fuzzer.AddCompilationTargets. The contract deployment order is defined by
// definitions, as well as those added by Fuzzer.AddCompilationTargets. The target contracts is defined by
// the Fuzzer.config.
func chainSetupFromCompilations(fuzzer *Fuzzer, testChain *chain.TestChain) error {
// Verify contract deployment order is not empty. If it's empty, but we only have one contract definition,
// we can infer the deployment order. Otherwise, we report an error.
if len(fuzzer.config.Fuzzing.DeploymentOrder) == 0 {
// Verify that target contracts is not empty. If it's empty, but we only have one contract definition,
// we can infer the target contracts. Otherwise, we report an error.
if len(fuzzer.config.Fuzzing.TargetContracts) == 0 {
if len(fuzzer.contractDefinitions) == 1 {
fuzzer.config.Fuzzing.DeploymentOrder = []string{fuzzer.contractDefinitions[0].Name()}
fuzzer.config.Fuzzing.TargetContracts = []string{fuzzer.contractDefinitions[0].Name()}
} else {
return fmt.Errorf("missing deployment order (update fuzzing.deploymentOrder in the project config " +
"or use the --deployment-order CLI flag)")
return fmt.Errorf("missing target contracts (update fuzzing.targetContracts in the project config " +
"or use the --target-contracts CLI flag)")
}
}

// Loop for all contracts to deploy
deployedContractAddr := make(map[string]common.Address)
for _, contractName := range fuzzer.config.Fuzzing.DeploymentOrder {
for _, contractName := range fuzzer.config.Fuzzing.TargetContracts {
// Look for a contract in our compiled contract definitions that matches this one
found := false
for _, contract := range fuzzer.contractDefinitions {
@@ -411,8 +411,8 @@ func chainSetupFromCompilations(fuzzer *Fuzzer, testChain *chain.TestChain) erro

// If we did not find a contract corresponding to this item in the deployment order, we throw an error.
if !found {
return fmt.Errorf("%v was specified in the deployment order (see fuzzing.deploymentOrder in the "+
"project config or the --deployment-order CLI flag) but was not found in the compilation artifacts", contractName)
return fmt.Errorf("%v was specified in the target contracts (see fuzzing.targetContracts in the "+
"project config or the --target-contracts CLI flag) but was not found in the compilation artifacts", contractName)
}
}
return nil
55 changes: 28 additions & 27 deletions fuzzing/fuzzer_test.go
Original file line number Diff line number Diff line change
@@ -21,7 +21,7 @@ func TestFuzzerHooks(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: "testdata/contracts/assertions/assert_immediate.sol",
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"TestContract"}
config.Fuzzing.TargetContracts = []string{"TestContract"}
config.Fuzzing.Testing.PropertyTesting.Enabled = false
config.Fuzzing.Testing.OptimizationTesting.Enabled = false
},
@@ -76,7 +76,7 @@ func TestAssertionMode(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: filePath,
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"TestContract"}
config.Fuzzing.TargetContracts = []string{"TestContract"}
config.Fuzzing.Testing.AssertionTesting.PanicCodeConfig.FailOnAssertion = true
config.Fuzzing.Testing.AssertionTesting.PanicCodeConfig.FailOnAllocateTooMuchMemory = true
config.Fuzzing.Testing.AssertionTesting.PanicCodeConfig.FailOnArithmeticUnderflow = true
@@ -107,7 +107,7 @@ func TestAssertionsNotRequire(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: "testdata/contracts/assertions/assert_not_require.sol",
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"TestContract"}
config.Fuzzing.TargetContracts = []string{"TestContract"}
config.Fuzzing.TestLimit = 500
config.Fuzzing.Testing.PropertyTesting.Enabled = false
config.Fuzzing.Testing.OptimizationTesting.Enabled = false
@@ -129,7 +129,7 @@ func TestAssertionsAndProperties(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: "testdata/contracts/assertions/assert_and_property_test.sol",
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"TestContract"}
config.Fuzzing.TargetContracts = []string{"TestContract"}
config.Fuzzing.TestLimit = 500
config.Fuzzing.Testing.StopOnFailedTest = false
config.Fuzzing.Testing.OptimizationTesting.Enabled = false
@@ -154,7 +154,7 @@ func TestOptimizationMode(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: filePath,
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"TestContract"}
config.Fuzzing.TargetContracts = []string{"TestContract"}
config.Fuzzing.TestLimit = 10_000 // this test should expose a failure quickly.
config.Fuzzing.Testing.PropertyTesting.Enabled = false
config.Fuzzing.Testing.AssertionTesting.Enabled = false
@@ -182,7 +182,7 @@ func TestChainBehaviour(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: "testdata/contracts/chain/tx_out_of_gas.sol",
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"TestContract"}
config.Fuzzing.TargetContracts = []string{"TestContract"}
config.Fuzzing.Workers = 1
config.Fuzzing.TestLimit = uint64(config.Fuzzing.CallSequenceLength) // we just need a few oog txs to test
config.Fuzzing.Timeout = 10 // to be safe, we set a 10s timeout
@@ -236,7 +236,7 @@ func TestCheatCodes(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: filePath,
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"TestContract"}
config.Fuzzing.TargetContracts = []string{"TestContract"}

// Some tests require full sequence + revert to test fully
config.Fuzzing.Workers = 3
@@ -276,7 +276,7 @@ func TestConsoleLog(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: filePath,
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"TestContract"}
config.Fuzzing.TargetContracts = []string{"TestContract"}
config.Fuzzing.TestLimit = 10000
config.Fuzzing.Testing.PropertyTesting.Enabled = false
config.Fuzzing.Testing.OptimizationTesting.Enabled = false
@@ -322,7 +322,7 @@ func TestDeploymentsInnerDeployments(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: filePath,
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"InnerDeploymentFactory"}
config.Fuzzing.TargetContracts = []string{"InnerDeploymentFactory"}
config.Fuzzing.TestLimit = 1_000 // this test should expose a failure quickly.
config.Fuzzing.Testing.StopOnFailedContractMatching = true
config.Fuzzing.Testing.TestAllContracts = true // test dynamically deployed contracts
@@ -345,7 +345,7 @@ func TestDeploymentsInnerDeployments(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: "testdata/contracts/deployments/inner_deployment_on_construction.sol",
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"InnerDeploymentFactory"}
config.Fuzzing.TargetContracts = []string{"InnerDeploymentFactory"}
config.Fuzzing.TestLimit = 1_000 // this test should expose a failure quickly.
config.Fuzzing.Testing.StopOnFailedContractMatching = true
config.Fuzzing.Testing.TestAllContracts = true // test dynamically deployed contracts
@@ -368,7 +368,7 @@ func TestDeploymentsInternalLibrary(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: "testdata/contracts/deployments/internal_library.sol",
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"TestInternalLibrary"}
config.Fuzzing.TargetContracts = []string{"TestInternalLibrary"}
config.Fuzzing.TestLimit = 100 // this test should expose a failure quickly.
config.Fuzzing.Testing.AssertionTesting.Enabled = false
config.Fuzzing.Testing.OptimizationTesting.Enabled = false
@@ -397,7 +397,7 @@ func TestDeploymentsSelfDestruct(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: filePath,
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"InnerDeploymentFactory"}
config.Fuzzing.TargetContracts = []string{"InnerDeploymentFactory"}
config.Fuzzing.TestLimit = 500 // this test should expose a failure quickly.
config.Fuzzing.Testing.StopOnNoTests = false
config.Fuzzing.Testing.AssertionTesting.Enabled = false
@@ -445,7 +445,7 @@ func TestExecutionTraces(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: filePath,
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"TestContract"}
config.Fuzzing.TargetContracts = []string{"TestContract"}
config.Fuzzing.Testing.PropertyTesting.Enabled = false
config.Fuzzing.Testing.OptimizationTesting.Enabled = false
},
@@ -485,7 +485,7 @@ func TestTestingScope(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: "testdata/contracts/deployments/testing_scope.sol",
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"TestContract"}
config.Fuzzing.TargetContracts = []string{"TestContract"}
config.Fuzzing.TestLimit = 1_000 // this test should expose a failure quickly.
config.Fuzzing.Testing.TestAllContracts = testingAllContracts
config.Fuzzing.Testing.StopOnFailedTest = false
@@ -519,7 +519,7 @@ func TestDeploymentsWithArgs(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: "testdata/contracts/deployments/deployment_with_args.sol",
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"DeploymentWithArgs", "Dependent"}
config.Fuzzing.TargetContracts = []string{"DeploymentWithArgs", "Dependent"}
config.Fuzzing.ConstructorArgs = map[string]map[string]any{
"DeploymentWithArgs": {
"_x": "123456789",
@@ -554,7 +554,7 @@ func TestValueGenerationGenerateAllTypes(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: "testdata/contracts/value_generation/generate_all_types.sol",
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"GenerateAllTypes"}
config.Fuzzing.TargetContracts = []string{"GenerateAllTypes"}
config.Fuzzing.TestLimit = 10_000
config.Fuzzing.Testing.AssertionTesting.Enabled = false
config.Fuzzing.Testing.OptimizationTesting.Enabled = false
@@ -589,7 +589,7 @@ func TestValueGenerationSolving(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: filePath,
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"TestContract"}
config.Fuzzing.TargetContracts = []string{"TestContract"}
config.Fuzzing.Testing.AssertionTesting.Enabled = false
config.Fuzzing.Testing.OptimizationTesting.Enabled = false
},
@@ -649,6 +649,7 @@ func TestASTValueExtraction(t *testing.T) {
config.Fuzzing.TestLimit = 1 // stop immediately to simply see what values were mined.
config.Fuzzing.Testing.PropertyTesting.Enabled = false
config.Fuzzing.Testing.OptimizationTesting.Enabled = false
config.Fuzzing.TargetContracts = []string{"TestContract"}
},
method: func(f *fuzzerTestContext) {
// Start the fuzzer
@@ -682,7 +683,7 @@ func TestVMCorrectness(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: "testdata/contracts/vm_tests/block_number_increasing.sol",
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"TestContract"}
config.Fuzzing.TargetContracts = []string{"TestContract"}
config.Fuzzing.MaxBlockTimestampDelay = 1 // this contract require calls every block
config.Fuzzing.MaxBlockNumberDelay = 1 // this contract require calls every block
config.Fuzzing.Testing.AssertionTesting.Enabled = false
@@ -703,7 +704,7 @@ func TestVMCorrectness(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: "testdata/contracts/vm_tests/block_number_increasing.sol",
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"TestContract"}
config.Fuzzing.TargetContracts = []string{"TestContract"}
config.Fuzzing.MaxBlockTimestampDelay = 1 // this contract require calls every block
config.Fuzzing.MaxBlockNumberDelay = 1 // this contract require calls every block
},
@@ -722,7 +723,7 @@ func TestVMCorrectness(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: "testdata/contracts/vm_tests/block_hash_store_check.sol",
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"TestContract"}
config.Fuzzing.TargetContracts = []string{"TestContract"}
config.Fuzzing.TestLimit = 1_000 // this test should expose a failure quickly.
config.Fuzzing.MaxBlockTimestampDelay = 1 // this contract require calls every block
config.Fuzzing.MaxBlockNumberDelay = 1 // this contract require calls every block
@@ -747,7 +748,7 @@ func TestCorpusReplayability(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: "testdata/contracts/value_generation/match_uints_xy.sol",
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"TestContract"}
config.Fuzzing.TargetContracts = []string{"TestContract"}
config.Fuzzing.CorpusDirectory = "corpus"
config.Fuzzing.Testing.AssertionTesting.Enabled = false
config.Fuzzing.Testing.OptimizationTesting.Enabled = false
@@ -788,14 +789,14 @@ func TestCorpusReplayability(t *testing.T) {
})
}

// TestDeploymentOrderWithCoverage will ensure that changing the deployment order does not lead to the same coverage
// This is also proof that changing the order changes the addresses of the contracts leading to the coverage not being
// useful.
// TestDeploymentOrderWithCoverage will ensure that changing the order of deployment for the target contracts does not
// lead to the same coverage. This is also proof that changing the order changes the addresses of the contracts leading
// to the coverage not being useful.
func TestDeploymentOrderWithCoverage(t *testing.T) {
runFuzzerTest(t, &fuzzerSolcFileTest{
filePath: "testdata/contracts/deployments/deployment_order.sol",
configUpdates: func(config *config.ProjectConfig) {
config.Fuzzing.DeploymentOrder = []string{"InheritedFirstContract", "InheritedSecondContract"}
config.Fuzzing.TargetContracts = []string{"InheritedFirstContract", "InheritedSecondContract"}
config.Fuzzing.Testing.AssertionTesting.Enabled = false
config.Fuzzing.Testing.OptimizationTesting.Enabled = false
},
@@ -820,8 +821,8 @@ func TestDeploymentOrderWithCoverage(t *testing.T) {
return nil
})

// Update the deployment order
f.fuzzer.config.Fuzzing.DeploymentOrder = []string{"InheritedSecondContract", "InheritedFirstContract"}
// Update the order of target contracts
f.fuzzer.config.Fuzzing.TargetContracts = []string{"InheritedSecondContract", "InheritedFirstContract"}

// Note that the fuzzer won't spin up any workers or fuzz anything. We just want to test that the coverage
// maps don't populate due to deployment order changes
Loading