diff --git a/Expecto.FsCheck/FsCheck.fs b/Expecto.FsCheck/FsCheck.fs index 3758cd20..16cb674d 100644 --- a/Expecto.FsCheck/FsCheck.fs +++ b/Expecto.FsCheck/FsCheck.fs @@ -71,7 +71,7 @@ module ExpectoFsCheck = (String.concat " " data.Labels) let focus = - sprintf "Focus on error:\n\t%s (%A, %A) \"%s\"" methodName (uint64 std) (uint64 gen) name + sprintf "Focus on error:\n\t%s (%A, %A, None) \"%s\"" methodName (uint64 std) (uint64 gen) name sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s" (numTests data.NumberOfTests) parameters shrunk @@ -92,7 +92,7 @@ module ExpectoFsCheck = MaxFail = 1000 // We're converting uint64s to a smaller type, but it shouldn't be an issue because users are only using the // values given in the test output, which are only ints when running FsCheck 2 - Replay = Option.map Random.StdGen (config.replay |> Option.map (fun (seed, gamma) -> int seed, int gamma)) + Replay = Option.map Random.StdGen (config.replay |> Option.map (fun (seed, gamma, _) -> int seed, int gamma)) Name = name StartSize = config.startSize EndSize = config.endSize diff --git a/Expecto.FsCheck3/FsCheck3.fs b/Expecto.FsCheck3/FsCheck3.fs index 55c8e4c4..39d948e5 100644 --- a/Expecto.FsCheck3/FsCheck3.fs +++ b/Expecto.FsCheck3/FsCheck3.fs @@ -47,7 +47,7 @@ module ExpectoFsCheck = | TestResult.Failed (_,_,_, Outcome.Failed (:? IgnoreException as e),_,_,_) -> raise e - | TestResult.Failed (data, original, shrunk, outcome,originalSeed,_finalSeed,size) -> + | TestResult.Failed (data, original, shrunk, outcome,originalSeed,finalSeed,size) -> let parameters = original |> List.map (sprintf "%A") @@ -70,12 +70,14 @@ module ExpectoFsCheck = | _ -> sprintf "Labels of failing property (one or more is failing): %s\n" (String.concat " " data.Labels) + let original = + sprintf "Original seed: (%A, %A)" originalSeed.Seed originalSeed.Gamma let focus = - sprintf "Focus on error:\n\t%s (%A, %A) \"%s\"" methodName originalSeed.Seed originalSeed.Gamma name + sprintf "Focus on error:\n\t%s (%A, %A, Some %A) \"%s\"" methodName finalSeed.Seed finalSeed.Gamma size name - sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s" + sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s\n%s" (numTests data.NumberOfTests) parameters shrunk - outcome labels (stampsToString data.Stamps) focus + outcome labels (stampsToString data.Stamps) original focus |> FailedException |> raise @@ -91,7 +93,7 @@ module ExpectoFsCheck = let config = Config.Default .WithMaxTest(config.maxTest) - .WithReplay(Option.map (fun (seed,gamma) -> {Rnd = Rnd(seed, gamma); Size = None}) config.replay) + .WithReplay(Option.map (fun (seed,gamma,size) -> {Rnd = Rnd(seed, gamma); Size = size}) config.replay) .WithName(name) .WithStartSize(config.startSize) .WithEndSize(config.endSize) diff --git a/Expecto.Tests.FsCheck3/FsCheck3Tests.fs b/Expecto.Tests.FsCheck3/FsCheck3Tests.fs index f1bbeb2b..def37fd2 100644 --- a/Expecto.Tests.FsCheck3/FsCheck3Tests.fs +++ b/Expecto.Tests.FsCheck3/FsCheck3Tests.fs @@ -60,7 +60,7 @@ let focused = testList "FsCheck focused" [ testCase "ignore me" <| ignore - etestProperty (1UL,3UL) "Deliberately failing test" <| + etestProperty (1UL,3UL,Some 50) "Deliberately failing test" <| fun a b c -> // wrong on purpose to test failures a * (b + c) = a * a + a * c @@ -87,14 +87,15 @@ let runFsCheckFocusedTests = match (getResult ["FsCheck focused";"Deliberately failing test"]).result with | TestResult.Failed actual -> let expected = " -Failed after 3 tests. Parameters: - -1 1 2 -Shrunk 2 times to: - -1 0 0 +Failed after 1 test. Parameters: + 47 43 -38 +Shrunk 9 times to: + 1 0 0 Result: Failed System.Exception: Expected true, got false. +Original seed: (1UL, 3UL) Focus on error: - etestProperty (1UL, 3UL) \"Deliberately failing test\"" + etestProperty (1UL, 3UL, Some 50) \"Deliberately failing test\"" Expect.equal actual expected "It should fail with the right message" | x -> failtestf "Expected Failed, actual was: %A" x @@ -104,7 +105,7 @@ let config = testList "FsCheck config" [ testCase "ignore me" ignore - etestPropertyWithConfig (1UL,3UL) FsCheckConfig.defaultConfig + etestPropertyWithConfig (1UL,3UL,Some 50) FsCheckConfig.defaultConfig "Deliberately failing test" <| fun a b c -> // wrong on purpose to test failures @@ -132,14 +133,15 @@ let runFsCheckConfigTests = match (getResult ["FsCheck config";"Deliberately failing test"]).result with | TestResult.Failed actual -> let expected = " -Failed after 3 tests. Parameters: - -1 1 2 -Shrunk 2 times to: - -1 0 0 +Failed after 1 test. Parameters: + 47 43 -38 +Shrunk 9 times to: + 1 0 0 Result: Failed System.Exception: Expected true, got false. +Original seed: (1UL, 3UL) Focus on error: - etestPropertyWithConfig (1UL, 3UL) \"Deliberately failing test\"" + etestPropertyWithConfig (1UL, 3UL, Some 50) \"Deliberately failing test\"" Expect.equal actual expected "It should fail with the right message." | x -> diff --git a/Expecto.Tests/FsCheckTests.fs b/Expecto.Tests/FsCheckTests.fs index a6059aca..ba26bc9e 100644 --- a/Expecto.Tests/FsCheckTests.fs +++ b/Expecto.Tests/FsCheckTests.fs @@ -60,7 +60,7 @@ let focused = testList "FsCheck focused" [ testCase "ignore me" <| ignore - etestProperty (1UL,2UL) "Deliberately failing test" <| + etestProperty (1UL,2UL,None) "Deliberately failing test" <| fun a b c -> // wrong on purpose to test failures a * (b + c) = a * a + a * c @@ -94,7 +94,7 @@ Shrunk 4 times to: Result: False Focus on error: - etestProperty (1UL, 2UL) \"Deliberately failing test\"" + etestProperty (1UL, 2UL, None) \"Deliberately failing test\"" Expect.equal actual expected "It should fail with the right message" | x -> failtestf "Expected Failed, actual was: %A" x @@ -104,7 +104,7 @@ let config = testList "FsCheck config" [ testCase "ignore me" ignore - etestPropertyWithConfig (1UL,2UL) FsCheckConfig.defaultConfig + etestPropertyWithConfig (1UL,2UL,None) FsCheckConfig.defaultConfig "Deliberately failing test" <| fun a b c -> // wrong on purpose to test failures @@ -139,7 +139,7 @@ Shrunk 4 times to: Result: False Focus on error: - etestPropertyWithConfig (1UL, 2UL) \"Deliberately failing test\"" + etestPropertyWithConfig (1UL, 2UL, None) \"Deliberately failing test\"" Expect.equal actual expected "It should fail with the right message." | x -> diff --git a/Expecto/Model.fs b/Expecto/Model.fs index aeb3c7dd..401869b3 100644 --- a/Expecto/Model.fs +++ b/Expecto/Model.fs @@ -15,8 +15,9 @@ type FsCheckConfig = startSize: int /// The size to use for the last test, when all the tests are passing. The size increases linearly between Start- and EndSize. endSize: int - /// If set, the seed to use to start testing. Allows reproduction of previous runs. - replay: (uint64 * uint64) option + /// (seed * gamma * size) corresponding to the FsCheck3 replay config. Allows reproduction of previous runs. The + /// size is ignored with FsCheck2. + replay: (uint64 * uint64 * int option) option /// The Arbitrary instances on this class will be merged in back to front order, i.e. instances for the same generated type at the front /// of the list will override those at the back. The instances on Arb.Default are always known, and are at the back (so they can always be /// overridden) diff --git a/README.md b/README.md index f1b33efd..e7d8499b 100644 --- a/README.md +++ b/README.md @@ -1372,5 +1372,5 @@ This might be due to how terminals/the locking thereof work: try running your te ## Migration notes ### 11.0.0 -- Any usages of the `replay` (a.k.a `stdGen` with `etestProperty*` functions) config with FsCheck tests will need to be updated to use `uint64` by appending `UL` to the literals, e.g. from `(1865288075, 296281834)` to `(1865288075UL, 296281834UL)`. +- Any usages of the `replay` (a.k.a `stdGen` with `etestProperty*` functions) config with FsCheck tests will need to be updated to use `uint64` by appending `UL` to the literals. They will also now require a third item indicating the size. E.g. from `(1865288075, 296281834)` to `(1865288075UL, 296281834UL, Some 3)`. With FsCheck 2 the size is ignored and `None` can be used. - FsCheck 2 is no longer supported, so we're switching Expecto.FsCheck to use FsCheck 3 by default, even though FsCheck 3 is still in release candidate state. If you still want FsCheck2, we will continue to release FsCheck2 support for the time being using a version suffix, e.g. [11.0.0-fscheck2](https://www.nuget.org/packages/Expecto.FsCheck/11.0.0-alpha1-fscheck2) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 61192c29..a2e83735 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,9 @@ +### 11.0.0-alpha4 - 2025-01-06 +* Breaking change: Add third item to `FsCheckConfig.replay` indicating the size + * Fixes issue where the replay seed without the size was playing all tests leading up to the failure, making debugging + more difficult + * Existing FsCheck 2 users can enter `None` for the size, because it is ignored. + ### 11.0.0-alpha3 - 2024-10-13 * Add testParamAsync and testParamTask (#512), thanks @1eyewonder