From 89a35bd4e35a6c34b34c2ba7d6188efc4a6b4833 Mon Sep 17 00:00:00 2001 From: Spencer Farley <2847259+farlee2121@users.noreply.github.com> Date: Mon, 6 Nov 2023 09:42:47 -0600 Subject: [PATCH 1/5] Allow color level to be set multiple times --- Expecto.Tests/Bug_RepeatedColourSetting.fs | 14 ++++++++++++++ Expecto.Tests/Expecto.Tests.fsproj | 1 + Expecto/Logging.fs | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 Expecto.Tests/Bug_RepeatedColourSetting.fs diff --git a/Expecto.Tests/Bug_RepeatedColourSetting.fs b/Expecto.Tests/Bug_RepeatedColourSetting.fs new file mode 100644 index 00000000..7567719a --- /dev/null +++ b/Expecto.Tests/Bug_RepeatedColourSetting.fs @@ -0,0 +1,14 @@ +module Bug_RepeatedColourSetting + +open Expecto +open Expecto.Logging + + +[] +let tests = + ftest "Colour can be set repeatedly" { + let colours = [|Colour0; Colour256|] + colours |> Array.iter ANSIOutputWriter.setColourLevel + + Expect.equal (ANSIOutputWriter.getColour ()) (Array.last colours) "Colour should be the last set value" + } diff --git a/Expecto.Tests/Expecto.Tests.fsproj b/Expecto.Tests/Expecto.Tests.fsproj index f9595e24..04b0b979 100644 --- a/Expecto.Tests/Expecto.Tests.fsproj +++ b/Expecto.Tests/Expecto.Tests.fsproj @@ -13,6 +13,7 @@ + diff --git a/Expecto/Logging.fs b/Expecto/Logging.fs index 2840e771..b1a645f8 100644 --- a/Expecto/Logging.fs +++ b/Expecto/Logging.fs @@ -879,7 +879,7 @@ module internal ANSIOutputWriter = override __.WriteLine() = write "\n" let mutable internal colours = None - let internal setColourLevel c = if colours.IsNone then colours <- Some c + let internal setColourLevel c = colours <- Some c let internal getColour() = Option.defaultValue Colour8 colours let colourReset = "\u001b[0m" From cbb623b656cf76d444d83906a27676f785e3695b Mon Sep 17 00:00:00 2001 From: Spencer Farley <2847259+farlee2121@users.noreply.github.com> Date: Mon, 6 Nov 2023 10:55:07 -0600 Subject: [PATCH 2/5] Add interactive test run method, runTestsReturnLogs --- Expecto/Expecto.fs | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/Expecto/Expecto.fs b/Expecto/Expecto.fs index 9de61755..2ab4ead9 100644 --- a/Expecto/Expecto.fs +++ b/Expecto/Expecto.fs @@ -643,3 +643,44 @@ module Tests = /// Returns 0 if all tests passed, otherwise 1 let runTestsInAssemblyWithCLIArgs cliArgs args = runTestsInAssemblyWithCLIArgsAndCancel CancellationToken.None cliArgs args + + let runTestsReturnLogs cliArgs args tests = + let tryBuildConfig cliArgs args = + let config = + Seq.fold (fun s a -> foldCLIArgumentToConfig a s) + ExpectoConfig.defaultConfig cliArgs + + match ExpectoConfig.fillFromArgs config args with + | ArgsUsage (usage, errors) -> + None + | ArgsList config + | ArgsRun config + | ArgsVersion config -> + Some config + + let literateOutputWriter (outputBuilder: Text.StringBuilder) (text: (string*ConsoleColor) list) : unit = + let colorizeLine (text, color) = ColourText.colouriseText color text + let sbAppend (builder: Text.StringBuilder) (text: string) = + builder.Append(text) + + text + |> List.iter (colorizeLine >> (sbAppend outputBuilder) >> ignore) + + let outputBuilder = System.Text.StringBuilder("") + + let verbosity = + tryBuildConfig cliArgs args + |> Option.defaultValue ExpectoConfig.defaultConfig + |> (fun config -> config.verbosity) + + Global.initialise + { Global.defaultConfig with + getLogger = fun name -> + Expecto.Logging.LiterateConsoleTarget( + name, + minLevel = verbosity, + outputWriter = (literateOutputWriter outputBuilder)) :> Expecto.Logging.Logger + } + + runTestsWithCLIArgs cliArgs args tests |> ignore + outputBuilder.ToString() \ No newline at end of file From 6d67df379e11a35fa8a7f3191eaf91b608a229ed Mon Sep 17 00:00:00 2001 From: Spencer Farley <2847259+farlee2121@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:09:06 -0600 Subject: [PATCH 3/5] Add documentation for new runTestsReturnLogs --- Expecto/Expecto.fs | 3 +++ README.md | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/Expecto/Expecto.fs b/Expecto/Expecto.fs index 2ab4ead9..f851e583 100644 --- a/Expecto/Expecto.fs +++ b/Expecto/Expecto.fs @@ -644,6 +644,9 @@ module Tests = let runTestsInAssemblyWithCLIArgs cliArgs args = runTestsInAssemblyWithCLIArgsAndCancel CancellationToken.None cliArgs args + /// Runs all given tests with the supplied typed command-line options. + /// Returns the console output as a string (with ANSI coloring by default) + /// Useful for interactive environments like F# interactive or notebooks let runTestsReturnLogs cliArgs args tests = let tryBuildConfig cliArgs args = let config = diff --git a/README.md b/README.md index 7dbf44e6..42810f1b 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ What follows is the Table of Contents for this README, which also serves as the - [`runTestsWithCLIArgsAndCancel`](#runtestswithcliargsandcancel) - [`runTestsInAssemblyWithCLIArgs`](#runtestsinassemblywithcliargs) - [`runTestsInAssemblyWithCLIArgsAndCancel`](#runtestsinassemblywithcliargsandcancel) + - [`runTestsReturnLogs`](#runtestsreturnlogs) - [Filtering with `filter`](#filtering-with-filter) - [Shuffling with `shuffle`](#shuffling-with-shuffle) - [Stress testing](#stress-testing) @@ -250,6 +251,22 @@ Signature `CancellationToken -> CLIArguments seq -> string[] -> int`. Runs the t assembly and also overrides the passed `CLIArguments` with the command line parameters. All tests need to be marked with the `[]` attribute. +### `runTestsReturnLogs` + +Signature `CLIArguments seq -> string[] -> Test -> string`. +Accepts the same arguments as `runTestsWithCLIArgs`, but returns the console output as a string. + +This is useful for interactive environments like [F# interactive](https://learn.microsoft.com/en-us/dotnet/fsharp/tools/fsharp-interactive/) and [Polyglot Notebooks](https://marketplace.visualstudio.com/items?itemName=ms-dotnettools.dotnet-interactive-vscode), where console output is not available but the returned string can be displayed instead. + +Note that ANSI colors are used by default, but can be turned off using `--colours 0`. +```fsharp +runTestsReturnLogs [] [|"--colours";"0"|] tests +``` + +Any valid CLI arguments work, including `--help` and `--list-tests`. + + + ### Filtering with `filter` You can single out tests by filtering them by name (e.g. in the From c3aae55b1ced87a9a4e4f890e996c57f82473290 Mon Sep 17 00:00:00 2001 From: Spencer Farley <2847259+farlee2121@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:02:54 -0600 Subject: [PATCH 4/5] Add a logging config option Aimed to allow an external interactive run experience --- Expecto/Expecto.Impl.fs | 4 ++++ Expecto/Expecto.fs | 49 +++++++++++++++++++---------------------- build.fsx | 24 ++++++++++---------- 3 files changed, 39 insertions(+), 38 deletions(-) diff --git a/Expecto/Expecto.Impl.fs b/Expecto/Expecto.Impl.fs index db4b0145..a39718ff 100644 --- a/Expecto/Expecto.Impl.fs +++ b/Expecto/Expecto.Impl.fs @@ -520,6 +520,9 @@ module Impl = colour: ColourLevel /// Split test names by `.` or `/` joinWith: JoinWith + /// A factory method taking the configured min log level and returning a logging config. + /// Can be used to swap out log targets like the LiterateConsoleTarget, TextWriterTarget, and OutputWindowTarget + loggingConfigFactory: (LogLevel -> LoggingConfig) option } static member defaultConfig = { runInParallel = true @@ -546,6 +549,7 @@ module Impl = noSpinner = false colour = Colour8 joinWith = JoinWith.Dot + loggingConfigFactory = None } member x.appendSummaryHandler handleSummary = diff --git a/Expecto/Expecto.fs b/Expecto/Expecto.fs index f851e583..8ad398e4 100644 --- a/Expecto/Expecto.fs +++ b/Expecto/Expecto.fs @@ -360,6 +360,10 @@ module Tests = type SummaryHandler = | SummaryHandler of (TestRunSummary -> unit) + [] + type LoggingConfigFactory = + | FromVerbosity of (LogLevel -> LoggingConfig) + /// The CLI arguments are the parameters that are possible to send to Expecto /// and change the runner's behaviour. type CLIArguments = @@ -421,6 +425,9 @@ module Tests = | Append_Summary_Handler of SummaryHandler /// Specify test names join character. | JoinWith of split: string + /// A factory method taking the configured min log level and returning a logging config. + /// Can be used to swap out log targets like the LiterateConsoleTarget, TextWriterTarget, and OutputWindowTarget + | LoggingConfigFactory of LoggingConfigFactory let options = [ "--sequenced", "Don't run the tests in parallel.", Args.none Sequenced @@ -508,6 +515,7 @@ module Tests = | Printer p -> fun o -> { o with printer = p } | Verbosity l -> fun o -> { o with verbosity = l } | Append_Summary_Handler (SummaryHandler h) -> fun o -> o.appendSummaryHandler h + | LoggingConfigFactory (FromVerbosity f) -> fun o -> { o with loggingConfigFactory = Some f} [] module ExpectoConfig = @@ -582,13 +590,18 @@ module Tests = let runTestsWithCLIArgsAndCancel (ct:CancellationToken) cliArgs args tests = let runTestsWithCancel (ct:CancellationToken) config (tests:Test) = ANSIOutputWriter.setColourLevel config.colour - Global.initialiseIfDefault - { Global.defaultConfig with - getLogger = fun name -> - LiterateConsoleTarget( - name, config.verbosity, - consoleSemaphore = Global.semaphore()) :> Logger - } + + let loggingConfig = + match config.loggingConfigFactory with + | Some factory -> factory config.verbosity |> Global.initialise + | None -> + { Global.defaultConfig with + getLogger = fun name -> + LiterateConsoleTarget( + name, config.verbosity, + consoleSemaphore = Global.semaphore()) :> Logger + } |> Global.initialiseIfDefault + config.logName |> Option.iter setLogName if config.failOnFocusedTests && passesFocusTestCheck config tests |> not then 1 @@ -648,18 +661,6 @@ module Tests = /// Returns the console output as a string (with ANSI coloring by default) /// Useful for interactive environments like F# interactive or notebooks let runTestsReturnLogs cliArgs args tests = - let tryBuildConfig cliArgs args = - let config = - Seq.fold (fun s a -> foldCLIArgumentToConfig a s) - ExpectoConfig.defaultConfig cliArgs - - match ExpectoConfig.fillFromArgs config args with - | ArgsUsage (usage, errors) -> - None - | ArgsList config - | ArgsRun config - | ArgsVersion config -> - Some config let literateOutputWriter (outputBuilder: Text.StringBuilder) (text: (string*ConsoleColor) list) : unit = let colorizeLine (text, color) = ColourText.colouriseText color text @@ -671,12 +672,7 @@ module Tests = let outputBuilder = System.Text.StringBuilder("") - let verbosity = - tryBuildConfig cliArgs args - |> Option.defaultValue ExpectoConfig.defaultConfig - |> (fun config -> config.verbosity) - - Global.initialise + let loggingConfigFactory verbosity = { Global.defaultConfig with getLogger = fun name -> Expecto.Logging.LiterateConsoleTarget( @@ -684,6 +680,7 @@ module Tests = minLevel = verbosity, outputWriter = (literateOutputWriter outputBuilder)) :> Expecto.Logging.Logger } - + + let cliArgs = (LoggingConfigFactory (FromVerbosity loggingConfigFactory)) :: cliArgs runTestsWithCLIArgs cliArgs args tests |> ignore outputBuilder.ToString() \ No newline at end of file diff --git a/build.fsx b/build.fsx index a9cd179e..06b0f41c 100644 --- a/build.fsx +++ b/build.fsx @@ -169,17 +169,17 @@ Target.create "Release" (fun _ -> Target.create "All" ignore -// "CheckEnv" -// ==> "Release" - -// "Clean" -// ==> "BuildExpecto" -// ==> "BuildBenchmarkDotNet" -// ==> "BuildTest" -// ==> "RunTest" -// ==> "Pack" -// ==> "All" -// ==> "Push" -// ==> "Release" +"CheckEnv" + ==> "Release" + +"Clean" + ==> "BuildExpecto" + ==> "BuildBenchmarkDotNet" + ==> "BuildTest" + ==> "RunTest" + ==> "Pack" + ==> "All" + ==> "Push" + ==> "Release" Target.runOrDefaultWithArguments "All" From c23d67f8ece85beeb20f8bed00715ae71a39ad78 Mon Sep 17 00:00:00 2001 From: Spencer Farley <2847259+farlee2121@users.noreply.github.com> Date: Thu, 23 Nov 2023 21:13:46 -0600 Subject: [PATCH 5/5] Basic tests for interactive test runs --- Expecto.Tests/Tests.fs | 67 ++++++++++++++++++++++++++++++++++++++++++ README.md | 2 ++ 2 files changed, 69 insertions(+) diff --git a/Expecto.Tests/Tests.fs b/Expecto.Tests/Tests.fs index 07911e34..8d17fe31 100644 --- a/Expecto.Tests/Tests.fs +++ b/Expecto.Tests/Tests.fs @@ -1821,4 +1821,71 @@ let theory = testTheoryTask "task odd numbers" [1; 3; 5;] <| fun x -> task { Expect.isTrue (x % 2 = 1) "should be odd" } + ] + +[] +let interactiveRunTests = + testList "running for interactive flows" [ + testList "logging config" [ + testCase "set loggingConfig via CLIArguments" <| fun () -> + let tests = testList "Test me" [ + testCase "I am a case" (fun () -> ()) + ] + + let globalLoggerOutput = new Text.StringBuilder() + let globalWriter = new StringWriter(globalLoggerOutput) + Global.initialise { + Global.defaultConfig with + getLogger = (fun name -> + TextWriterTarget(name, LogLevel.Info, globalWriter) + ) + } + + (runTestsWithCLIArgs [] [|"--help"|] tests) |> ignore + + let cliArgLoggerOutput = new Text.StringBuilder() + let cliArgWriter = new StringWriter(cliArgLoggerOutput) + let loggingConfigFactory _ = + { + Global.defaultConfig with + getLogger = (fun name -> + TextWriterTarget(name, LogLevel.Info, cliArgWriter) + ) + } + + (runTestsWithCLIArgs [LoggingConfigFactory (FromVerbosity loggingConfigFactory)] [|"--help"|] tests) |> ignore + + Expect.equal (string cliArgLoggerOutput) (string globalLoggerOutput) "Caputred output should be the same with Global.initialize or CLIArguments.LoggingConfigFactory" + ] + + testList "runTestsReturnLogs" [ + testCase "can print help" <| fun () -> + let tests = (testList "hi" []) + let output = runTestsReturnLogs [] [|"--help"|] tests + Expect.stringContains output "Options:" "" + Expect.stringContains output "--help Show this help message." "help message should contain the --help description" + + + testCase "can list tests" <| fun () -> + let tests = (testList "hi" [ + testCase "case1" (fun () -> ()) + testCase "case2" (fun () -> ()) + ]) + let expectedTestNames = ["hi.case1"; "hi.case2"] + let testListText = String.Join(Environment.NewLine, expectedTestNames) + + let output = runTestsReturnLogs [] [|"--list-tests"|] tests + Expect.stringContains output testListText "" + + testCase "can run tests and show basic status counts" <| fun () -> + let tests = testList "Interactive Tests" [ + testCase "I pass" (fun () -> ()) + test "I fail" { + Expect.equal true false "Should fail" + } + ] + + let output = runTestsReturnLogs [] [|"--colours"; "0"|] tests + Expect.stringContains output "1 passed, 0 ignored, 1 failed, 0 errored" "" + ] ] \ No newline at end of file diff --git a/README.md b/README.md index 1a370aad..1d7701c6 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,8 @@ runTestsWithCLIArgs [] [||] simpleTest which returns 1 if any tests failed, otherwise 0. Useful for returning to the operating system as error code. +For interactive environments, you can alternatively call [`runTestsReturnLogs`](#runtestsreturnlogs), which returns the console output as a string. + It's worth noting that `<|` is just a way to change the associativity of the language parser. In other words; it's equivalent to: