diff --git a/src/NBomber/Api/CSharp.fs b/src/NBomber/Api/CSharp.fs index cd478b57..094051ab 100644 --- a/src/NBomber/Api/CSharp.fs +++ b/src/NBomber/Api/CSharp.fs @@ -75,6 +75,13 @@ type Scenario = static member WithResetIterationOnFail(scenario: ScenarioProps, shouldReset: bool) = scenario |> FSharp.Scenario.withResetIterationOnFail shouldReset + /// Sets and overrides the default max fail count. + /// When a scenario reaches max fail count, NBomber will stop the whole load test. + /// By default MaxFailCount = 5_000 + [] + static member WithMaxFailCount(scenario: ScenarioProps, failCount: int) = + scenario |> FSharp.Scenario.withMaxFailCount failCount + [] type NBomberRunner = @@ -171,13 +178,6 @@ type NBomberRunner = static member EnableHintsAnalyzer(context: NBomberContext, enable: bool) = context |> FSharp.NBomberRunner.enableHintsAnalyzer enable - /// Sets and overrides the default max fail count. - /// In case of any scenario is reaching max fail count, then NBomber will stop the whole load test. - /// By default MaxFailCount = 5_000 - [] - static member WithMaxFailCount(context: NBomberContext, failCount: int) = - context |> FSharp.NBomberRunner.withMaxFailCount failCount - [] static member Run(context: NBomberContext) = match FSharp.NBomberRunner.run context with diff --git a/src/NBomber/Api/FSharp.fs b/src/NBomber/Api/FSharp.fs index 074843e3..3279313d 100644 --- a/src/NBomber/Api/FSharp.fs +++ b/src/NBomber/Api/FSharp.fs @@ -56,7 +56,8 @@ module Scenario = Run = Some run WarmUpDuration = Some Constants.DefaultWarmUpDuration LoadSimulations = [LoadSimulation.KeepConstant(copies = Constants.DefaultCopiesCount, during = Constants.DefaultSimulationDuration)] - ResetIterationOnFail = true } + ResetIterationOnFail = true + MaxFailCount = Constants.ScenarioMaxFailCount } /// Creates empty scenario. /// An empty scenario is useful when you want to create the scenario to do only initialization or cleaning and execute it separately. @@ -68,7 +69,8 @@ module Scenario = Run = None WarmUpDuration = None LoadSimulations = [LoadSimulation.KeepConstant(copies = Constants.DefaultCopiesCount, during = Constants.DefaultSimulationDuration)] - ResetIterationOnFail = true } + ResetIterationOnFail = true + MaxFailCount = Constants.ScenarioMaxFailCount } /// Initializes scenario. /// You can use it to for example to prepare your target system or to parse and apply configuration. @@ -102,6 +104,12 @@ module Scenario = let withResetIterationOnFail (shouldReset: bool) (scenario: ScenarioProps) = { scenario with ResetIterationOnFail = shouldReset } + /// Sets and overrides the default max fail count. + /// When a scenario reaches max fail count, NBomber will stop the whole load test. + /// By default MaxFailCount = 5_000 + let withMaxFailCount (failCount: int) (scenario: ScenarioProps) = + { scenario with MaxFailCount = failCount } + /// NBomberRunner is responsible for registering and running scenarios. /// Also it provides configuration points related to infrastructure, reporting, loading plugins. [] @@ -228,12 +236,6 @@ module NBomberRunner = let enableHintsAnalyzer (enable: bool) (context: NBomberContext) = { context with EnableHintsAnalyzer = enable } - /// Sets and overrides the default max fail count. - /// In case of any scenario is reaching max fail count, then NBomber will stop the whole load test. - /// By default MaxFailCount = 5_000 - let withMaxFailCount (failCount: int) (context: NBomberContext) = - { context with MaxFailCount = failCount } - let internal executeCliArgs (args) (context: NBomberContext) = let loadConfigFn (loadConfig) (configPath) (context) = diff --git a/src/NBomber/Configuration.fs b/src/NBomber/Configuration.fs index eeba21f5..5ec9ed44 100644 --- a/src/NBomber/Configuration.fs +++ b/src/NBomber/Configuration.fs @@ -10,6 +10,7 @@ type ScenarioSetting = { WarmUpDuration: TimeSpan option LoadSimulationsSettings: LoadSimulation list option [] CustomSettings: string option + MaxFailCount: int option } type GlobalSettings = { @@ -19,7 +20,6 @@ type GlobalSettings = { ReportFormats: ReportFormat list option ReportingInterval: TimeSpan option EnableHintsAnalyzer: bool option - MaxFailCount: int option } type NBomberConfig = { diff --git a/src/NBomber/Constants.fs b/src/NBomber/Constants.fs index 9b6592e2..a2b964f7 100644 --- a/src/NBomber/Constants.fs +++ b/src/NBomber/Constants.fs @@ -67,4 +67,4 @@ let EmptyInfraConfig = ConfigurationBuilder().Build() :> IConfiguration [] let StatsRounding = 2 [] -let DefaultMaxFailCount = 5_000 +let ScenarioMaxFailCount = 5_000 diff --git a/src/NBomber/Contracts.fs b/src/NBomber/Contracts.fs index c60f21b6..81cabecd 100644 --- a/src/NBomber/Contracts.fs +++ b/src/NBomber/Contracts.fs @@ -26,7 +26,6 @@ type NBomberContext = { WorkerPlugins: IWorkerPlugin list EnableHintsAnalyzer: bool TargetScenarios: string list option - MaxFailCount: int } with [] @@ -47,7 +46,6 @@ type NBomberContext = { WorkerPlugins = List.empty EnableHintsAnalyzer = false TargetScenarios = None - MaxFailCount = Constants.DefaultMaxFailCount } namespace NBomber.Contracts.Internal @@ -83,7 +81,6 @@ type SessionArgs = { member this.GetReportingInterval() = this.NBomberConfig.GlobalSettings.Value.ReportingInterval.Value member this.GetReportFolder() = this.NBomberConfig.GlobalSettings.Value.ReportFolder.Value member this.GetTargetScenarios() = this.NBomberConfig.TargetScenarios.Value - member this.GetMaxFailCount() = this.NBomberConfig.GlobalSettings.Value.MaxFailCount.Value member this.SetTargetScenarios(targetScenarios) = let nbConfig = { this.NBomberConfig with TargetScenarios = Some targetScenarios } diff --git a/src/NBomber/Domain/Concurrency/Scheduler/ScenarioScheduler.fs b/src/NBomber/Domain/Concurrency/Scheduler/ScenarioScheduler.fs index dfc86d53..ab012e35 100644 --- a/src/NBomber/Domain/Concurrency/Scheduler/ScenarioScheduler.fs +++ b/src/NBomber/Domain/Concurrency/Scheduler/ScenarioScheduler.fs @@ -124,7 +124,7 @@ type ScenarioScheduler(scnCtx: ScenarioContextArgs, scenarioClusterCount: int) = _oneTimeScheduler.Stop() let execScheduler () = - if _isWorking && scnCtx.ScenarioStatsActor.ScenarioFailCount > scnCtx.MaxFailCount then + if _isWorking && scnCtx.ScenarioStatsActor.ScenarioFailCount > _scenario.MaxFailCount then stop() scnCtx.ExecStopCommand(StopCommand.StopTest $"Stopping test because of too many fails. Scenario '{_scenario.ScenarioName}' contains '{scnCtx.ScenarioStatsActor.ScenarioFailCount}' fails.") diff --git a/src/NBomber/Domain/DomainTypes.fs b/src/NBomber/Domain/DomainTypes.fs index 3f0107c9..6c337255 100644 --- a/src/NBomber/Domain/DomainTypes.fs +++ b/src/NBomber/Domain/DomainTypes.fs @@ -40,4 +40,5 @@ type Scenario = { IsEnabled: bool // used for stats in the cluster mode IsInitialized: bool ResetIterationOnFail: bool + MaxFailCount: int } diff --git a/src/NBomber/Domain/Scenario.fs b/src/NBomber/Domain/Scenario.fs index 852952b8..a05f194d 100644 --- a/src/NBomber/Domain/Scenario.fs +++ b/src/NBomber/Domain/Scenario.fs @@ -76,20 +76,21 @@ let createScenarioInfo (scenarioName: string, duration: TimeSpan, threadNumber: let createScenario (scn: ScenarioProps) = result { let! timeline = scn.LoadSimulations |> LoadTimeLine.createWithDuration - let! scenario = Validation.validate scn + let! scnProps = Validation.validate scn - return { ScenarioName = scenario.ScenarioName - Init = scenario.Init - Clean = scenario.Clean - Run = scenario.Run + return { ScenarioName = scnProps.ScenarioName + Init = scnProps.Init + Clean = scnProps.Clean + Run = scnProps.Run LoadTimeLine = timeline.LoadTimeLine - WarmUpDuration = scenario.WarmUpDuration + WarmUpDuration = scnProps.WarmUpDuration PlanedDuration = timeline.ScenarioDuration ExecutedDuration = None CustomSettings = String.Empty IsEnabled = true IsInitialized = false - ResetIterationOnFail = scn.ResetIterationOnFail } + ResetIterationOnFail = scnProps.ResetIterationOnFail + MaxFailCount = scnProps.MaxFailCount } } let createScenarios (scenarios: ScenarioProps list) = result { @@ -125,13 +126,14 @@ let applySettings (settings: ScenarioSetting list) (scenarios: Scenario list) = { scenario with LoadTimeLine = timeLine.LoadTimeLine WarmUpDuration = settings.WarmUpDuration PlanedDuration = timeLine.ScenarioDuration - CustomSettings = settings.CustomSettings |> Option.defaultValue "" } + CustomSettings = settings.CustomSettings |> Option.defaultValue "" + MaxFailCount = settings.MaxFailCount |> Option.defaultValue Constants.ScenarioMaxFailCount } scenarios |> List.map(fun scn -> settings - |> List.tryPick(fun x -> - if x.ScenarioName = scn.ScenarioName then Some(scn, x) + |> List.tryPick(fun setting -> + if setting.ScenarioName = scn.ScenarioName then Some(scn, setting) else None ) |> Option.map updateScenario diff --git a/src/NBomber/Domain/ScenarioContext.fs b/src/NBomber/Domain/ScenarioContext.fs index 69f5220e..20050540 100644 --- a/src/NBomber/Domain/ScenarioContext.fs +++ b/src/NBomber/Domain/ScenarioContext.fs @@ -15,7 +15,6 @@ type ScenarioContextArgs = { ScenarioOperation: ScenarioOperation ScenarioStatsActor: ScenarioStatsActor ExecStopCommand: StopCommand -> unit - MaxFailCount: int } type ScenarioContext(scenarioInfo, args: ScenarioContextArgs) = diff --git a/src/NBomber/DomainServices/NBomberContext.fs b/src/NBomber/DomainServices/NBomberContext.fs index 67e941b2..06d98d83 100644 --- a/src/NBomber/DomainServices/NBomberContext.fs +++ b/src/NBomber/DomainServices/NBomberContext.fs @@ -2,9 +2,7 @@ open System open System.IO - open FsToolkit.ErrorHandling - open NBomber open NBomber.Extensions.Internal open NBomber.Configuration @@ -105,16 +103,6 @@ let getEnableHintAnalyzer (context: NBomberContext) = |> tryGetFromConfig |> Option.defaultValue context.EnableHintsAnalyzer -let getMaxFailCount (context: NBomberContext) = - let tryGetFromConfig (ctx: NBomberContext) = option { - let! config = ctx.NBomberConfig - let! settings = config.GlobalSettings - return! settings.MaxFailCount - } - context - |> tryGetFromConfig - |> Option.defaultValue context.MaxFailCount - let private getReportFolder (context: NBomberContext) = let tryGetFromConfig (ctx: NBomberContext) = option { let! config = ctx.NBomberConfig @@ -171,7 +159,6 @@ let createSessionArgs (testInfo: TestInfo) (scenarios: DomainTypes.Scenario list let! reportingInterval = context |> getReportingInterval let! scenariosSettings = context |> getScenariosSettings scenarios let enableHintsAnalyzer = context |> getEnableHintAnalyzer - let maxFailCount = context |> getMaxFailCount let nbConfig = { TestSuite = Some testInfo.TestSuite @@ -184,7 +171,6 @@ let createSessionArgs (testInfo: TestInfo) (scenarios: DomainTypes.Scenario list ReportFormats = Some reportFormats ReportingInterval = Some reportingInterval EnableHintsAnalyzer = Some enableHintsAnalyzer - MaxFailCount = Some maxFailCount } } diff --git a/src/NBomber/DomainServices/TestHost/TestHost.fs b/src/NBomber/DomainServices/TestHost/TestHost.fs index 7883e2f2..f864ae29 100644 --- a/src/NBomber/DomainServices/TestHost/TestHost.fs +++ b/src/NBomber/DomainServices/TestHost/TestHost.fs @@ -68,7 +68,6 @@ type internal TestHost(dep: IGlobalDependency, regScenarios: Scenario list) as t ScenarioOperation = operation ScenarioStatsActor = createStatsActor _log scn (_sessionArgs.GetReportingInterval()) ExecStopCommand = execStopCommand - MaxFailCount = _sessionArgs.GetMaxFailCount() } let count = getScenarioClusterCount scn.ScenarioName new ScenarioScheduler(scnDep, count) diff --git a/src/NBomber/NBomber.fsproj b/src/NBomber/NBomber.fsproj index ec1d12e8..756e9661 100644 --- a/src/NBomber/NBomber.fsproj +++ b/src/NBomber/NBomber.fsproj @@ -69,7 +69,7 @@ - + diff --git a/src/NBomber/Resources/HtmlReport/assets/js/index.js b/src/NBomber/Resources/HtmlReport/assets/js/index.js index c230619c..748ed66a 100644 --- a/src/NBomber/Resources/HtmlReport/assets/js/index.js +++ b/src/NBomber/Resources/HtmlReport/assets/js/index.js @@ -196,8 +196,9 @@ const initApp = (appContainer, viewModel) => { props: ['stepStats', 'showCharts'], template: '#status-codes-template', data: function() { - const allOkStatuses = this.stepStats.flatMap(x => x.Ok.StatusCodes); - const allFailStatuses = this.stepStats.flatMap(x => x.Fail.StatusCodes); + const globalInfo = this.stepStats.find(x => x.StepName === 'global information'); + const allOkStatuses = globalInfo.Ok.StatusCodes; + const allFailStatuses = globalInfo.Fail.StatusCodes; const okReqCount = this.stepStats.reduce((acc, val) => acc + val.Ok.Request.Count, 0); const failReqCount = this.stepStats.reduce((acc, val) => acc + val.Fail.Request.Count, 0); diff --git a/tests/NBomber.IntegrationTests/Concurrency/ConstantActorSchedulerTests.fs b/tests/NBomber.IntegrationTests/Concurrency/ConstantActorSchedulerTests.fs index 5b9ed8b8..502744af 100644 --- a/tests/NBomber.IntegrationTests/Concurrency/ConstantActorSchedulerTests.fs +++ b/tests/NBomber.IntegrationTests/Concurrency/ConstantActorSchedulerTests.fs @@ -40,7 +40,6 @@ let internal baseScnDep = { ScenarioOperation = ScenarioOperation.Bombing ScenarioStatsActor = ScenarioStatsActor(logger, baseScenario, Constants.DefaultReportingInterval) ExecStopCommand = fun _ -> () - MaxFailCount = Constants.DefaultMaxFailCount } [] diff --git a/tests/NBomber.IntegrationTests/Concurrency/OneTimeActorSchedulerTests.fs b/tests/NBomber.IntegrationTests/Concurrency/OneTimeActorSchedulerTests.fs index 02188faa..3f7a4a5c 100644 --- a/tests/NBomber.IntegrationTests/Concurrency/OneTimeActorSchedulerTests.fs +++ b/tests/NBomber.IntegrationTests/Concurrency/OneTimeActorSchedulerTests.fs @@ -40,7 +40,6 @@ let internal baseScnDep = { ScenarioOperation = ScenarioOperation.Bombing ScenarioStatsActor = ScenarioStatsActor(logger, baseScenario, Constants.DefaultReportingInterval) ExecStopCommand = fun _ -> () - MaxFailCount = Constants.DefaultMaxFailCount } [] diff --git a/tests/NBomber.IntegrationTests/Configuration/test_config.json b/tests/NBomber.IntegrationTests/Configuration/test_config.json index bc507ce9..1e8746b3 100644 --- a/tests/NBomber.IntegrationTests/Configuration/test_config.json +++ b/tests/NBomber.IntegrationTests/Configuration/test_config.json @@ -23,7 +23,9 @@ "TargetHost": "localhost", "MsgSizeInBytes": 1000, "PauseMs": 100 - } + }, + + "MaxFailCount": 500 } ], @@ -33,7 +35,6 @@ "ReportFormats": ["Html", "Txt"], "ReportingInterval": "00:00:30", "EnableHintsAnalyzer": false, - "DefaultStepTimeoutMs" : 200, - "MaxFailCount": 500 + "DefaultStepTimeoutMs" : 200 } } diff --git a/tests/NBomber.IntegrationTests/NBomberContextTests.fs b/tests/NBomber.IntegrationTests/NBomberContextTests.fs index af7487a5..13e6b531 100644 --- a/tests/NBomber.IntegrationTests/NBomberContextTests.fs +++ b/tests/NBomber.IntegrationTests/NBomberContextTests.fs @@ -26,7 +26,6 @@ let baseGlobalSettings = { ReportFormats = None ReportingInterval = None EnableHintsAnalyzer = None - MaxFailCount = None } let baseScenarioSetting = { @@ -34,6 +33,7 @@ let baseScenarioSetting = { WarmUpDuration = None LoadSimulationsSettings = None CustomSettings = None + MaxFailCount = None } let baseScenario = diff --git a/tests/NBomber.IntegrationTests/ScenarioTests/InitCleanStopTests.fs b/tests/NBomber.IntegrationTests/ScenarioTests/InitCleanStopTests.fs index 5268609f..3661f3d1 100644 --- a/tests/NBomber.IntegrationTests/ScenarioTests/InitCleanStopTests.fs +++ b/tests/NBomber.IntegrationTests/ScenarioTests/InitCleanStopTests.fs @@ -183,7 +183,7 @@ let ``Test execution should be stopped if too many errors`` () = |> NBomberRunner.run |> Result.getOk |> fun stats -> - test <@ stats.ScenarioStats[0].Fail.Request.Count >= Constants.DefaultMaxFailCount @> + test <@ stats.ScenarioStats[0].Fail.Request.Count >= Constants.ScenarioMaxFailCount @> [] let ``withMaxFailCount should configure MaxFailCount`` () = @@ -197,14 +197,14 @@ let ``withMaxFailCount should configure MaxFailCount`` () = }) |> Scenario.withoutWarmUp |> Scenario.withLoadSimulations [InjectPerSec(1, duration)] + |> Scenario.withMaxFailCount maxFailCount NBomberRunner.registerScenarios [scenario1] |> NBomberRunner.withoutReports - |> NBomberRunner.withMaxFailCount maxFailCount |> NBomberRunner.run |> Result.getOk |> fun stats -> - test <@ stats.ScenarioStats[0].Fail.Request.Count <> Constants.DefaultMaxFailCount @> + test <@ stats.ScenarioStats[0].Fail.Request.Count <> Constants.ScenarioMaxFailCount @> test <@ stats.ScenarioStats[0].Fail.Request.Count = 2 @> [] @@ -225,10 +225,10 @@ let ``MaxFailCount should be tracked only for scenarios failures, not steps fail |> Scenario.withoutWarmUp |> Scenario.withResetIterationOnFail false |> Scenario.withLoadSimulations [KeepConstant(10, duration)] + |> Scenario.withMaxFailCount maxFailCount NBomberRunner.registerScenarios [scenario1] |> NBomberRunner.withoutReports - |> NBomberRunner.withMaxFailCount maxFailCount |> NBomberRunner.run |> Result.getOk |> fun stats -> diff --git a/tests/NBomber.IntegrationTests/ScenarioTests/ValidationTests.fs b/tests/NBomber.IntegrationTests/ScenarioTests/ValidationTests.fs index 3fddd0d1..8a9f99f1 100644 --- a/tests/NBomber.IntegrationTests/ScenarioTests/ValidationTests.fs +++ b/tests/NBomber.IntegrationTests/ScenarioTests/ValidationTests.fs @@ -29,6 +29,7 @@ let ``applyScenariosSettings() should override initial settings if the scenario WarmUpDuration = Some warmUp1 LoadSimulationsSettings = Some [LoadSimulation.KeepConstant(10, duration1)] CustomSettings = Some "some data" + MaxFailCount = Some Constants.ScenarioMaxFailCount } let originalScenarios = @@ -58,6 +59,7 @@ let ``applyScenariosSettings() should skip applying settings when scenario name WarmUpDuration = Some warmUp1 LoadSimulationsSettings = Some [LoadSimulation.RampConstant(5, duration1)] CustomSettings = None + MaxFailCount = Some Constants.ScenarioMaxFailCount } let scenario =