From 48b6fbd64f6c795bc19f0a3ee7cdfc2da5d24f3b Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 23 Dec 2024 08:43:15 -0700 Subject: [PATCH] Move most docs from README to docfx --- README.md | 205 +----------------------- Xunit.Combinatorial.sln | 8 +- docfx/docs/combinatorial-vs-pairwise.md | 45 ++++++ docfx/docs/features.md | 3 - docfx/docs/getting-started.md | 23 ++- docfx/docs/toc.yml | 7 +- docfx/docs/value-sources.md | 61 +++++++ docfx/index.md | 11 +- docfx/toc.yml | 2 + samples/.editorconfig | 63 ++++++++ samples/GettingStarted.cs | 186 +++++++++++++++++++++ samples/samples.csproj | 16 ++ src/Xunit.Combinatorial/README.md | 2 +- 13 files changed, 416 insertions(+), 216 deletions(-) create mode 100644 docfx/docs/combinatorial-vs-pairwise.md delete mode 100644 docfx/docs/features.md create mode 100644 docfx/docs/value-sources.md create mode 100644 samples/.editorconfig create mode 100644 samples/GettingStarted.cs create mode 100644 samples/samples.csproj diff --git a/README.md b/README.md index c4d430a..ddf0ae4 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,7 @@ This project is available as a [NuGet package][NuPkg]. The v1.x versions of this package support xunit 2. The v2.x versions of this package support xunit 3. -## Examples - -### Auto-generated values +## Example Suppose you have this test method: @@ -46,203 +44,6 @@ The `CombinatorialDataAttribute` will supply Xunit with both `true` and `false` arguments to run the test method with, resulting in two invocations of your test method with individual results reported for each invocation. -### Custom-supplied values - -To supply your own values to pass in for each parameter, use the -`CombinatorialValuesAttribute`: - -```cs -[Theory, CombinatorialData] -public void CheckValidAge([CombinatorialValues(5, 18, 21, 25)] int age) -{ - // verifications here -} -``` - -This will run your test method four times with each of the prescribed values. - -### Combinatorial effects - -Of course it wouldn't be combinatorial without multiple parameters: - -```cs -[Theory, CombinatorialData] -public void CheckValidAge( - [CombinatorialValues(5, 18, 21, 25)] int age, - bool friendlyOfficer) -{ - // This will run with all combinations: - // 5 true - // 18 true - // 21 true - // 25 true - // 5 false - // 18 false - // 21 false - // 25 false -} -``` - -Once you have more that two parameters, the number of test cases can grow -dramatically in order to cover every possible combination. -Consider this test with 3 parameters, each taking just two values: - -```cs -[Theory, CombinatorialData] -public void CheckValidAge(bool p1, bool p2, bool p3) -{ - // Combinatorial generates these 8 test cases: - // false false false - // false false true - // false true false - // false true true - // true false false - // true false true - // true true false - // true true true -} -``` - -We already have 8 test cases. With more parameters or more values per parameter -the test cases can quickly grow to a very large number. -This can cause your test runs to take too long. This level of -exhaustive testing is often not necessary as many bugs will show up whenever -two parameters are specific values. This is called "pairwise testing" and -it generates far fewer test cases than combinatorial testing because -it only ensures there is a test case covering every combination of two -parameters, and thus can "compress" the test cases by making each test case -significantly test more than one pair. - -To use pairwise testing, use the `PairwiseDataAttribute` instead of the -`CombinatorialDataAttribute`: - -```cs -[Theory, PairwiseData] -public void CheckValidAge(bool p1, bool p2, bool p3) -{ - // Pairwise generates these 4 test cases: - // false false false - // false true true - // true false true - // true true false -} -``` - -We have cut the number of test cases in half by using pairwise instead of -combinatorial. In many cases the test reduction can be much greater. -Notice that although the test cases are fewer, you can still find a test -case that covers any *two* parameter values (thus *pair*wise). - -### Values over a range - -To run a test with a parameter over a range of values, we have -`CombinatorialRangeAttribute` to generate tests over intervals of integers. - -```cs -[Theory, CombinatorialData] -public void CombinatorialCustomRange( - [CombinatorialRange(0, 5)] int p1, - [CombinatorialRange(0, 3, 2)] int p2) -{ - // Combinatorial generates these test cases: - // 0 0 - // 1 0 - // 2 0 - // 3 0 - // 4 0 - // 0 2 - // 1 2 - // 2 2 - // 3 2 - // 4 2 -} -``` - -`CombinatorialRangeAttribute` has two distinct constructors. -When supplied with two integers `from` and `count`, Xunit -will create a test case where the parameter equals `from`, and -it will increment the parameter by 1 for `count` number of cases. - -In the second constructor, `CombinatorialRangeAttribute` -accepts three integer parameters. In the generated cases, the -parameter value will step up from the first integer to the -second integer, and the third integer specifies the interval of -which to increment. - -### Value generated by a member - -The `CombinatorialMemberDataAttribute` may be used to generate values for an individual Theory parameter -using a static member on the test class. The static member may be a field, property or method. - -A value-generating method is used here: - -```cs -public static IEnumerable GetRange(int start, int count) -{ - return Enumerable.Range(start, count); -} - -[Theory, CombinatorialData] -public void CombinatorialMemberDataFromParameterizedMethods( - [CombinatorialMemberData(nameof(GetRange), 0, 5)] int p1) -{ - Assert.True(true); -} -``` - -A value-generating property is used here: - -```cs -public static IEnumerable IntPropertyValues => GetIntMethodValues(); - -public static IEnumerable GetIntMethodValues() -{ - for (int i = 0; i < 5; i++) - { - yield return Random.Next(); - } -} - -[Theory, CombinatorialData] -public void CombinatorialMemberDataFromProperties( - [CombinatorialMemberData(nameof(IntPropertyValues))] int p1) -{ - Assert.True(true); -} -``` - -A value-generating field also works: - -```cs -public static readonly IEnumerable IntFieldValues = Enumerable.Range(0, 5).Select(_ => Random.Next()); - -[Theory, CombinatorialData] -public void CombinatorialMemberDataFromFields( - [CombinatorialMemberData(nameof(IntFieldValues))] int p2) -{ - Assert.True(true); -} -``` - -### Randomly generated values - -The `CombinatorialRandomDataAttribute` can be applied to theory parameters to generate random integer values. -The min, max, and number of values can all be set via named parameters. - -```cs -[Theory, CombinatorialData] -public void CombinatorialRandomValuesCount( - [CombinatorialRandomData(Count = 10)] int p1) -{ - Assert.InRange(p1, 0, int.MaxValue); -} - -[Theory, CombinatorialData] -public void CombinatorialRandomValuesCountMinMaxValues( - [CombinatorialRandomData(Count = 10, Minimum = -20, Maximum = -5)] int p1) -{ - Assert.InRange(p1, -20, -5); -} -``` +[Learn much more on our docs site](https://aarnott.github.io/Xunit.Combinatorial/). - [NuPkg]: https://www.nuget.org/packages/Xunit.Combinatorial +[NuPkg]: https://www.nuget.org/packages/Xunit.Combinatorial diff --git a/Xunit.Combinatorial.sln b/Xunit.Combinatorial.sln index 6a259e2..20933b4 100644 --- a/Xunit.Combinatorial.sln +++ b/Xunit.Combinatorial.sln @@ -1,4 +1,4 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 +Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.1.31824.51 MinimumVisualStudioVersion = 10.0.40219.1 @@ -34,6 +34,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{8565DFAF-D test\Directory.Build.targets = test\Directory.Build.targets EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "samples", "samples\samples.csproj", "{0059A5FA-3211-42C9-9828-578DFC77AD14}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -48,6 +50,10 @@ Global {81D4F7CC-A88A-40E4-A1D2-A34BDE0BC2DF}.Debug|Any CPU.Build.0 = Debug|Any CPU {81D4F7CC-A88A-40E4-A1D2-A34BDE0BC2DF}.Release|Any CPU.ActiveCfg = Release|Any CPU {81D4F7CC-A88A-40E4-A1D2-A34BDE0BC2DF}.Release|Any CPU.Build.0 = Release|Any CPU + {0059A5FA-3211-42C9-9828-578DFC77AD14}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0059A5FA-3211-42C9-9828-578DFC77AD14}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0059A5FA-3211-42C9-9828-578DFC77AD14}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0059A5FA-3211-42C9-9828-578DFC77AD14}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/docfx/docs/combinatorial-vs-pairwise.md b/docfx/docs/combinatorial-vs-pairwise.md new file mode 100644 index 0000000..31d5a5c --- /dev/null +++ b/docfx/docs/combinatorial-vs-pairwise.md @@ -0,0 +1,45 @@ +# Combinatorial vs. pairwise + +Combinatorial and pairwise are equivalent while using only 1 or 2 parameters in your theory. +For 1-2 parameters, each of these schemes would produce a test case of every combination of every allowed value for each parameter. + +For example, consider this two parameter theory: + +[!code-csharp[](../../samples/GettingStarted.cs#CombinatorialTwoParameters)] + +## Combinatorial testing + +Once you have more than two parameters, the number of test cases grow exponentially in order to cover every possible combination when using @Xunit.CombinatorialDataAttribute. +Consider this test with 3 parameters, each taking just two values: + +[!code-csharp[](../../samples/GettingStarted.cs#CombinatorialThreeParameters)] + +We already have 8 test cases for just 3 @System.Boolean parameters. +With more parameters or more values per parameter the test cases can quickly grow to a very large number. +In general, the exponential function is: + +$$a^p$$ + +Where `a` is the number of allowed possible values for an argument and `p` is the number of parameters. +Or if parameters each have a unique number of possible values, the combinatorial explosion is modeled as: + +$$p_1 \times p_2 \times p_3 \times ...$$ + +Where pn is the number of possibles values of the parameter at index `n`. + +## Pairwise testing + +An exponential explosion of test cases can cause your test runs to take too long. +This level of exhaustive testing is often not necessary as many bugs will show up given a combination of just two parameters with particular values. +Pairwise testing focuses on this idea and it generates far fewer test cases than combinatorial testing because it only ensures there is a test case covering every combination of two parameters. +It does this in a clever way that can "compress" the test cases by making each test case significantly test more than one pair. + +To use pairwise testing, use the @Xunit.PairwiseDataAttribute instead of the +@Xunit.CombinatorialDataAttribute: + +[!code-csharp[](../../samples/GettingStarted.cs#PairwiseThreeParameters)] + +We have cut the number of test cases in half by using pairwise instead of combinatorial. +As parameter count rises or allowed values per parameter are more than 2, the test case reduction by switching from combinatorial to pairwise can be much greater. + +Notice that although the test cases are fewer, you can still find a test case that covers any *two* parameter values (thus *pair*wise). diff --git a/docfx/docs/features.md b/docfx/docs/features.md deleted file mode 100644 index eb3fbeb..0000000 --- a/docfx/docs/features.md +++ /dev/null @@ -1,3 +0,0 @@ -# Features - -TODO diff --git a/docfx/docs/getting-started.md b/docfx/docs/getting-started.md index eff2c2f..0ec4e96 100644 --- a/docfx/docs/getting-started.md +++ b/docfx/docs/getting-started.md @@ -5,8 +5,25 @@ Consume this library via its NuGet Package. Click on the badge to find its latest version and the instructions for consuming it that best apply to your project. -[![NuGet package](https://img.shields.io/nuget/v/Library.svg)](https://nuget.org/packages/Library) +[![NuGet package](https://img.shields.io/nuget/v/Xunit.Combinatorial.svg)](https://nuget.org/packages/Xunit.Combinatorial) -## Usage +> [!NOTE] +> Xunit.Combinatorial v1.x supports Xunit 2. +> +> Xunit.Combinatorial v2.x supports Xunit 3. -TODO +## Introductory example + +Suppose you have this test method: + +[!code-csharp[](../../samples/GettingStarted.cs#CheckFileSystemFact)] + +To arrange for your test method to be invoked twice, once for each of two modes, add a `bool` parameter, make it a theory, and add @Xunit.CombinatorialDataAttribute or @Xunit.PairwiseDataAttribute. + +[!code-csharp[](../../samples/GettingStarted.cs#CombinatorialBool)] + +The @Xunit.CombinatorialDataAttribute or @Xunit.CombinatorialDataAttribute will supply Xunit with both `true` and `false` arguments to run the test method with, resulting in two invocations of your test method with individual results reported for each invocation. + +[Learn more about the difference between @Xunit.CombinatorialDataAttribute and @Xunit.CombinatorialDataAttribute](combinatorial-vs-pairwise.md). + +[Learn more about supported parameter types and where values come from](value-sources.md). diff --git a/docfx/docs/toc.yml b/docfx/docs/toc.yml index 518ffad..993ce12 100644 --- a/docfx/docs/toc.yml +++ b/docfx/docs/toc.yml @@ -1,5 +1,4 @@ items: -- name: Features - href: features.md -- name: Getting Started - href: getting-started.md +- href: getting-started.md +- href: combinatorial-vs-pairwise.md +- href: value-sources.md diff --git a/docfx/docs/value-sources.md b/docfx/docs/value-sources.md new file mode 100644 index 0000000..8aa230d --- /dev/null +++ b/docfx/docs/value-sources.md @@ -0,0 +1,61 @@ +# Value sources + +In this article we will generally use @Xunit.CombinatorialDataAttribute. +Please note that @Xunit.PairwiseDataAttribute is equally applicable in all these samples. + +## Enumerable values + +Parameter types with allowed values that can already be enumerated will be provided those values by default. +Consider this case of using @System.Boolean as the parameter type: + +[!code-csharp[](../../samples/GettingStarted.cs#CombinatorialBool)] + +The @Xunit.CombinatorialDataAttribute (or @"Xunit.PairwiseDataAttribute") will supply Xunit with both `true` and `false` arguments to run the test method with, resulting in two invocations of your test method with individual results reported for each invocation. + +Using a C# `enum` type will similarly yield each enum value automatically for test case generation. + +## Custom-supplied values + +To supply your own values to pass in for each parameter, use the +@Xunit.CombinatorialValuesAttribute: + +[!code-csharp[](../../samples/GettingStarted.cs#CombinatorialValues)] + +This will run your test method four times with each of the prescribed values. + +## Values over a range + +To run a test with a parameter over a range of values, we have +@Xunit.CombinatorialRangeAttribute to generate tests over intervals of integers. + +[!code-csharp[](../../samples/GettingStarted.cs#ValuesOverRange)] + +@Xunit.CombinatorialRangeAttribute has two distinct constructors. +When supplied with two integers `from` and `count`, Xunit will create a test case where the parameter equals `from`, and it will increment the parameter by 1 for `count` number of cases. + +In the second constructor, @Xunit.CombinatorialRangeAttribute accepts three integer parameters. +In the generated cases, the parameter value will step up from the first integer to the second integer, and the third integer specifies the interval of which to increment. + +## Value generated by a member + +The @Xunit.CombinatorialMemberDataAttribute may be used to generate values for an individual Theory parameter using a static member on the test class. +The static member may be a field, property or method. + +A value-generating method is used here: + +[!code-csharp[](../../samples/GettingStarted.cs#GeneratedByMethod)] + +A value-generating property is used here: + +[!code-csharp[](../../samples/GettingStarted.cs#GeneratedByProperty)] + +A value-generating field also works: + +[!code-csharp[](../../samples/GettingStarted.cs#GeneratedByField)] + +## Randomly generated values + +The @Xunit.CombinatorialRandomDataAttribute can be applied to theory parameters to generate random integer values. +The min, max, and number of values can all be set via named parameters. + +[!code-csharp[](../../samples/GettingStarted.cs#CombinatorialRandomData)] diff --git a/docfx/index.md b/docfx/index.md index 391ee6e..fb495e2 100644 --- a/docfx/index.md +++ b/docfx/index.md @@ -4,6 +4,13 @@ _layout: landing # Overview -This is your docfx landing page. +Check out our [getting started](docs/getting-started.md) docs. -Click "Docs" across the top to get started. +## Features + +- Combinatorial for exhaustive testing. +- Pairwise for reasonable coverage without exponentially growing test cases. +- Simplify test theories with automatically generated inputs. + - All values in a range + - Random values + - Values generated from members diff --git a/docfx/toc.yml b/docfx/toc.yml index 8e9a670..ce617ad 100644 --- a/docfx/toc.yml +++ b/docfx/toc.yml @@ -3,3 +3,5 @@ items: href: docs/ - name: API href: api/ +- name: GitHub + href: https://github.com/AArnott/Xunit.Combinatorial diff --git a/samples/.editorconfig b/samples/.editorconfig new file mode 100644 index 0000000..84d3968 --- /dev/null +++ b/samples/.editorconfig @@ -0,0 +1,63 @@ +[*.cs] + +indent_style = space + +# SA1108: Block statements should not contain embedded comments +dotnet_diagnostic.SA1108.severity = none + +# SA1123: Do not place regions within elements +dotnet_diagnostic.SA1123.severity = none + +# SA1124: Do not use regions +dotnet_diagnostic.SA1124.severity = none + +# SA1200: Using directives should be placed correctly +dotnet_diagnostic.SA1200.severity = none + +# SA1201: Elements should appear in the correct order +dotnet_diagnostic.SA1201.severity = silent + +# SA1205: Partial elements should declare access +dotnet_diagnostic.SA1205.severity = none + +# SA1400: Access modifier should be declared +dotnet_diagnostic.SA1400.severity = none + +# SA1402: File may only contains a single type +dotnet_diagnostic.SA1402.severity = none + +# SA1403: File may only contain a single namespace +dotnet_diagnostic.SA1403.severity = none + +# SA1502: Element should not be on a single line +dotnet_diagnostic.SA1502.severity = none + +# SA1515: Single-line comment should be preceded by blank line +dotnet_diagnostic.SA1515.severity = none + +# SA1516: Elements should be separated by blank line +dotnet_diagnostic.SA1516.severity = none + +# SA1600: Elements should be documented +dotnet_diagnostic.SA1600.severity = silent + +# SA1601: Partial elements should be documented +dotnet_diagnostic.SA1601.severity = silent + +# SA1649: File name should match first type name +dotnet_diagnostic.SA1649.severity = none + +# IDE0051: Remove unused private members +dotnet_diagnostic.IDE0051.severity = none + +# CS1591: Missing XML comment for publicly visible type or member +dotnet_diagnostic.CS1591.severity = silent + +# CA1822: Mark members as static +dotnet_diagnostic.CA1822.severity = silent + +# CA1062: Validate arguments of public methods +dotnet_diagnostic.CA1062.severity = silent + +# xUnit1026: Theory methods should use all of their parameters +dotnet_diagnostic.xUnit1026.severity = suggestion diff --git a/samples/GettingStarted.cs b/samples/GettingStarted.cs new file mode 100644 index 0000000..2c7d8b2 --- /dev/null +++ b/samples/GettingStarted.cs @@ -0,0 +1,186 @@ +// Copyright (c) Andrew Arnott. All rights reserved. +// Licensed under the Ms-PL license. See LICENSE file in the project root for full license information. + +using Xunit; + +public class Facts +{ + #region CheckFileSystemFact + [Fact] + public void CheckFileSystem() + { + // verifications here + } + #endregion +} + +public class CombinatorialSimple +{ + #region CombinatorialBool + [Theory, CombinatorialData] + public void CheckFileSystem(bool recursive) + { + // verifications here + } + #endregion + + #region CombinatorialValues + [Theory, CombinatorialData] + public void CheckValidAge([CombinatorialValues(5, 18, 21, 25)] int age) + { + // verifications here + } + #endregion +} + +public class CombinatorialTwoParameters +{ + #region CombinatorialTwoParameters + [Theory, CombinatorialData] + public void CheckValidAge( + [CombinatorialValues(5, 18, 21, 25)] int age, + bool friendlyOfficer) + { + // This will run with all combinations: + // 5 true + // 18 true + // 21 true + // 25 true + // 5 false + // 18 false + // 21 false + // 25 false + } + #endregion +} + +public class CombinatorialThreeParameters +{ + #region CombinatorialThreeParameters + [Theory, CombinatorialData] + public void CheckValidAge(bool p1, bool p2, bool p3) + { + // Combinatorial generates these 8 test cases: + // false false false + // false false true + // false true false + // false true true + // true false false + // true false true + // true true false + // true true true + } + #endregion +} + +public class PairwiseThreeParameters +{ + #region PairwiseThreeParameters + [Theory, PairwiseData] + public void CheckValidAge(bool p1, bool p2, bool p3) + { + // Pairwise generates these 4 test cases: + // false false false + // false true true + // true false true + // true true false + } + #endregion +} + +public class ValuesOverRange +{ + #region ValuesOverRange + [Theory, CombinatorialData] + public void CombinatorialCustomRange( + [CombinatorialRange(0, 5)] int p1, + [CombinatorialRange(0, 3, 2)] int p2) + { + // Combinatorial generates these test cases: + // 0 0 + // 1 0 + // 2 0 + // 3 0 + // 4 0 + // 0 2 + // 1 2 + // 2 2 + // 3 2 + // 4 2 + } + #endregion +} + +public class GeneratedByMethod +{ + #region GeneratedByMethod + public static IEnumerable GetRange(int start, int count) + { + return Enumerable.Range(start, count); + } + + [Theory, CombinatorialData] + public void CombinatorialMemberDataFromParameterizedMethods( + [CombinatorialMemberData(nameof(GetRange), 0, 5)] int p1) + { + Assert.True(true); + } + #endregion +} + +public class GeneratedByProperty +{ + #region GeneratedByProperty + public static IEnumerable IntPropertyValues + { + get + { + for (int i = 0; i < 5; i++) + { + yield return Random.Shared.Next(); + } + } + } + + [Theory, CombinatorialData] + public void CombinatorialMemberDataFromProperties( + [CombinatorialMemberData(nameof(IntPropertyValues))] int p1) + { + Assert.True(true); + } + #endregion +} + +public class GeneratedByField +{ + #region GeneratedByField + public static readonly IEnumerable IntFieldValues = + Enumerable.Range(0, 5).Select(_ => Random.Shared.Next()); + + [Theory, CombinatorialData] + public void CombinatorialMemberDataFromFields( + [CombinatorialMemberData(nameof(IntFieldValues))] int p2) + { + Assert.True(true); + } + #endregion +} + +public class CombinatorialRandomData +{ + #region CombinatorialRandomData + [Theory, CombinatorialData] + public void CombinatorialRandomValuesCount( + [CombinatorialRandomData(Count = 10)] int p1) + { + Assert.InRange(p1, 0, int.MaxValue); + } + + [Theory, CombinatorialData] + public void CombinatorialRandomValuesCountMinMaxValues( + [CombinatorialRandomData(Count = 10, Minimum = -20, Maximum = -5)] int p1) + { + Assert.InRange(p1, -20, -5); + } + #endregion +} diff --git a/samples/samples.csproj b/samples/samples.csproj new file mode 100644 index 0000000..75f2541 --- /dev/null +++ b/samples/samples.csproj @@ -0,0 +1,16 @@ + + + + net9.0 + false + + + + + + + + + + + diff --git a/src/Xunit.Combinatorial/README.md b/src/Xunit.Combinatorial/README.md index 60d82c3..2337ceb 100644 --- a/src/Xunit.Combinatorial/README.md +++ b/src/Xunit.Combinatorial/README.md @@ -3,4 +3,4 @@ The v1.x versions of this package support xunit 2. The v2.x versions of this package support xunit 3. -Check out the documentation at [the GitHub README](https://github.com/AArnott/Xunit.Combinatorial?tab=readme-ov-file#readme). +[Check out the full documentation](https://aarnott.github.io/Xunit.Combinatorial/).