-
-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add crd test command and GitHub action config (#96)
* feat: add crd testing capability Signed-off-by: Gergely Brautigam <[email protected]> * updated the validator and updated the string matcher Signed-off-by: Gergely Brautigam <[email protected]> * using path now and checking the right versions for the right snapshots * do not clutter the output with the general CLI help output on failure * added skipping random generation and fixed additional properties parsing * adding readme, usage and release document --------- Signed-off-by: Gergely Brautigam <[email protected]>
- Loading branch information
Showing
38 changed files
with
3,484 additions
and
52 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| package cmd | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "os" | ||
|
|
||
| "github.com/fatih/color" | ||
| "github.com/jedib0t/go-pretty/v6/table" | ||
| "github.com/jedib0t/go-pretty/v6/text" | ||
| "github.com/spf13/cobra" | ||
|
|
||
| "github.com/Skarlso/crd-to-sample-yaml/pkg/tests" | ||
| ) | ||
|
|
||
| const wrapLen = 80 | ||
|
|
||
| var ( | ||
| // testCmd is root for various `test ...` commands. | ||
| testCmd = &cobra.Command{ | ||
| Use: "test", | ||
| Short: "Run a set of tests to check CRD schema stability.", | ||
| Run: runTest, | ||
| } | ||
|
|
||
| testArgs struct { | ||
| update bool | ||
| } | ||
| ) | ||
|
|
||
| func init() { | ||
| rootCmd.AddCommand(testCmd) | ||
|
|
||
| f := testCmd.PersistentFlags() | ||
| f.BoolVarP(&testArgs.update, "update", "u", false, "Update any existing snapshots.") | ||
| } | ||
|
|
||
| func runTest(cmd *cobra.Command, args []string) { | ||
| if len(args) == 0 { | ||
| fmt.Fprintf(os.Stderr, "test needs an argument where the tests are located at") | ||
|
|
||
| os.Exit(1) | ||
| } | ||
|
|
||
| path := args[0] | ||
| runner := tests.NewSuiteRunner(path, testArgs.update) | ||
| outcome, err := runner.Run(cmd.Context()) | ||
| if err != nil { | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| if err := displayWarnings(outcome); err != nil { | ||
| os.Exit(1) | ||
| } | ||
| } | ||
|
|
||
| func displayWarnings(warnings []tests.Outcome) error { | ||
| errs := 0 | ||
|
|
||
| t := table.NewWriter() | ||
| t.SetOutputMirror(os.Stdout) | ||
| t.AppendHeader(table.Row{"Status", "It", "Matcher", "Error", "Template"}) | ||
| rows := make([]table.Row, 0, len(warnings)) | ||
| for _, w := range warnings { | ||
| if w.Error != nil { | ||
| errs++ | ||
| } | ||
|
|
||
| status := color.GreenString(w.Status) | ||
| if w.Status == "FAIL" { | ||
| status = color.RedString(w.Status) | ||
| } | ||
| var errText string | ||
| if w.Error != nil { | ||
| errText = text.WrapText(w.Error.Error(), wrapLen) | ||
| } | ||
| rows = append(rows, table.Row{ | ||
| status, w.Name, w.Matcher, errText, w.Template, | ||
| }) | ||
| } | ||
| t.AppendRows(rows) | ||
| t.AppendSeparator() | ||
| t.Render() | ||
|
|
||
| fmt.Fprintf(os.Stdout, "\nTests total: %d, failed: %d, passed: %d\n", len(warnings), errs, len(warnings)-errs) | ||
|
|
||
| if errs > 0 { | ||
| return fmt.Errorf("%d test(s) failed", errs) | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,141 @@ | ||
| # CRD Testing with CTY | ||
|
|
||
| From version `v0.8.0` cty supports the command `test`. | ||
|
|
||
| `test` supports testing CRD schemas against snapshots of generated YAML files that satisfy the schema | ||
| or small snippets of yaml strings. | ||
|
|
||
| Adding this test to your CRDs makes sure that any modification on the CRD will not break a generated snapshot of the | ||
| CRD. This is from version to version, meaning the tests will make sure that API version is respected. | ||
|
|
||
| ## Example | ||
|
|
||
| Let's look at an example. | ||
|
|
||
| Consider the following test suite definition that loosely follows the syntax of helm unittest definitions: | ||
|
|
||
| ```yaml | ||
| suite: test crd bootstrap | ||
| template: crd-bootstrap/crds/bootstrap_crd.yaml # should point to a CRD that is regularly updated like in a helm chart. | ||
| tests: | ||
| - it: matches bootstrap crds correctly | ||
| asserts: | ||
| - matchSnapshot: | ||
| # this will generate one snapshot / CRD version and match all of them to the right version of the CRD | ||
| path: sample-tests/__snapshots__ | ||
| - matchSnapshot: | ||
| path: sample-tests/__snapshots__ | ||
| # generates a yaml file | ||
| minimal: true | ||
| - it: matches some custom stuff | ||
| asserts: | ||
| - matchString: | ||
| apiVersion: v1alpha1 # this will match this exact version only from the list of versions in the CRD | ||
| kind: Bootstrap | ||
| spec: | ||
| source: | ||
| url: | ||
| url: https://github.com/Skarlso/test | ||
| ``` | ||
| Put this into a file called `bootstrap_test.yaml`. | ||
|
|
||
| **IMPORTANT**: `test` will only consider yaml files that end with `_test.yaml`. | ||
|
|
||
| One test is per CRD file. A single CRD file, however, can contain multiple apiVersions. Therefore, it's important to | ||
| only target a specific version with a snapshot, otherwise, we might be testing something that is broken intentionally. | ||
|
|
||
| Now, we can run test like this: | ||
|
|
||
| ``` | ||
| ./bin/cty test sample-tests | ||
| ``` | ||
|
|
||
| The locations in the suite are relative to the execution location. | ||
|
|
||
| Running test like this, will match the following snapshots with the given CRD assuming we have two version v1alpha1 and | ||
| v1beta1: | ||
| - bootstrap_crd-v1alpha1.yaml | ||
| - bootstrap_crd-v1alpha1.min.yaml | ||
| - bootstrap_crd-v1beta1.yaml | ||
| - bootstrap_crd-v1beta1.min.yaml | ||
|
|
||
| **_Note_**: At this release version the minimal version needs to be adjusted because it will generate an empty object without the closing `{}`. | ||
|
|
||
| If everything is okay, it will generate an output like this: | ||
|
|
||
| ``` | ||
| ./bin/cty test sample-tests | ||
| +--------+----------------------------------+---------------+-------+--------------------------------------+ | ||
| | STATUS | IT | MATCHER | ERROR | TEMPLATE | | ||
| +--------+----------------------------------+---------------+-------+--------------------------------------+ | ||
| | PASS | matches bootstrap crds correctly | matchSnapshot | | sample-tests/crds/bootstrap_crd.yaml | | ||
| | PASS | matches bootstrap crds correctly | matchSnapshot | | sample-tests/crds/bootstrap_crd.yaml | | ||
| | PASS | matches some custom stuff | matchString | | sample-tests/crds/bootstrap_crd.yaml | | ||
| +--------+----------------------------------+---------------+-------+--------------------------------------+ | ||
| Tests total: 3, failed: 0, passed: 3 | ||
| ``` | ||
|
|
||
| If there _was_ an error, it should look something like this: | ||
|
|
||
| ``` | ||
| ./bin/cty test sample-tests | ||
| +--------+----------------------------------+---------------+----------------------------------------------------------------------------------+--------------------------------------+ | ||
| | STATUS | IT | MATCHER | ERROR | TEMPLATE | | ||
| +--------+----------------------------------+---------------+----------------------------------------------------------------------------------+--------------------------------------+ | ||
| | PASS | matches bootstrap crds correctly | matchSnapshot | | sample-tests/crds/bootstrap_crd.yaml | | ||
| | PASS | matches bootstrap crds correctly | matchSnapshot | | sample-tests/crds/bootstrap_crd.yaml | | ||
| | FAIL | matches some custom stuff | matchString | matcher returned failure: failed to validate kind Bootstrap and version v1alpha1 | sample-tests/crds/bootstrap_crd.yaml | | ||
| | | | | : spec.source.url in body must be of type object: "null" | | | ||
| +--------+----------------------------------+---------------+----------------------------------------------------------------------------------+--------------------------------------+ | ||
| Tests total: 3, failed: 1, passed: 2 | ||
| ``` | ||
|
|
||
| In the above failing example, we forgot to define the URL field for source. Similarly, if the regex changes for a field | ||
| we should error and be alerted that it's a breaking change for any existing users. | ||
|
|
||
| ``` | ||
| ./bin/cty test sample-tests | ||
| +--------+-----------------------------------+---------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------+ | ||
| | STATUS | IT | MATCHER | ERROR | TEMPLATE | | ||
| +--------+-----------------------------------+---------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------+ | ||
| | PASS | matches AWSCluster crds correctly | matchSnapshot | | sample-tests/crds/infrastructure.cluster.x-k8s.io_awsclusters.yaml | | ||
| | PASS | matches AWSCluster crds correctly | matchSnapshot | | sample-tests/crds/infrastructure.cluster.x-k8s.io_awsclusters.yaml | | ||
| | PASS | matches AWSCluster crds correctly | matchString | | sample-tests/crds/infrastructure.cluster.x-k8s.io_awsclusters.yaml | | ||
| | FAIL | matches AWSCluster crds correctly | matchString | matcher returned failure: failed to validate kind AWSCluster and version v1beta2 | sample-tests/crds/infrastructure.cluster.x-k8s.io_awsclusters.yaml | | ||
| | | | | : spec.s3Bucket.name in body should match '^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$' | | | ||
| | PASS | matches bootstrap crds correctly | matchSnapshot | | sample-tests/crds/bootstrap_crd.yaml | | ||
| | PASS | matches bootstrap crds correctly | matchSnapshot | | sample-tests/crds/bootstrap_crd.yaml | | ||
| | PASS | matches some custom stuff | matchString | | sample-tests/crds/bootstrap_crd.yaml | | ||
| +--------+-----------------------------------+---------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------+ | ||
|
|
||
| Tests total: 7, failed: 1, passed: 6 | ||
| ``` | ||
| ## Updating Snapshots | ||
| In order to generate snapshots for CRDs, simply add `--update` to the command: | ||
| ``` | ||
| ./bin/cty test sample-tests --update | ||
| ``` | ||
| It should generate all snapshots and overwrite existing snapshots under the specified folder of the snapshot matcher. | ||
| Meaning consider the following yaml snippet from the above test: | ||
| ```yaml | ||
| asserts: | ||
| - matchSnapshot: | ||
| # this will generate one snapshot / CRD version and match all of them to the right version of the CRD | ||
| path: sample-tests/__snapshots__ | ||
| ``` | ||
|
|
||
| Provided this `path` the generated snapshots would end up under `sample-tests/__snapshots__` folder with a generated | ||
| name that will match the `template` field in the suite. If `template` field is changed, regenerate the tests and | ||
| delete any outdated snapshots. | ||
|
|
||
| ## Examples | ||
|
|
||
| For further examples, please see under [sample-tests](./sample-tests). |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| # v0.8.0 | ||
|
|
||
| ## MAJOR UPDATE | ||
|
|
||
| ### Changes to how values are generated | ||
|
|
||
| This update contains a few modifications to the way we generate samples. These modifications are the following: | ||
|
|
||
| - if enum values are defined for a property, choose the first one from the list whatever that is | ||
| - if there is a minimum defined for integer types, the minimum value is used | ||
| - comment is added to list items of what type they are and how much the minimum value for them is | ||
| ```yaml | ||
| volumeIDs: [] # minItems 0 of type string | ||
| ``` | ||
| - unless `no-random` is defined, now given a `Pattern` that contains a valid regex a valid value is generated that satisfies the regex | ||
| and the regex's value is commented after the value | ||
| ```yaml | ||
| name: xwjhylgy2ruc # ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ | ||
| ``` | ||
|
|
||
| The random generation can be skipped by providing the following flag to `cty`: `--no-random`. | ||
|
|
||
| ### New `test` command | ||
|
|
||
| A new command has been added that lets users unit test schema validation for generated YAML files to CRDs. | ||
|
|
||
| To read more about it, check out the readme: `crd-testing-README.md`. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.