Skip to content

Commit

Permalink
[KEP-0009] feat: add expression based assertions (#576)
Browse files Browse the repository at this point in the history
This PR adds CEL-expression based assertions to `TestAsserts`. See https://github.com/kudobuilder/kuttl/blob/main/keps/0009-expression-based-assertions.md for more details.

Signed-off-by: Kumar Mallikarjuna <[email protected]>
  • Loading branch information
kumar-mallikarjuna authored Jan 24, 2025
1 parent 69968d7 commit 947aa91
Show file tree
Hide file tree
Showing 19 changed files with 844 additions and 6 deletions.
41 changes: 36 additions & 5 deletions docs/testing/reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -102,15 +102,26 @@ commands:
collectors:
- type: pod
pod: nginx
resourceRefs:
- apiVersion: apps/v1
kind: Deployment
namespace: kube-system
name: coredns
ref: coredns_deployment
assertAll:
- celExpr: "coredns_deployment.spec.replicas >= 2"
```

Supported settings:

Field | Type | Description | Default
--------|------|-------------------------------------------------------|-------------
timeout | int | Number of seconds that the test is allowed to run for | 30
collectors | list of [collectors](#collectors) | The collectors to be invoked to gather information upon step failure | N/A
commands | list of [commands](#commands) | Commands to run prior to the beginning of the test step. | N/A
Field | Type | Description | Default
--------|-----------------------------------------------------|--------------------------------------------------------------------------------------------------|-------------
timeout | int | Number of seconds that the test is allowed to run for. | 30
collectors | list of [collectors](#collectors) | The collectors to be invoked to gather information upon step failure. | N/A
commands | list of [commands](#commands) | Commands to run prior to the beginning of the test step. | N/A
resourceRefs | list of [resource references](#resource-references) | References to resources used in the expression-based assertions. | N/A
assertAll | list of [Expressions](#expressions) | List of expressions _all_ must evaluate to `true` for a successful assertion. | N/A
assertAny | list of [Expressions](#expressions) | List of expressions _at least_ one of which must evaluate to `true` for a successful assertion. | N/A

## TestFile

Expand Down Expand Up @@ -168,3 +179,23 @@ skipLogOutput | bool | If set, the output from the command is *not* logged. Us
timeout | int | Override the TestSuite timeout for this command (in seconds).

*Note*: The current working directory (CWD) for `command`/`script` is the test directory.

## Resource References

The `Resource References` objects are used by `TestAssert` for declaring identifiers for expression based assertions.

Field | Type | Description
--------------|--------|---------------------------------------------------------------------
apiVersion | string | apiVersion of the target resource.
kind | string | Kind of the target resource.
namespace | string | Namespace of the target resource. When not specified, defaults to the namespace of the current test.
name | string | Name of the target resource.
ref | string | Identifier for the resource used in the expressions.

## Expressions

The `Expressions` objects are used by `TestAssert` for declaring expressions used in assertions.

Field | Type | Description
--------------|--------|---------------------------------------------------------------------
celExpr | string | CEL Expression as per https://github.com/google/cel-spec/.
7 changes: 7 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ require (
github.com/docker/docker v27.4.1+incompatible
github.com/dustin/go-humanize v1.0.1
github.com/dustinkirkland/golang-petname v0.0.0-20191129215211-8e5a1ed0cff0
github.com/google/cel-go v0.22.0
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2
github.com/spf13/cobra v1.8.1
Expand All @@ -25,10 +26,12 @@ require (
)

require (
cel.dev/expr v0.18.0 // indirect
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
github.com/BurntSushi/toml v1.4.0 // indirect
github.com/Microsoft/go-winio v0.5.1 // indirect
github.com/alessio/shellescape v1.4.2 // indirect
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/distribution/reference v0.6.0 // indirect
Expand Down Expand Up @@ -68,12 +71,14 @@ require (
github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/stoewer/go-strcase v1.3.0 // indirect
github.com/x448/float16 v0.8.4 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
go.opentelemetry.io/otel v1.28.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 // indirect
go.opentelemetry.io/otel/metric v1.28.0 // indirect
go.opentelemetry.io/otel/trace v1.28.0 // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/mod v0.22.0 // indirect
golang.org/x/net v0.33.0 // indirect
golang.org/x/oauth2 v0.23.0 // indirect
Expand All @@ -83,6 +88,8 @@ require (
golang.org/x/text v0.21.0 // indirect
golang.org/x/time v0.7.0 // indirect
golang.org/x/tools v0.28.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect
google.golang.org/protobuf v1.35.1 // indirect
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
Expand Down
9 changes: 8 additions & 1 deletion go.sum
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
cel.dev/expr v0.18.0 h1:CJ6drgk+Hf96lkLikr4rFf19WrU0BOWEihyZnI2TAzo=
cel.dev/expr v0.18.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
Expand All @@ -8,6 +10,8 @@ github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6
github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0=
github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
Expand Down Expand Up @@ -69,6 +73,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/cel-go v0.22.0 h1:b3FJZxpiv1vTMo2/5RDUqAHPxkT8mmMfJIrq1llbf7g=
github.com/google/cel-go v0.22.0/go.mod h1:BuznPXXfQDpXKWQ9sPW3TzlAJN5zzFe+i9tIs0yC4s8=
github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
Expand Down Expand Up @@ -162,6 +168,8 @@ github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3k
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs=
github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
Expand Down Expand Up @@ -253,7 +261,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ=
google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw=
google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 h1:2035KHhUv+EpyB+hWgJnaWKJOdX1E95w2S8Rr4uWKTs=
Expand Down
65 changes: 65 additions & 0 deletions pkg/apis/testharness/v1beta1/expression.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
package v1beta1

import (
"errors"
"fmt"
"strings"

"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
)

var (
errAPIVersionInvalid = errors.New("apiVersion not of the format (<group>/)<version>")
errKindNotSpecified = errors.New("kind not specified")
errNameNotSpecified = errors.New("name not specified")
errRefNotSpecified = errors.New("ref not specified")
)

func (t *TestResourceRef) BuildResourceReference() (namespacedName types.NamespacedName, referencedResource *unstructured.Unstructured) {
referencedResource = &unstructured.Unstructured{}
apiVersionSplit := strings.Split(t.APIVersion, "/")
gvk := schema.GroupVersionKind{
Version: apiVersionSplit[len(apiVersionSplit)-1],
Kind: t.Kind,
}
if len(apiVersionSplit) > 1 {
gvk.Group = apiVersionSplit[0]
}
referencedResource.SetGroupVersionKind(gvk)

namespacedName = types.NamespacedName{
Namespace: t.Namespace,
Name: t.Name,
}

return
}

func (t *TestResourceRef) Validate() error {
apiVersionSplit := strings.Split(t.APIVersion, "/")
switch {
case t.APIVersion == "" || len(apiVersionSplit) > 2:
return errAPIVersionInvalid
case t.Kind == "":
return errKindNotSpecified
case t.Name == "":
return errNameNotSpecified
case t.Ref == "":
return errRefNotSpecified
}

return nil
}

func (t *TestResourceRef) String() string {
return fmt.Sprintf(
"apiVersion=%v, kind=%v, namespace=%v, name=%v, ref=%v",
t.APIVersion,
t.Kind,
t.Namespace,
t.Name,
t.Ref,
)
}
181 changes: 181 additions & 0 deletions pkg/apis/testharness/v1beta1/expression_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
package v1beta1

import (
"reflect"
"testing"

"github.com/stretchr/testify/assert"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
)

func TestValidate(t *testing.T) {
testCases := []struct {
name string
testResourceRef TestResourceRef
errored bool
expectedError error
}{
{
name: "apiVersion is not specified",
testResourceRef: TestResourceRef{
Kind: "Pod",
Namespace: "test",
Name: "test-pod",
Ref: "testPod",
},
errored: true,
expectedError: errAPIVersionInvalid,
},
{
name: "apiVersion is invalid",
testResourceRef: TestResourceRef{
APIVersion: "x/y/z",
Kind: "Pod",
Namespace: "test",
Name: "test-pod",
Ref: "testPod",
},
errored: true,
expectedError: errAPIVersionInvalid,
},
{
name: "apiVersion is valid and group is vacuous",
testResourceRef: TestResourceRef{
APIVersion: "v1",
Kind: "Pod",
Namespace: "test",
Name: "test-pod",
Ref: "testPod",
},
errored: false,
},
{
name: "apiVersion has both group name and version",
testResourceRef: TestResourceRef{
APIVersion: "apps/v1",
Kind: "Deployment",
Namespace: "test",
Name: "test-deployment",
Ref: "testDeployment",
},
errored: false,
},
{
name: "kind is not specified",
testResourceRef: TestResourceRef{
APIVersion: "apps/v1",
Namespace: "test",
Name: "test-deployment",
Ref: "testDeployment",
},
errored: true,
expectedError: errKindNotSpecified,
},
{
name: "name is not specified",
testResourceRef: TestResourceRef{
APIVersion: "apps/v1",
Kind: "Deployment",
Namespace: "test",
Ref: "testDeployment",
},
errored: true,
expectedError: errNameNotSpecified,
},
{
name: "ref is not specified",
testResourceRef: TestResourceRef{
APIVersion: "apps/v1",
Kind: "Deployment",
Namespace: "test",
Name: "test-deployment",
},
errored: true,
expectedError: errRefNotSpecified,
},
{
name: "all attributes are present and valid",
testResourceRef: TestResourceRef{
APIVersion: "apps/v1",
Kind: "Deployment",
Namespace: "test",
Name: "test-deployment",
Ref: "testDeployment",
},
errored: false,
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
err := tc.testResourceRef.Validate()
if !tc.errored {
assert.NoError(t, err)
} else {
assert.ErrorIs(t, err, tc.expectedError)
}
})
}
}

func TestBuildResourceReference(t *testing.T) {
buildObject := func(gvk schema.GroupVersionKind) *unstructured.Unstructured {
obj := &unstructured.Unstructured{}
obj.SetGroupVersionKind(gvk)
return obj
}

testCases := []struct {
name string
testResourceRef TestResourceRef
namespacedName types.NamespacedName
resourceReference *unstructured.Unstructured
}{
{
name: "group name is vacuous",
testResourceRef: TestResourceRef{
APIVersion: "v1",
Kind: "Pod",
Namespace: "test",
Name: "test-pod",
Ref: "testPod",
},
namespacedName: types.NamespacedName{
Namespace: "test",
Name: "test-pod",
},
resourceReference: buildObject(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}),
},
{
name: "group name is present",
testResourceRef: TestResourceRef{
APIVersion: "apps/v1",
Kind: "Deployment",
Namespace: "test",
Name: "test-deployment",
Ref: "testDeployment",
},
namespacedName: types.NamespacedName{
Namespace: "test",
Name: "test-deployment",
},
resourceReference: buildObject(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}),
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
namspacedName, referencedResource := tc.testResourceRef.BuildResourceReference()
assert.Equal(t, tc.namespacedName, namspacedName)
assert.True(
t,
reflect.DeepEqual(tc.resourceReference, referencedResource),
"constructed unstructured reference does not match, expected '%s', got '%s'",
tc.resourceReference,
referencedResource,
)
})
}
}
Loading

0 comments on commit 947aa91

Please sign in to comment.