Skip to content

Commit

Permalink
test: add E2E test benchmark
Browse files Browse the repository at this point in the history
  • Loading branch information
hperl committed Jan 7, 2025
1 parent bb61839 commit 05f5180
Show file tree
Hide file tree
Showing 7 changed files with 162 additions and 82 deletions.
29 changes: 15 additions & 14 deletions internal/e2e/cli_client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,13 @@ import (
"testing"
"time"

"github.com/ory/herodot"
"github.com/ory/x/cmdx"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
grpcHealthV1 "google.golang.org/grpc/health/grpc_health_v1"

"github.com/ory/herodot"
"github.com/ory/x/cmdx"

gprclient "github.com/ory/keto/cmd/client"
cliexpand "github.com/ory/keto/cmd/expand"
clirelationtuple "github.com/ory/keto/cmd/relationtuple"
Expand All @@ -30,19 +31,19 @@ type cliClient struct {
c *cmdx.CommandExecuter
}

func (g *cliClient) queryNamespaces(t *testing.T) (res ketoapi.GetNamespacesResponse) {
func (g *cliClient) queryNamespaces(t testing.TB) (res ketoapi.GetNamespacesResponse) {
t.Skip("not implemented for the CLI")
return
}

var _ client = (*cliClient)(nil)

func (g *cliClient) oplCheckSyntax(t *testing.T, _ []byte) []*ketoapi.ParseError {
func (g *cliClient) oplCheckSyntax(t testing.TB, content []byte) []*ketoapi.ParseError {
t.Skip("not implemented as a command yet")
return []*ketoapi.ParseError{}
}

func (g *cliClient) createTuple(t *testing.T, r *ketoapi.RelationTuple) {
func (g *cliClient) createTuple(t testing.TB, r *ketoapi.RelationTuple) {
tupleEnc, err := json.Marshal(r)
require.NoError(t, err)

Expand Down Expand Up @@ -78,7 +79,7 @@ func (g *cliClient) assembleQueryFlags(q *ketoapi.RelationQuery, opts []x.Pagina
return flags
}

func (g *cliClient) queryTuple(t *testing.T, q *ketoapi.RelationQuery, opts ...x.PaginationOptionSetter) *ketoapi.GetResponse {
func (g *cliClient) queryTuple(t testing.TB, q *ketoapi.RelationQuery, opts ...x.PaginationOptionSetter) *ketoapi.GetResponse {
out := g.c.ExecNoErr(t, append(g.assembleQueryFlags(q, opts), "relation-tuple", "get")...)

var resp ketoapi.GetResponse
Expand All @@ -87,13 +88,13 @@ func (g *cliClient) queryTuple(t *testing.T, q *ketoapi.RelationQuery, opts ...x
return &resp
}

func (g *cliClient) queryTupleErr(t *testing.T, expected herodot.DefaultError, q *ketoapi.RelationQuery, opts ...x.PaginationOptionSetter) {
func (g *cliClient) queryTupleErr(t testing.TB, expected herodot.DefaultError, q *ketoapi.RelationQuery, opts ...x.PaginationOptionSetter) {
stdErr := g.c.ExecExpectedErr(t, append(g.assembleQueryFlags(q, opts), "relation-tuple", "get")...)
assert.Contains(t, stdErr, expected.GRPCCodeField.String())
assert.Contains(t, stdErr, expected.Error())
}

func (g *cliClient) check(t *testing.T, r *ketoapi.RelationTuple) bool {
func (g *cliClient) check(t testing.TB, r *ketoapi.RelationTuple) bool {
var sub string
if r.SubjectID != nil {
sub = *r.SubjectID
Expand All @@ -106,23 +107,23 @@ func (g *cliClient) check(t *testing.T, r *ketoapi.RelationTuple) bool {
return res.Allowed
}

func (g *cliClient) batchCheckErr(t *testing.T, requestTuples []*ketoapi.RelationTuple, expected herodot.DefaultError) {
func (g *cliClient) batchCheckErr(t testing.TB, requestTuples []*ketoapi.RelationTuple, expected herodot.DefaultError) {
t.Skip("not implemented for the CLI")
}

func (g *cliClient) batchCheck(t *testing.T, requestTuples []*ketoapi.RelationTuple) []checkResponse {
func (g *cliClient) batchCheck(t testing.TB, r []*ketoapi.RelationTuple) []checkResponse {
t.Skip("not implemented for the CLI")
return nil
}

func (g *cliClient) expand(t *testing.T, r *ketoapi.SubjectSet, depth int) *ketoapi.Tree[*ketoapi.RelationTuple] {
func (g *cliClient) expand(t testing.TB, r *ketoapi.SubjectSet, depth int) *ketoapi.Tree[*ketoapi.RelationTuple] {
out := g.c.ExecNoErr(t, "expand", r.Relation, r.Namespace, r.Object, "--"+cliexpand.FlagMaxDepth, fmt.Sprintf("%d", depth), "--"+cmdx.FlagFormat, string(cmdx.FormatJSON))
res := ketoapi.Tree[*ketoapi.RelationTuple]{}
require.NoError(t, json.Unmarshal([]byte(out), &res))
return &res
}

func (g *cliClient) waitUntilLive(t *testing.T) {
func (g *cliClient) waitUntilLive(t testing.TB) {
flags := make([]string, len(g.c.PersistentArgs))
copy(flags, g.c.PersistentArgs)

Expand All @@ -140,7 +141,7 @@ func (g *cliClient) waitUntilLive(t *testing.T) {
require.Equal(t, grpcHealthV1.HealthCheckResponse_SERVING.String()+"\n", out)
}

func (g *cliClient) deleteTuple(t *testing.T, r *ketoapi.RelationTuple) {
func (g *cliClient) deleteTuple(t testing.TB, r *ketoapi.RelationTuple) {
tupleEnc, err := json.Marshal(r)
require.NoError(t, err)

Expand All @@ -149,6 +150,6 @@ func (g *cliClient) deleteTuple(t *testing.T, r *ketoapi.RelationTuple) {
assert.Len(t, stderr, 0, stdout)
}

func (g *cliClient) deleteAllTuples(t *testing.T, q *ketoapi.RelationQuery) {
func (g *cliClient) deleteAllTuples(t testing.TB, q *ketoapi.RelationQuery) {
_ = g.c.ExecNoErr(t, append(g.assembleQueryFlags(q, nil), "relation-tuple", "delete-all", "--force")...)
}
76 changes: 76 additions & 0 deletions internal/e2e/e2e_benchmark_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package e2e

import (
"fmt"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/ory/keto/internal/namespace"
"github.com/ory/keto/internal/x/dbx"
"github.com/ory/keto/ketoapi"
"github.com/ory/x/pointerx"
)

func BenchmarkE2E(b *testing.B) {

dsn := dbx.GetSqlite(b, dbx.SQLiteMemory)
ctx, reg, namespaceTestMgr, getAddr := newInitializedReg(b, dsn, map[string]interface{}{"log.level": "panic"})
closeServer := startServer(ctx, b, reg)
b.Cleanup(closeServer)

_, _, readAddr := getAddr(b, "read")
_, _, writeAddr := getAddr(b, "write")
_, _, oplAddr := getAddr(b, "opl")

for _, cl := range []client{
newGrpcClient(b, ctx,
readAddr,
writeAddr,
oplAddr,
),
&restClient{
readURL: "http://" + readAddr,
writeURL: "http://" + writeAddr,
oplSyntaxURL: "http://" + oplAddr,
},
&sdkClient{
readRemote: readAddr,
writeRemote: writeAddr,
syntaxRemote: oplAddr,
},
} {

b.Run(fmt.Sprintf("client=%T", cl), func(b *testing.B) {
n := &namespace.Namespace{Name: "test"}
namespaceTestMgr.add(b, n)
cl.waitUntilLive(b)
b.ResetTimer()
for i := 0; i < b.N; i++ {
tuple := &ketoapi.RelationTuple{
Namespace: n.Name,
Object: fmt.Sprintf("object %d for client %T", i, cl),
Relation: "access",
SubjectID: pointerx.Ptr("client"),
}
cl.createTuple(b, tuple)

resp := cl.queryTuple(b, &ketoapi.RelationQuery{Namespace: &tuple.Namespace})
require.Len(b, resp.RelationTuples, 1)
assert.Equal(b, tuple, resp.RelationTuples[0])

assert.True(b, cl.check(b, tuple))
batchResult := cl.batchCheck(b, []*ketoapi.RelationTuple{tuple})
require.Len(b, batchResult, 1)
assert.True(b, batchResult[0].allowed)
assert.Empty(b, batchResult[0].errorMessage)

cl.deleteTuple(b, tuple)
resp = cl.queryTuple(b, &ketoapi.RelationQuery{Namespace: &tuple.Namespace})
require.Len(b, resp.RelationTuples, 0)
}
})
}

}
33 changes: 17 additions & 16 deletions internal/e2e/full_suit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,14 @@ import (
"testing"
"time"

"github.com/ory/herodot"
"github.com/ory/x/cmdx"
prometheus "github.com/ory/x/prometheusx"
"github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/ory/herodot"
"github.com/ory/x/cmdx"
prometheus "github.com/ory/x/prometheusx"

"github.com/ory/keto/cmd"
cliclient "github.com/ory/keto/cmd/client"
"github.com/ory/keto/internal/relationtuple"
Expand All @@ -28,21 +29,21 @@ import (
type (
transactClient interface {
client
transactTuples(t *testing.T, ins []*ketoapi.RelationTuple, del []*ketoapi.RelationTuple)
transactTuples(t testing.TB, ins []*ketoapi.RelationTuple, del []*ketoapi.RelationTuple)
}
client interface {
createTuple(t *testing.T, r *ketoapi.RelationTuple)
deleteTuple(t *testing.T, r *ketoapi.RelationTuple)
deleteAllTuples(t *testing.T, q *ketoapi.RelationQuery)
queryTuple(t *testing.T, q *ketoapi.RelationQuery, opts ...x.PaginationOptionSetter) *ketoapi.GetResponse
queryTupleErr(t *testing.T, expected herodot.DefaultError, q *ketoapi.RelationQuery, opts ...x.PaginationOptionSetter)
check(t *testing.T, r *ketoapi.RelationTuple) bool
batchCheck(t *testing.T, r []*ketoapi.RelationTuple) []checkResponse
batchCheckErr(t *testing.T, requestTuples []*ketoapi.RelationTuple, expected herodot.DefaultError)
expand(t *testing.T, r *ketoapi.SubjectSet, depth int) *ketoapi.Tree[*ketoapi.RelationTuple]
oplCheckSyntax(t *testing.T, content []byte) []*ketoapi.ParseError
waitUntilLive(t *testing.T)
queryNamespaces(t *testing.T) ketoapi.GetNamespacesResponse
createTuple(t testing.TB, r *ketoapi.RelationTuple)
deleteTuple(t testing.TB, r *ketoapi.RelationTuple)
deleteAllTuples(t testing.TB, q *ketoapi.RelationQuery)
queryTuple(t testing.TB, q *ketoapi.RelationQuery, opts ...x.PaginationOptionSetter) *ketoapi.GetResponse
queryTupleErr(t testing.TB, expected herodot.DefaultError, q *ketoapi.RelationQuery, opts ...x.PaginationOptionSetter)
check(t testing.TB, r *ketoapi.RelationTuple) bool
batchCheck(t testing.TB, r []*ketoapi.RelationTuple) []checkResponse
batchCheckErr(t testing.TB, requestTuples []*ketoapi.RelationTuple, expected herodot.DefaultError)
expand(t testing.TB, r *ketoapi.SubjectSet, depth int) *ketoapi.Tree[*ketoapi.RelationTuple]
oplCheckSyntax(t testing.TB, content []byte) []*ketoapi.ParseError
waitUntilLive(t testing.TB)
queryNamespaces(t testing.TB) ketoapi.GetNamespacesResponse
}
)

Expand Down
34 changes: 17 additions & 17 deletions internal/e2e/grpc_client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,15 @@ import (

rts "github.com/ory/keto/proto/ory/keto/relation_tuples/v1alpha2"

"github.com/ory/herodot"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
grpcHealthV1 "google.golang.org/grpc/health/grpc_health_v1"
"google.golang.org/grpc/status"

"github.com/ory/herodot"

"github.com/ory/keto/internal/x"
)

Expand All @@ -30,7 +31,7 @@ type grpcClient struct {
ctx context.Context
}

func newGrpcClient(t *testing.T, ctx context.Context, readRemote, writeRemote, oplSyntaxRemote string) *grpcClient {
func newGrpcClient(t testing.TB, ctx context.Context, readRemote, writeRemote, oplSyntaxRemote string) *grpcClient {
read, err := grpc.NewClient(readRemote, grpc.WithTransportCredentials(insecure.NewCredentials()))
require.NoError(t, err)
t.Cleanup(func() { read.Close() })
Expand All @@ -54,7 +55,7 @@ func newGrpcClient(t *testing.T, ctx context.Context, readRemote, writeRemote, o
}
}

func (g *grpcClient) queryNamespaces(t *testing.T) (apiResponse ketoapi.GetNamespacesResponse) {
func (g *grpcClient) queryNamespaces(t testing.TB) (apiResponse ketoapi.GetNamespacesResponse) {
client := rts.NewNamespacesServiceClient(g.read)
res, err := client.ListNamespaces(g.ctx, &rts.ListNamespacesRequest{})
require.NoError(t, err)
Expand All @@ -65,7 +66,7 @@ func (g *grpcClient) queryNamespaces(t *testing.T) (apiResponse ketoapi.GetNames

var _ transactClient = (*grpcClient)(nil)

func (g *grpcClient) createTuple(t *testing.T, r *ketoapi.RelationTuple) {
func (g *grpcClient) createTuple(t testing.TB, r *ketoapi.RelationTuple) {
g.transactTuples(t, []*ketoapi.RelationTuple{r}, nil)
}

Expand All @@ -83,7 +84,7 @@ func (*grpcClient) createQuery(q *ketoapi.RelationQuery) *rts.RelationQuery {
return query
}

func (g *grpcClient) queryTuple(t *testing.T, q *ketoapi.RelationQuery, opts ...x.PaginationOptionSetter) *ketoapi.GetResponse {
func (g *grpcClient) queryTuple(t testing.TB, q *ketoapi.RelationQuery, opts ...x.PaginationOptionSetter) *ketoapi.GetResponse {
c := rts.NewReadServiceClient(g.read)
pagination := x.GetPaginationOptions(opts...)

Expand All @@ -106,7 +107,7 @@ func (g *grpcClient) queryTuple(t *testing.T, q *ketoapi.RelationQuery, opts ...
}
}

func (g *grpcClient) queryTupleErr(t *testing.T, expected herodot.DefaultError, q *ketoapi.RelationQuery, opts ...x.PaginationOptionSetter) {
func (g *grpcClient) queryTupleErr(t testing.TB, expected herodot.DefaultError, q *ketoapi.RelationQuery, opts ...x.PaginationOptionSetter) {
c := rts.NewReadServiceClient(g.read)
pagination := x.GetPaginationOptions(opts...)

Expand All @@ -121,7 +122,7 @@ func (g *grpcClient) queryTupleErr(t *testing.T, expected herodot.DefaultError,
assert.Equal(t, expected.GRPCCodeField, s.Code(), "%+v", err)
}

func (g *grpcClient) check(t *testing.T, r *ketoapi.RelationTuple) bool {
func (g *grpcClient) check(t testing.TB, r *ketoapi.RelationTuple) bool {
c := rts.NewCheckServiceClient(g.read)

req := &rts.CheckRequest{
Expand All @@ -147,8 +148,7 @@ type checkResponse struct {
errorMessage string
}

func (g *grpcClient) batchCheckErr(t *testing.T, requestTuples []*ketoapi.RelationTuple,
expected herodot.DefaultError) {
func (g *grpcClient) batchCheckErr(t testing.TB, requestTuples []*ketoapi.RelationTuple, expected herodot.DefaultError) {

_, err := g.doBatchCheck(t, requestTuples)
require.Error(t, err)
Expand All @@ -158,7 +158,7 @@ func (g *grpcClient) batchCheckErr(t *testing.T, requestTuples []*ketoapi.Relati
assert.Contains(t, s.Message(), expected.Reason())
}

func (g *grpcClient) batchCheck(t *testing.T, requestTuples []*ketoapi.RelationTuple) []checkResponse {
func (g *grpcClient) batchCheck(t testing.TB, requestTuples []*ketoapi.RelationTuple) []checkResponse {
resp, err := g.doBatchCheck(t, requestTuples)
require.NoError(t, err)

Expand All @@ -173,7 +173,7 @@ func (g *grpcClient) batchCheck(t *testing.T, requestTuples []*ketoapi.RelationT
return checkResponses
}

func (g *grpcClient) doBatchCheck(_ *testing.T, requestTuples []*ketoapi.RelationTuple) (*rts.BatchCheckResponse, error) {
func (g *grpcClient) doBatchCheck(_ testing.TB, requestTuples []*ketoapi.RelationTuple) (*rts.BatchCheckResponse, error) {

c := rts.NewCheckServiceClient(g.read)

Expand All @@ -199,7 +199,7 @@ func (g *grpcClient) doBatchCheck(_ *testing.T, requestTuples []*ketoapi.Relatio
return c.BatchCheck(g.ctx, req)
}

func (g *grpcClient) expand(t *testing.T, r *ketoapi.SubjectSet, depth int) *ketoapi.Tree[*ketoapi.RelationTuple] {
func (g *grpcClient) expand(t testing.TB, r *ketoapi.SubjectSet, depth int) *ketoapi.Tree[*ketoapi.RelationTuple] {
c := rts.NewExpandServiceClient(g.read)

resp, err := c.Expand(g.ctx, &rts.ExpandRequest{
Expand All @@ -211,7 +211,7 @@ func (g *grpcClient) expand(t *testing.T, r *ketoapi.SubjectSet, depth int) *ket
return ketoapi.TreeFromProto[*ketoapi.RelationTuple](resp.Tree)
}

func (g *grpcClient) waitUntilLive(t *testing.T) {
func (g *grpcClient) waitUntilLive(t testing.TB) {
require.EventuallyWithT(t, func(t *assert.CollectT) {
c := grpcHealthV1.NewHealthClient(g.read)

Expand All @@ -221,11 +221,11 @@ func (g *grpcClient) waitUntilLive(t *testing.T) {
}, 2*time.Second, 10*time.Millisecond)
}

func (g *grpcClient) deleteTuple(t *testing.T, r *ketoapi.RelationTuple) {
func (g *grpcClient) deleteTuple(t testing.TB, r *ketoapi.RelationTuple) {
g.transactTuples(t, nil, []*ketoapi.RelationTuple{r})
}

func (g *grpcClient) deleteAllTuples(t *testing.T, q *ketoapi.RelationQuery) {
func (g *grpcClient) deleteAllTuples(t testing.TB, q *ketoapi.RelationQuery) {
c := rts.NewWriteServiceClient(g.write)
query := &rts.RelationQuery{
Namespace: q.Namespace,
Expand All @@ -244,7 +244,7 @@ func (g *grpcClient) deleteAllTuples(t *testing.T, q *ketoapi.RelationQuery) {
require.NoError(t, err)
}

func (g *grpcClient) transactTuples(t *testing.T, ins []*ketoapi.RelationTuple, del []*ketoapi.RelationTuple) {
func (g *grpcClient) transactTuples(t testing.TB, ins []*ketoapi.RelationTuple, del []*ketoapi.RelationTuple) {
c := rts.NewWriteServiceClient(g.write)

deltas := make([]*rts.RelationTupleDelta, len(ins)+len(del))
Expand All @@ -268,7 +268,7 @@ func (g *grpcClient) transactTuples(t *testing.T, ins []*ketoapi.RelationTuple,
require.NoError(t, err)
}

func (g *grpcClient) oplCheckSyntax(t *testing.T, content []byte) (parseErrors []*ketoapi.ParseError) {
func (g *grpcClient) oplCheckSyntax(t testing.TB, content []byte) (parseErrors []*ketoapi.ParseError) {
c := opl.NewSyntaxServiceClient(g.oplSyntax)

res, err := c.Check(g.ctx, &opl.CheckRequest{Content: content})
Expand Down
Loading

0 comments on commit 05f5180

Please sign in to comment.