diff --git a/api/proto b/api/proto index 02fad3a55..bcf939a5d 160000 --- a/api/proto +++ b/api/proto @@ -1 +1 @@ -Subproject commit 02fad3a551487d3c6221f5c5460d7fff17aa0e19 +Subproject commit bcf939a5d28dd9bb0044bb3cdb2579c955f8c86b diff --git a/api/server/v1/marshaler.go b/api/server/v1/marshaler.go index 29c4d8168..f810c3256 100644 --- a/api/server/v1/marshaler.go +++ b/api/server/v1/marshaler.go @@ -118,9 +118,14 @@ func (x *SearchRequest) UnmarshalJSON(data []byte) error { case "sort": // delaying the sort deserialization x.Sort = value - case "fields": - // not decoding it here and let it decode during fields parsing - x.Fields = value + case "include_fields": + if err := jsoniter.Unmarshal(value, &x.IncludeFields); err != nil { + return err + } + case "exclude_fields": + if err := jsoniter.Unmarshal(value, &x.ExcludeFields); err != nil { + return err + } case "page_size": if err := jsoniter.Unmarshal(value, &x.PageSize); err != nil { return err @@ -306,6 +311,7 @@ type collDesc struct { Collection string `json:"collection"` Metadata *CollectionMetadata `json:"metadata"` Schema json.RawMessage `json:"schema"` + Size int64 `json:"size"` } func (x *DescribeCollectionResponse) MarshalJSON() ([]byte, error) { @@ -313,6 +319,7 @@ func (x *DescribeCollectionResponse) MarshalJSON() ([]byte, error) { Collection: x.Collection, Metadata: x.Metadata, Schema: x.Schema, + Size: x.Size, }) } @@ -321,9 +328,11 @@ func (x *DescribeDatabaseResponse) MarshalJSON() ([]byte, error) { Db string `json:"db"` Metadata *DatabaseMetadata `json:"metadata"` Collections []*collDesc `json:"collections"` + Size int64 `json:"size"` }{ Db: x.Db, Metadata: x.Metadata, + Size: x.Size, } for _, v := range x.Collections { @@ -331,6 +340,7 @@ func (x *DescribeDatabaseResponse) MarshalJSON() ([]byte, error) { Collection: v.Collection, Metadata: v.Metadata, Schema: v.Schema, + Size: v.Size, }) } diff --git a/api/server/v1/marshaler_test.go b/api/server/v1/marshaler_test.go index c4bcb7958..62633576e 100644 --- a/api/server/v1/marshaler_test.go +++ b/api/server/v1/marshaler_test.go @@ -33,7 +33,7 @@ func TestJSONEncoding(t *testing.T) { t.Run("unmarshal SearchRequest", func(t *testing.T) { inputDoc := []byte(`{"q":"my search text","search_fields":["first_name","last_name"], "filter":{"last_name":"Steve"},"facet":{"facet stat":0}, - "sort":[{"salary":"$asc"}],"fields":["employment","history"]}`) + "sort":[{"salary":"$asc"}],"include_fields":["employment","history"]}`) req := &SearchRequest{} err := json.Unmarshal(inputDoc, req) @@ -43,7 +43,7 @@ func TestJSONEncoding(t *testing.T) { require.Equal(t, []byte(`{"last_name":"Steve"}`), req.GetFilter()) require.Equal(t, []byte(`{"facet stat":0}`), req.GetFacet()) require.Equal(t, []byte(`[{"salary":"$asc"}]`), req.GetSort()) - require.Equal(t, []byte(`["employment","history"]`), req.GetFields()) + require.Equal(t, []string{"employment", "history"}, req.GetIncludeFields()) }) t.Run("marshal SearchResponse", func(t *testing.T) { diff --git a/api/server/v1/validator.go b/api/server/v1/validator.go index d516ce76d..37731e120 100644 --- a/api/server/v1/validator.go +++ b/api/server/v1/validator.go @@ -103,6 +103,10 @@ func (x *SearchRequest) Validate() error { return err } + if len(x.IncludeFields) > 0 && len(x.ExcludeFields) > 0 { + return Errorf(Code_INVALID_ARGUMENT, "Cannot use both `include_fields` and `exclude_fields` together") + } + if err := isValidPaginationParam("page", int(x.Page)); err != nil { return err } diff --git a/docker/Dockerfile.local b/docker/Dockerfile.local index 24645ebab..581dfee95 100644 --- a/docker/Dockerfile.local +++ b/docker/Dockerfile.local @@ -72,6 +72,7 @@ RUN echo "#!/bin/bash\n \ fdbserver --listen-address 127.0.0.1:4500 --public-address 127.0.0.1:4500 --datadir /var/lib/foundationdb/data --logdir /var/lib/foundationdb/logs --locality-zoneid tigris --locality-machineid tigris & \n\ export TIGRIS_SERVER_SEARCH_AUTH_KEY=ts_dev_key \n \ export TIGRIS_SERVER_SEARCH_HOST=localhost \n \ + export TIGRIS_SERVER_CDC_ENABLED=true \n \ fdbcli --exec 'configure new single memory' \n \ /server/service\n" >/server/service.sh diff --git a/go.mod b/go.mod index cb8212a93..22f8be270 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,8 @@ require ( github.com/auth0/go-jwt-middleware/v2 v2.0.1 github.com/buger/jsonparser v1.1.1 github.com/davecgh/go-spew v1.1.1 - github.com/deepmap/oapi-codegen v1.11.0 github.com/fsnotify/fsnotify v1.5.4 github.com/fullstorydev/grpchan v1.1.1 - github.com/getkin/kin-openapi v0.97.0 github.com/go-chi/chi/v5 v5.0.7 github.com/go-chi/cors v1.2.1 github.com/golang/protobuf v1.5.2 @@ -19,7 +17,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.2.0.20211207221722-a5b9e0b0458c github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20211207221722-a5b9e0b0458c - github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.3 + github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.0 github.com/hashicorp/go-multierror v1.1.1 github.com/json-iterator/go v1.1.12 github.com/m3db/prometheus_client_golang v1.12.8 @@ -30,41 +28,42 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.12.0 github.com/stretchr/testify v1.7.2 - github.com/tigrisdata/tigris-client-go v1.0.0-alpha.18 + github.com/tigrisdata/tigris-client-go v1.0.0-alpha.20 github.com/typesense/typesense-go v0.5.0 github.com/uber-go/tally v3.5.0+incompatible github.com/ugorji/go/codec v1.2.7 github.com/valyala/bytebufferpool v1.0.0 - google.golang.org/genproto v0.0.0-20220615141314-f1464d18c36b - google.golang.org/grpc v1.47.0 + google.golang.org/genproto v0.0.0-20220725144611-272f38e5d71b + google.golang.org/grpc v1.48.0 google.golang.org/protobuf v1.28.0 - gopkg.in/DataDog/dd-trace-go.v1 v1.38.1 + gopkg.in/DataDog/dd-trace-go.v1 v1.40.1 gopkg.in/gavv/httpexpect.v1 v1.1.3 gopkg.in/yaml.v2 v2.4.0 ) require ( - cloud.google.com/go/compute v1.6.1 // indirect - github.com/DataDog/datadog-agent/pkg/obfuscate v0.0.0-20211129110424-6491aa3bf583 // indirect - github.com/DataDog/datadog-go v4.8.2+incompatible // indirect - github.com/DataDog/datadog-go/v5 v5.0.2 // indirect + cloud.google.com/go/compute v1.7.0 // indirect + github.com/DataDog/datadog-agent/pkg/obfuscate v0.38.0 // indirect + github.com/DataDog/datadog-go/v5 v5.1.1 // indirect github.com/DataDog/gostackparse v0.5.0 // indirect - github.com/DataDog/sketches-go v1.0.0 // indirect - github.com/Microsoft/go-winio v0.5.1 // indirect + github.com/DataDog/sketches-go v1.4.1 // indirect + github.com/Microsoft/go-winio v0.5.2 // indirect github.com/ajg/form v1.5.1 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/deepmap/oapi-codegen v1.11.0 // indirect github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 // indirect github.com/gertd/go-pluralize v0.2.1 // indirect + github.com/getkin/kin-openapi v0.97.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/swag v0.21.1 // indirect github.com/golang/glog v1.0.0 // indirect github.com/google/go-querystring v1.0.0 // indirect - github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect + github.com/google/pprof v0.0.0-20220608213341-c488b8fa1db3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/iancoleman/strcase v0.2.0 // indirect @@ -90,15 +89,15 @@ require ( github.com/pelletier/go-toml/v2 v2.0.2 // indirect github.com/philhofer/fwd v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/procfs v0.8.0 // indirect github.com/sergi/go-diff v1.0.0 // indirect github.com/smartystreets/goconvey v1.7.2 // indirect github.com/sony/gobreaker v0.5.0 // indirect - github.com/spf13/afero v1.8.2 // indirect + github.com/spf13/afero v1.9.2 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.0 // indirect - github.com/tinylib/msgp v1.1.2 // indirect + github.com/tinylib/msgp v1.1.6 // indirect github.com/twmb/murmur3 v1.1.6 // indirect github.com/valyala/fasthttp v1.34.0 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect @@ -109,12 +108,12 @@ require ( github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect github.com/yudai/pp v2.0.1+incompatible // indirect go.uber.org/atomic v1.9.0 // indirect - golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect - golang.org/x/net v0.0.0-20220615171555-694bf12d69de // indirect - golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb // indirect - golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect + golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect + golang.org/x/net v0.0.0-20220726230323-06994584191e // indirect + golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c // indirect + golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect + golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/grpc/examples v0.0.0-20220215234149-ec717cad7395 // indirect diff --git a/go.sum b/go.sum index ca257939c..9e0920f2d 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= @@ -29,6 +30,7 @@ cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -39,10 +41,12 @@ cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTB cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= -cloud.google.com/go/compute v1.6.1 h1:2sMmt8prCn7DPaG4Pmh0N3Inmc8cT8ae5k1M6VJ9Wqc= cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0 h1:v/k9Eueb8aAJ0vZuxKMrgm6kPhCLZU9HxFU+AFDs9Uk= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -54,6 +58,7 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= @@ -64,26 +69,31 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/datadog-agent/pkg/obfuscate v0.0.0-20211129110424-6491aa3bf583 h1:3nVO1nQyh64IUY6BPZUpMYMZ738Pu+LsMt3E0eqqIYw= github.com/DataDog/datadog-agent/pkg/obfuscate v0.0.0-20211129110424-6491aa3bf583/go.mod h1:EP9f4GqaDJyP1F5jTNMtzdIpw3JpNs3rMSJOnYywCiw= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.38.0 h1:Mk1GqUitDrsZBvNIR4G/GF1HUw3gegs8h38/rtv0Els= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.38.0/go.mod h1:MxVcCIC42tBIjPm93BHdh9/vw2LivRiptj3HygI+GGQ= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/DataDog/datadog-go v4.8.2+incompatible h1:qbcKSx29aBLD+5QLvlQZlGmRMF/FfGqFLFev/1TDzRo= github.com/DataDog/datadog-go v4.8.2+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/DataDog/datadog-go/v5 v5.0.2 h1:UFtEe7662/Qojxkw1d6SboAeA0CPI3naKhVASwFn+04= github.com/DataDog/datadog-go/v5 v5.0.2/go.mod h1:ZI9JFB4ewXbw1sBnF4sxsR2k1H3xjV+PUAOUsHvKpcU= +github.com/DataDog/datadog-go/v5 v5.1.0/go.mod h1:KhiYb2Badlv9/rofz+OznKoEF5XKTonWyhx5K83AP8E= +github.com/DataDog/datadog-go/v5 v5.1.1 h1:JLZ6s2K1pG2h9GkvEvMdEGqMDyVLEAccdX5TltWcLMU= +github.com/DataDog/datadog-go/v5 v5.1.1/go.mod h1:KhiYb2Badlv9/rofz+OznKoEF5XKTonWyhx5K83AP8E= github.com/DataDog/gostackparse v0.5.0 h1:jb72P6GFHPHz2W0onsN51cS3FkaMDcjb0QzgxxA4gDk= github.com/DataDog/gostackparse v0.5.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM= -github.com/DataDog/sketches-go v1.0.0 h1:chm5KSXO7kO+ywGWJ0Zs6tdmWU8PBXSbywFVciL6BG4= -github.com/DataDog/sketches-go v1.0.0/go.mod h1:O+XkJHWk9w4hDwY2ZUDU31ZC9sNYlYo8DiFsxjYeo1k= +github.com/DataDog/sketches-go v1.2.1/go.mod h1:1xYmPLY1So10AwxV6MJV0J53XVH+WL9Ad1KetxVivVI= +github.com/DataDog/sketches-go v1.4.1 h1:j5G6as+9FASM2qC36lvpvQAj9qsv/jUs3FtO8CwZNAY= +github.com/DataDog/sketches-go v1.4.1/go.mod h1:xJIXldczJyyjnbDop7ZZcLxJdV3+7Kra7H1KMgpgkLk= github.com/DataDog/zstd v1.3.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/sarama v1.22.0/go.mod h1:lm3THZ8reqBDBQKQyb5HB3sY1lKp3grEbQ81aWSgPp4= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= @@ -97,6 +107,7 @@ github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apple/foundationdb/bindings/go v0.0.0-20220521054011-a88e049b28d8 h1:B1KM1sz2bMjLThSQZSg+2kE2OBFMbtGdDcekqj0t2z0= github.com/apple/foundationdb/bindings/go v0.0.0-20220521054011-a88e049b28d8/go.mod h1:w63jdZTFCtvdjsUj5yrdKgjxaAD5uXQX6hJ7EaiLFRs= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -163,6 +174,7 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go. github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/deepmap/oapi-codegen v1.11.0 h1:f/X2NdIkaBKsSdpeuwLnY/vDI0AtPUrmB5LMgc7YD+A= github.com/deepmap/oapi-codegen v1.11.0/go.mod h1:k+ujhoQGxmQYBZBbxhOZNZf4j08qv5mC+OH+fFTnKxM= +github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= @@ -175,6 +187,7 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -219,7 +232,6 @@ github.com/getkin/kin-openapi v0.97.0/go.mod h1:w4lRPHiyOdwGbOkLIyk+P0qCwlu7TXPC github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.7.0/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= @@ -264,36 +276,13 @@ github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8w github.com/go-redis/redis/v7 v7.1.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg= github.com/go-redis/redis/v8 v8.0.0/go.mod h1:isLoQT/NFSP7V67lyvM9GmdvLdyZ7pEhsXvvyQtnQTo= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= -github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= -github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= -github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= -github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= -github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= -github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= -github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gocql/gocql v0.0.0-20220224095938-0eacd3183625/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -301,9 +290,9 @@ github.com/gofiber/fiber/v2 v2.11.0/go.mod h1:oZTLWqYnqpMMuF922SjGbsYZsdpE1MCfh4 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= @@ -368,6 +357,7 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= @@ -394,26 +384,30 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210423192551-a2663126120b/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20220608213341-c488b8fa1db3 h1:mpL/HvfIgIejhVwAfxBQkwEjlhP5o0O9RAeTAjpwzxc= +github.com/google/pprof v0.0.0-20220608213341-c488b8fa1db3/go.mod h1:gSuNB+gJaOiQKLEZ+q+PK9Mq3SOzhRcw2GsGS/FhYDk= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.5.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -425,8 +419,8 @@ github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20201002093600-73c github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20211207221722-a5b9e0b0458c h1:7EiDBX4Vu3/AqgvkV+8IcVLCk20eV52Rtc/rlANuYUg= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20211207221722-a5b9e0b0458c/go.mod h1:hTxjzRcX49ogbTGVJ1sM5mz5s+SSgiGIyL3jjPxl32E= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.3 h1:BGNSrTRW4rwfhJiFwvwF4XQ0Y72Jj9YEgxVrtovbD5o= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.3/go.mod h1:VHn7KgNsRriXa4mcgtkpR00OXyQY6g67JWMvn+R27A4= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.0 h1:Ghn7copILfeIg0y8sTGRppI1bd8I4l2VN3cob0Xeqwg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.0/go.mod h1:dnjr4snxnhRSn5GWqJUva2AoMbeaxyAcepvc0Tg8lXk= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/consul/api v1.0.0/go.mod h1:mbFwfRxOTDHZpT3iUsMAFcLNoVm6Xbe1xZ6KiSm8FY0= github.com/hashicorp/consul/internal v0.1.0/go.mod h1:zi9bMZYbiPHyAjgBWo7kCUcy5l2NrTdrkVupCc7Oo6c= @@ -483,7 +477,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= @@ -549,15 +542,15 @@ github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuT github.com/jhump/protoreflect v1.12.0 h1:1NQ4FpWMgn3by/n1X0fbeKEUxP1wBt7+Oitpv01HR10= github.com/jhump/protoreflect v1.12.0/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= github.com/jinzhu/copier v0.3.4 h1:mfU6jI9PtCeUjkjQ322dlff9ELjGDu975C2p/nrubVI= -github.com/jinzhu/gorm v1.9.1/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= +github.com/jinzhu/gorm v1.9.10/go.mod h1:Kh6hTsSGffh4ui079FHrR5Gg+5D0hgihqDcsDN2BBJY= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.3/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -573,18 +566,14 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.1.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= @@ -621,6 +610,7 @@ github.com/lestrrat-go/jwx v1.2.24/go.mod h1:zoNuZymNl5lgdcu6P7K6ie2QRll5HVfF4xw github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= @@ -649,8 +639,6 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/matryer/moq v0.2.7/go.mod h1:kITsx543GOENm48TUAQyJ9+SAvFSr7iGQXPoth/VUBk= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= @@ -672,6 +660,7 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -735,9 +724,9 @@ github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5h github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.2 h1:+jQXlF3scKIcSEKkdHzXhCTDLPFi5r1wnK6yPS+49Gw= @@ -762,15 +751,18 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= @@ -778,15 +770,15 @@ github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+ github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= @@ -812,7 +804,6 @@ github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9Nz github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -828,15 +819,13 @@ github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= -github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= +github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= +github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= @@ -874,10 +863,11 @@ github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= -github.com/tigrisdata/tigris-client-go v1.0.0-alpha.18 h1:opvqzYQARvdWLoiZOW02+TcTF0ZujkTwJYpUNYFaqC8= -github.com/tigrisdata/tigris-client-go v1.0.0-alpha.18/go.mod h1:/L4tilHC26feio8KDuitajcGXabCLAElYtUQRkewkyM= -github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ= +github.com/tigrisdata/tigris-client-go v1.0.0-alpha.20 h1:J4BazYr7EZjhlfZZEHRTQ7OYbfRi4ypwySLxxkXmZTc= +github.com/tigrisdata/tigris-client-go v1.0.0-alpha.20/go.mod h1:/L4tilHC26feio8KDuitajcGXabCLAElYtUQRkewkyM= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw= +github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twitchtv/twirp v8.1.1+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= @@ -896,7 +886,6 @@ github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKn github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.26.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfYYdPJBl8BA= -github.com/valyala/fasthttp v1.32.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus= github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= @@ -937,7 +926,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/zenazn/goji v1.0.1/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= +go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -960,6 +950,10 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go4.org/intern v0.0.0-20211027215823-ae77deb06f29 h1:UXLjNohABv4S58tHmeuIZDO6e3mHpW2Dx33gaNt03LE= +go4.org/intern v0.0.0-20211027215823-ae77deb06f29/go.mod h1:cS2ma+47FKrLPdXFpr7CuxiTW3eyJbWew4qx0qtQWDA= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 h1:Tx9kY6yUkLge/pFG7IEMwDZy6CS2ajFc9TvQdPCW0uA= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -969,7 +963,6 @@ golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -993,8 +986,8 @@ golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1042,6 +1035,7 @@ golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1089,15 +1083,16 @@ golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220513224357-95641704303c/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220615171555-694bf12d69de h1:ogOG2+P6LjO2j55AkRScrkB2BFpd+Z8TY2wcM0Z3MGo= -golang.org/x/net v0.0.0-20220615171555-694bf12d69de/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220726230323-06994584191e h1:wOQNKh1uuDGRnmgF0jDxh7ctgGy/3P4rYWQRVJD4/Yg= +golang.org/x/net v0.0.0-20220726230323-06994584191e/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1118,13 +1113,13 @@ golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb h1:8tDJ3aechhddbdPAxpycgXHJRMLpk/Ab+aa4OgdN5/g= golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c h1:q3gFqPqH7NVofKo3c3yETAP//pPI+G5mvB7qqj1Y5kY= +golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1132,6 +1127,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1139,6 +1135,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1146,11 +1143,9 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1224,9 +1219,13 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c h1:aFV+BgZ4svzjfabn8ERpuB4JI4N6/rdy1iusx77G3oU= -golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1247,9 +1246,11 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w= golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 h1:ftMN5LMiBFjbzleLqtoBZk7KdJwhuybIU+FckUHgoyQ= +golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1259,9 +1260,6 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -1307,6 +1305,7 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1328,8 +1327,10 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1367,6 +1368,9 @@ google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/S google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1377,6 +1381,7 @@ google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1420,6 +1425,7 @@ google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= @@ -1453,9 +1459,16 @@ google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220615141314-f1464d18c36b h1:2LXbOcxY7BehyA9yu5hxYzaY67bLaJQhBX9O1zxxVis= -google.golang.org/genproto v0.0.0-20220615141314-f1464d18c36b/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220725144611-272f38e5d71b h1:SfSkJugek6xm7lWywqth4r2iTrYLpD8lOj1nMIIhMNM= +google.golang.org/genproto v0.0.0-20220725144611-272f38e5d71b/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1486,8 +1499,11 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w= +google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/examples v0.0.0-20210424002626-9572fd6faeae/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE= google.golang.org/grpc/examples v0.0.0-20220215234149-ec717cad7395 h1:2JLuOFd8BzB9flLGg1uCjb2PFaBYQWA2HYC3ceAKmuw= @@ -1507,8 +1523,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/DataDog/dd-trace-go.v1 v1.38.1 h1:nAKgcpJLXRHF56cKCP3bN8gTTQmmNAZFEblbyGKhKTo= -gopkg.in/DataDog/dd-trace-go.v1 v1.38.1/go.mod h1:GBhK4yaMJ1h329ivtKAqRNe1EZ944UnZwtz5lh7CnJc= +gopkg.in/DataDog/dd-trace-go.v1 v1.40.1 h1:ou2cMah30qvQEMTYPF0CVOhDd2ji2+WQk9/meYFuZ84= +gopkg.in/DataDog/dd-trace-go.v1 v1.40.1/go.mod h1:tlSNIf2aKOah7PmoEP4qQETNVKgonk5BWwNnblw8C8w= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1553,6 +1569,7 @@ gorm.io/driver/sqlserver v1.0.4/go.mod h1:ciEo5btfITTBCj9BkoUVDvgQbUdLWQNqdFY5OG gorm.io/gorm v1.9.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v1.20.0/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v1.20.6/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1560,6 +1577,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 h1:acCzuUSQ79tGsM/O50VRFySfMm19IoMKL+sZztZkCxw= +inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6/go.mod h1:y3MGhcFMlh0KZPMuXXow8mpjxxAk3yoDNsp4cQz54i8= k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k= diff --git a/internal/data.go b/internal/data.go index 5213e6ed3..4a66aaf5c 100644 --- a/internal/data.go +++ b/internal/data.go @@ -24,6 +24,10 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) +var ( + UserTableKeyPrefix = []byte("data") +) + var ( bh codec.BincHandle ) @@ -135,6 +139,9 @@ func Encode(data *TableData) ([]byte, error) { // Decode is used to decode the raw bytes to TableData. The raw bytes are returned from the storage and the kvStore is // calling Decode to convert these raw bytes back to TableData. func Decode(b []byte) (*TableData, error) { + if len(b) == 0 { + return nil, api.Errorf(api.Code_INTERNAL, "unable to decode table data is empty") + } dataType := DataType(b[0]) return decodeInternal(dataType, b[1:]) } diff --git a/query/read/fields.go b/query/read/fields.go index b4e0ce1f4..38cf38bb6 100644 --- a/query/read/fields.go +++ b/query/read/fields.go @@ -52,7 +52,7 @@ func BuildFields(reqFields jsoniter.RawMessage) (*FieldFactory, error) { return err } - factory.addField(&SimpleField{ + factory.AddField(&SimpleField{ Name: string(key), Incl: include, }) @@ -64,7 +64,7 @@ func BuildFields(reqFields jsoniter.RawMessage) (*FieldFactory, error) { return err } - factory.addField(&SimpleField{ + factory.AddField(&SimpleField{ Name: string(key), Incl: include == 1, }) @@ -74,7 +74,7 @@ func BuildFields(reqFields jsoniter.RawMessage) (*FieldFactory, error) { if err != nil { return err } - factory.addField(NewExprField(string(key), expr)) + factory.AddField(NewExprField(string(key), expr)) default: return api.Errorf(api.Code_INVALID_ARGUMENT, "only boolean/integer is supported as value") } @@ -93,7 +93,7 @@ type FieldFactory struct { FetchedValues map[string]*JSONObject } -func (factory *FieldFactory) addField(f Field) { +func (factory *FieldFactory) AddField(f Field) { if !f.Include() { factory.Exclude[f.Alias()] = f return diff --git a/query/search/search.go b/query/search/search.go index deea0b338..b63f7033c 100644 --- a/query/search/search.go +++ b/query/search/search.go @@ -16,6 +16,7 @@ package search import ( "github.com/tigrisdata/tigris/query/filter" + "github.com/tigrisdata/tigris/query/read" ) const ( @@ -23,11 +24,12 @@ const ( ) type Query struct { - Q string - Fields []string - Facets Facets - PageSize int - WrappedF *filter.WrappedFilter + Q string + SearchFields []string + Facets Facets + PageSize int + WrappedF *filter.WrappedFilter + ReadFields *read.FieldFactory } func (q *Query) ToSearchFacetSize() int { @@ -63,7 +65,7 @@ func (q *Query) ToSearchFacets() string { func (q *Query) ToSearchFields() string { var fields string - for i, f := range q.Fields { + for i, f := range q.SearchFields { if i != 0 { fields += "," } @@ -104,7 +106,12 @@ func (b *Builder) Facets(facets Facets) *Builder { } func (b *Builder) SearchFields(f []string) *Builder { - b.query.Fields = f + b.query.SearchFields = f + return b +} + +func (b *Builder) ReadFields(f *read.FieldFactory) *Builder { + b.query.ReadFields = f return b } diff --git a/server/config/options.go b/server/config/options.go index ed17931cb..7eedd9540 100644 --- a/server/config/options.go +++ b/server/config/options.go @@ -36,6 +36,7 @@ type Config struct { Profiling ProfilingConfig `yaml:"profiling" json:"profiling"` Metrics MetricsConfig FoundationDB FoundationDBConfig + Quota QuotaConfig } type AuthConfig struct { @@ -159,6 +160,12 @@ var DefaultConfig = Config{ ResponseTime: true, }, }, + Quota: QuotaConfig{ + Enabled: false, + RateLimit: 1000, // requests per second + WriteThroughputLimit: 10000000, // bytes per second + DataSizeLimit: 10000000000, // bytes + }, } // FoundationDBConfig keeps FoundationDB configuration parameters @@ -173,3 +180,10 @@ type SearchConfig struct { ReadEnabled bool `mapstructure:"read_enabled" yaml:"read_enabled" json:"read_enabled"` WriteEnabled bool `mapstructure:"write_enabled" yaml:"write_enabled" json:"write_enabled"` } + +type QuotaConfig struct { + Enabled bool + RateLimit int `mapstructure:"rate_limit" yaml:"rate_limit" json:"rate_limit"` + WriteThroughputLimit int `mapstructure:"write_throughput_limit" yaml:"write_throughput_limit" json:"write_throughput_limit"` + DataSizeLimit int64 `mapstructure:"data_size_limit" yaml:"data_size_limit" json:"data_size_limit"` +} diff --git a/server/main.go b/server/main.go index f083566e2..28e454912 100644 --- a/server/main.go +++ b/server/main.go @@ -22,6 +22,7 @@ import ( "github.com/tigrisdata/tigris/server/metadata" "github.com/tigrisdata/tigris/server/metrics" "github.com/tigrisdata/tigris/server/muxer" + "github.com/tigrisdata/tigris/server/quota" "github.com/tigrisdata/tigris/server/tracing" "github.com/tigrisdata/tigris/server/transaction" "github.com/tigrisdata/tigris/store/kv" @@ -62,17 +63,31 @@ func main() { log.Fatal().Err(err).Msg("error initializing kv store") } - searchStore, err := search.NewStore(&config.DefaultConfig.Search) + var searchStore search.Store + if config.DefaultConfig.Tracing.Enabled { + searchStore, err = search.NewStoreWithMetrics(&config.DefaultConfig.Search) + } else { + searchStore, err = search.NewStore(&config.DefaultConfig.Search) + } if err != nil { log.Fatal().Err(err).Msg("error initializing search store") } - tenantMgr := metadata.NewTenantManager(kvStore) + tenantMgr := metadata.NewTenantManager(kvStore, searchStore) log.Info().Msg("initialized tenant manager") + txMgr := transaction.NewManager(kvStore) log.Info().Msg("initialized transaction manager") + + if err = tenantMgr.EnsureDefaultNamespace(txMgr); err != nil { + log.Fatal().Err(err).Msg("error initializing default namespace") + } + + quota.Init(tenantMgr, txMgr, &config.DefaultConfig.Quota) + mx := muxer.NewMuxer(&config.DefaultConfig, tenantMgr, txMgr) mx.RegisterServices(kvStore, searchStore, tenantMgr, txMgr) + if err := mx.Start(config.DefaultConfig.Server.Host, config.DefaultConfig.Server.Port); err != nil { log.Fatal().Err(err).Msgf("error starting server") } diff --git a/server/metadata/encoding/dictionary.go b/server/metadata/encoding/dictionary.go index 4abb8f1fc..994e1b20a 100644 --- a/server/metadata/encoding/dictionary.go +++ b/server/metadata/encoding/dictionary.go @@ -197,16 +197,16 @@ func (r *reservedSubspace) allocateToken(ctx context.Context, tx transaction.Tx, return newReservedValue, nil } -// DictionaryEncoder is used to replace variable length strings to their corresponding codes to encode it. Compression +// MetadataDictionary is used to replace variable length strings to their corresponding codes to allocateAndSave it. Compression // is achieved by replacing long strings with a simple 4byte representation. -type DictionaryEncoder struct { +type MetadataDictionary struct { MDNameRegistry reservedSb *reservedSubspace } -func NewDictionaryEncoder(mdNameRegistry MDNameRegistry) *DictionaryEncoder { - return &DictionaryEncoder{ +func NewMetadataDictionary(mdNameRegistry MDNameRegistry) *MetadataDictionary { + return &MetadataDictionary{ MDNameRegistry: mdNameRegistry, reservedSb: newReservedSubspace(mdNameRegistry), } @@ -214,11 +214,11 @@ func NewDictionaryEncoder(mdNameRegistry MDNameRegistry) *DictionaryEncoder { // ReserveNamespace is the first step in the encoding and the mapping is passed the caller. As this is the first encoded // integer the caller needs to make sure a unique value is assigned to this namespace. -func (k *DictionaryEncoder) ReserveNamespace(ctx context.Context, tx transaction.Tx, namespace string, id uint32) error { +func (k *MetadataDictionary) ReserveNamespace(ctx context.Context, tx transaction.Tx, namespace string, id uint32) error { return k.reservedSb.reserveNamespace(ctx, tx, namespace, id) } -func (k *DictionaryEncoder) GetNamespaces(ctx context.Context, tx transaction.Tx) (map[string]uint32, error) { +func (k *MetadataDictionary) GetNamespaces(ctx context.Context, tx transaction.Tx) (map[string]uint32, error) { if err := k.reservedSb.reload(ctx, tx); err != nil { return nil, err } @@ -226,7 +226,7 @@ func (k *DictionaryEncoder) GetNamespaces(ctx context.Context, tx transaction.Tx return k.reservedSb.getNamespaces(), nil } -func (k *DictionaryEncoder) EncodeDatabaseName(ctx context.Context, tx transaction.Tx, dbName string, namespaceId uint32) (uint32, error) { +func (k *MetadataDictionary) CreateDatabase(ctx context.Context, tx transaction.Tx, dbName string, namespaceId uint32) (uint32, error) { if err := k.validNamespaceId(namespaceId); err != nil { return InvalidId, err } @@ -235,12 +235,12 @@ func (k *DictionaryEncoder) EncodeDatabaseName(ctx context.Context, tx transacti } key := keys.NewKey(k.EncodingSubspaceName(), encVersion, UInt32ToByte(namespaceId), dbKey, dbName, keyEnd) - return k.encode(ctx, tx, key, dbKey) + return k.allocateAndSave(ctx, tx, key, dbKey) } -// EncodeDatabaseAsDropped will remove the "created" entry from the encoding subspace and will add a "dropped" entry with the same +// DropDatabase will remove the "created" entry from the encoding subspace and will add a "dropped" entry with the same // value. -func (k *DictionaryEncoder) EncodeDatabaseAsDropped(ctx context.Context, tx transaction.Tx, dbName string, namespaceId uint32, existingId uint32) error { +func (k *MetadataDictionary) DropDatabase(ctx context.Context, tx transaction.Tx, dbName string, namespaceId uint32, existingId uint32) error { if err := k.validNamespaceId(namespaceId); err != nil { return err } @@ -251,10 +251,10 @@ func (k *DictionaryEncoder) EncodeDatabaseAsDropped(ctx context.Context, tx tran // remove existing entry toDeleteKey := keys.NewKey(k.EncodingSubspaceName(), encVersion, UInt32ToByte(namespaceId), dbKey, dbName, keyEnd) newKey := keys.NewKey(k.EncodingSubspaceName(), encVersion, UInt32ToByte(namespaceId), dbKey, dbName, keyDroppedEnd) - return k.encodeAsDropped(ctx, tx, toDeleteKey, newKey, existingId, dbKey) + return k.delete(ctx, tx, toDeleteKey, newKey, existingId, dbKey) } -func (k *DictionaryEncoder) EncodeCollectionName(ctx context.Context, tx transaction.Tx, collection string, namespaceId uint32, dbId uint32) (uint32, error) { +func (k *MetadataDictionary) CreateCollection(ctx context.Context, tx transaction.Tx, collection string, namespaceId uint32, dbId uint32) (uint32, error) { if err := k.validNamespaceId(namespaceId); err != nil { return InvalidId, err } @@ -266,10 +266,10 @@ func (k *DictionaryEncoder) EncodeCollectionName(ctx context.Context, tx transac } key := keys.NewKey(k.EncodingSubspaceName(), encVersion, UInt32ToByte(namespaceId), UInt32ToByte(dbId), collectionKey, collection, keyEnd) - return k.encode(ctx, tx, key, collectionKey) + return k.allocateAndSave(ctx, tx, key, collectionKey) } -func (k *DictionaryEncoder) EncodeCollectionAsDropped(ctx context.Context, tx transaction.Tx, collection string, namespaceId uint32, dbId uint32, existingId uint32) error { +func (k *MetadataDictionary) DropCollection(ctx context.Context, tx transaction.Tx, collection string, namespaceId uint32, dbId uint32, existingId uint32) error { if err := k.validNamespaceId(namespaceId); err != nil { return err } @@ -283,10 +283,10 @@ func (k *DictionaryEncoder) EncodeCollectionAsDropped(ctx context.Context, tx tr // remove existing entry toDeleteKey := keys.NewKey(k.EncodingSubspaceName(), encVersion, UInt32ToByte(namespaceId), UInt32ToByte(dbId), collectionKey, collection, keyEnd) newKey := keys.NewKey(k.EncodingSubspaceName(), encVersion, UInt32ToByte(namespaceId), UInt32ToByte(dbId), collectionKey, collection, keyDroppedEnd) - return k.encodeAsDropped(ctx, tx, toDeleteKey, newKey, existingId, collectionKey) + return k.delete(ctx, tx, toDeleteKey, newKey, existingId, collectionKey) } -func (k *DictionaryEncoder) EncodeIndexName(ctx context.Context, tx transaction.Tx, indexName string, namespaceId uint32, dbId uint32, collId uint32) (uint32, error) { +func (k *MetadataDictionary) CreateIndex(ctx context.Context, tx transaction.Tx, indexName string, namespaceId uint32, dbId uint32, collId uint32) (uint32, error) { if err := k.validNamespaceId(namespaceId); err != nil { return InvalidId, err } @@ -301,10 +301,10 @@ func (k *DictionaryEncoder) EncodeIndexName(ctx context.Context, tx transaction. } key := keys.NewKey(k.EncodingSubspaceName(), encVersion, UInt32ToByte(namespaceId), UInt32ToByte(dbId), UInt32ToByte(collId), indexKey, indexName, keyEnd) - return k.encode(ctx, tx, key, indexKey) + return k.allocateAndSave(ctx, tx, key, indexKey) } -func (k *DictionaryEncoder) EncodeIndexAsDropped(ctx context.Context, tx transaction.Tx, indexName string, namespaceId uint32, dbId uint32, collId uint32, existingId uint32) error { +func (k *MetadataDictionary) DropIndex(ctx context.Context, tx transaction.Tx, indexName string, namespaceId uint32, dbId uint32, collId uint32, existingId uint32) error { if err := k.validNamespaceId(namespaceId); err != nil { return err } @@ -320,10 +320,10 @@ func (k *DictionaryEncoder) EncodeIndexAsDropped(ctx context.Context, tx transac toDeleteKey := keys.NewKey(k.EncodingSubspaceName(), encVersion, UInt32ToByte(namespaceId), UInt32ToByte(dbId), UInt32ToByte(collId), indexKey, indexName, keyEnd) newKey := keys.NewKey(k.EncodingSubspaceName(), encVersion, UInt32ToByte(namespaceId), UInt32ToByte(dbId), UInt32ToByte(collId), indexKey, indexName, keyDroppedEnd) - return k.encodeAsDropped(ctx, tx, toDeleteKey, newKey, existingId, indexKey) + return k.delete(ctx, tx, toDeleteKey, newKey, existingId, indexKey) } -func (k *DictionaryEncoder) encodeAsDropped(ctx context.Context, tx transaction.Tx, toDeleteKey keys.Key, newKey keys.Key, newValue uint32, encName string) error { +func (k *MetadataDictionary) delete(ctx context.Context, tx transaction.Tx, toDeleteKey keys.Key, newKey keys.Key, newValue uint32, encName string) error { if err := tx.Delete(ctx, toDeleteKey); err != nil { log.Debug().Str("key", toDeleteKey.String()).Err(err).Str("type", encName).Msg("existing entry deletion failed") return err @@ -340,7 +340,7 @@ func (k *DictionaryEncoder) encodeAsDropped(ctx context.Context, tx transaction. return nil } -func (k *DictionaryEncoder) encode(ctx context.Context, tx transaction.Tx, key keys.Key, encName string) (uint32, error) { +func (k *MetadataDictionary) allocateAndSave(ctx context.Context, tx transaction.Tx, key keys.Key, encName string) (uint32, error) { reserveToken, err := k.reservedSb.allocateToken(ctx, tx, string(k.EncodingSubspaceName())) if err != nil { return InvalidId, err @@ -356,28 +356,28 @@ func (k *DictionaryEncoder) encode(ctx context.Context, tx transaction.Tx, key k return reserveToken, nil } -func (k *DictionaryEncoder) validNamespaceId(id uint32) error { +func (k *MetadataDictionary) validNamespaceId(id uint32) error { if id == InvalidId { return api.Errorf(api.Code_INVALID_ARGUMENT, "invalid namespace id") } return nil } -func (k *DictionaryEncoder) validDatabaseId(id uint32) error { +func (k *MetadataDictionary) validDatabaseId(id uint32) error { if id == InvalidId { return api.Errorf(api.Code_INVALID_ARGUMENT, "invalid database id") } return nil } -func (k *DictionaryEncoder) validCollectionId(id uint32) error { +func (k *MetadataDictionary) validCollectionId(id uint32) error { if id == InvalidId { return api.Errorf(api.Code_INVALID_ARGUMENT, "invalid collection id") } return nil } -func (k *DictionaryEncoder) GetDatabases(ctx context.Context, tx transaction.Tx, namespaceId uint32) (map[string]uint32, error) { +func (k *MetadataDictionary) GetDatabases(ctx context.Context, tx transaction.Tx, namespaceId uint32) (map[string]uint32, error) { databases := make(map[string]uint32) it, err := tx.Read(ctx, keys.NewKey(k.EncodingSubspaceName(), encVersion, UInt32ToByte(namespaceId), dbKey)) if err != nil { @@ -423,7 +423,7 @@ func (k *DictionaryEncoder) GetDatabases(ctx context.Context, tx transaction.Tx, return databases, it.Err() } -func (k *DictionaryEncoder) GetCollections(ctx context.Context, tx transaction.Tx, namespaceId uint32, databaseId uint32) (map[string]uint32, error) { +func (k *MetadataDictionary) GetCollections(ctx context.Context, tx transaction.Tx, namespaceId uint32, databaseId uint32) (map[string]uint32, error) { var collections = make(map[string]uint32) it, err := tx.Read(ctx, keys.NewKey(k.EncodingSubspaceName(), encVersion, UInt32ToByte(namespaceId), UInt32ToByte(databaseId))) if err != nil { @@ -470,7 +470,7 @@ func (k *DictionaryEncoder) GetCollections(ctx context.Context, tx transaction.T return collections, it.Err() } -func (k *DictionaryEncoder) GetIndexes(ctx context.Context, tx transaction.Tx, namespaceId uint32, databaseId uint32, collId uint32) (map[string]uint32, error) { +func (k *MetadataDictionary) GetIndexes(ctx context.Context, tx transaction.Tx, namespaceId uint32, databaseId uint32, collId uint32) (map[string]uint32, error) { var indexes = make(map[string]uint32) it, err := tx.Read(ctx, keys.NewKey(k.EncodingSubspaceName(), encVersion, UInt32ToByte(namespaceId), UInt32ToByte(databaseId), UInt32ToByte(collId))) if err != nil { @@ -517,22 +517,22 @@ func (k *DictionaryEncoder) GetIndexes(ctx context.Context, tx transaction.Tx, n return indexes, it.Err() } -func (k *DictionaryEncoder) GetDatabaseId(ctx context.Context, tx transaction.Tx, dbName string, namespaceId uint32) (uint32, error) { +func (k *MetadataDictionary) GetDatabaseId(ctx context.Context, tx transaction.Tx, dbName string, namespaceId uint32) (uint32, error) { key := keys.NewKey(k.EncodingSubspaceName(), encVersion, UInt32ToByte(namespaceId), dbKey, dbName, keyEnd) return k.getId(ctx, tx, key) } -func (k *DictionaryEncoder) GetCollectionId(ctx context.Context, tx transaction.Tx, collName string, namespaceId uint32, dbId uint32) (uint32, error) { +func (k *MetadataDictionary) GetCollectionId(ctx context.Context, tx transaction.Tx, collName string, namespaceId uint32, dbId uint32) (uint32, error) { key := keys.NewKey(k.EncodingSubspaceName(), encVersion, UInt32ToByte(namespaceId), UInt32ToByte(dbId), collectionKey, collName, keyEnd) return k.getId(ctx, tx, key) } -func (k *DictionaryEncoder) GetIndexId(ctx context.Context, tx transaction.Tx, indexName string, namespaceId uint32, dbId uint32, collId uint32) (uint32, error) { +func (k *MetadataDictionary) GetIndexId(ctx context.Context, tx transaction.Tx, indexName string, namespaceId uint32, dbId uint32, collId uint32) (uint32, error) { key := keys.NewKey(k.EncodingSubspaceName(), encVersion, UInt32ToByte(namespaceId), UInt32ToByte(dbId), UInt32ToByte(collId), indexKey, indexName, keyEnd) return k.getId(ctx, tx, key) } -func (k *DictionaryEncoder) getId(ctx context.Context, tx transaction.Tx, key keys.Key) (uint32, error) { +func (k *MetadataDictionary) getId(ctx context.Context, tx transaction.Tx, key keys.Key) (uint32, error) { it, err := tx.Read(ctx, key) if err != nil { return InvalidId, err @@ -553,7 +553,7 @@ func (k *DictionaryEncoder) getId(ctx context.Context, tx transaction.Tx, key ke // decode is currently only use for debugging purpose, once we have a layer on top of this encoding then we leverage this // method -func (k *DictionaryEncoder) decode(_ context.Context, fdbKey kv.Key) (map[string]interface{}, error) { +func (k *MetadataDictionary) decode(_ context.Context, fdbKey kv.Key) (map[string]interface{}, error) { var decoded = make(map[string]interface{}) if len(fdbKey) > 0 { decoded["version"] = fdbKey[0] diff --git a/server/metadata/encoding/dictionary_test.go b/server/metadata/encoding/dictionary_test.go index 3599e4477..7e2b53f7f 100644 --- a/server/metadata/encoding/dictionary_test.go +++ b/server/metadata/encoding/dictionary_test.go @@ -38,7 +38,7 @@ func TestDictionaryEncoding(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - k := NewDictionaryEncoder(&TestMDNameRegistry{ + k := NewMetadataDictionary(&TestMDNameRegistry{ ReserveSB: "test_reserved", EncodingSB: "test_encoding", }) @@ -54,19 +54,19 @@ func TestDictionaryEncoding(t *testing.T) { tx, err = tm.StartTx(ctx) require.NoError(t, err) - dbId, err := k.EncodeDatabaseName(ctx, tx, "db-1", 1234) + dbId, err := k.CreateDatabase(ctx, tx, "db-1", 1234) require.NoError(t, err) require.NoError(t, tx.Commit(ctx)) tx, err = tm.StartTx(ctx) require.NoError(t, err) - collId, err := k.EncodeCollectionName(ctx, tx, "coll-1", 1234, 1) + collId, err := k.CreateCollection(ctx, tx, "coll-1", 1234, 1) require.NoError(t, err) require.NoError(t, tx.Commit(ctx)) tx, err = tm.StartTx(ctx) require.NoError(t, err) - indexId, err := k.EncodeIndexName(ctx, tx, "pkey", 1234, 1, 2) + indexId, err := k.CreateIndex(ctx, tx, "pkey", 1234, 1, 2) require.NoError(t, err) require.NoError(t, tx.Commit(ctx)) @@ -103,7 +103,7 @@ func TestDictionaryEncodingDropped(t *testing.T) { defer cancel() t.Run("drop_database", func(t *testing.T) { - k := NewDictionaryEncoder(&TestMDNameRegistry{ + k := NewMetadataDictionary(&TestMDNameRegistry{ ReserveSB: "test_reserved", EncodingSB: "test_encoding", }) @@ -120,7 +120,7 @@ func TestDictionaryEncodingDropped(t *testing.T) { tx, err = tm.StartTx(ctx) require.NoError(t, err) - dbId, err := k.EncodeDatabaseName(ctx, tx, "db-1", 1234) + dbId, err := k.CreateDatabase(ctx, tx, "db-1", 1234) require.NoError(t, err) require.NoError(t, tx.Commit(ctx)) @@ -133,7 +133,7 @@ func TestDictionaryEncodingDropped(t *testing.T) { tx, err = tm.StartTx(ctx) require.NoError(t, err) - err = k.EncodeDatabaseAsDropped(ctx, tx, "db-1", 1234, dbId) + err = k.DropDatabase(ctx, tx, "db-1", 1234, dbId) require.NoError(t, err) require.NoError(t, tx.Commit(ctx)) @@ -145,7 +145,7 @@ func TestDictionaryEncodingDropped(t *testing.T) { require.Equal(t, v, InvalidId) }) t.Run("drop_collection", func(t *testing.T) { - k := NewDictionaryEncoder(&TestMDNameRegistry{ + k := NewMetadataDictionary(&TestMDNameRegistry{ ReserveSB: "test_reserved", EncodingSB: "test_encoding", }) @@ -161,13 +161,13 @@ func TestDictionaryEncodingDropped(t *testing.T) { tx, err = tm.StartTx(ctx) require.NoError(t, err) - dbId, err := k.EncodeDatabaseName(ctx, tx, "db-1", 1234) + dbId, err := k.CreateDatabase(ctx, tx, "db-1", 1234) require.NoError(t, err) require.NoError(t, tx.Commit(ctx)) tx, err = tm.StartTx(ctx) require.NoError(t, err) - collId, err := k.EncodeCollectionName(ctx, tx, "coll-1", 1234, dbId) + collId, err := k.CreateCollection(ctx, tx, "coll-1", 1234, dbId) require.NoError(t, err) require.NoError(t, tx.Commit(ctx)) @@ -180,7 +180,7 @@ func TestDictionaryEncodingDropped(t *testing.T) { tx, err = tm.StartTx(ctx) require.NoError(t, err) - err = k.EncodeCollectionAsDropped(ctx, tx, "coll-1", 1234, dbId, collId) + err = k.DropCollection(ctx, tx, "coll-1", 1234, dbId, collId) require.NoError(t, err) require.NoError(t, tx.Commit(ctx)) @@ -192,7 +192,7 @@ func TestDictionaryEncodingDropped(t *testing.T) { require.Equal(t, v, InvalidId) }) t.Run("drop_index", func(t *testing.T) { - k := NewDictionaryEncoder(&TestMDNameRegistry{ + k := NewMetadataDictionary(&TestMDNameRegistry{ ReserveSB: "test_reserved", EncodingSB: "test_encoding", }) @@ -208,19 +208,19 @@ func TestDictionaryEncodingDropped(t *testing.T) { tx, err = tm.StartTx(ctx) require.NoError(t, err) - dbId, err := k.EncodeDatabaseName(ctx, tx, "db-1", 1234) + dbId, err := k.CreateDatabase(ctx, tx, "db-1", 1234) require.NoError(t, err) require.NoError(t, tx.Commit(ctx)) tx, err = tm.StartTx(ctx) require.NoError(t, err) - collId, err := k.EncodeCollectionName(ctx, tx, "coll-1", 1234, dbId) + collId, err := k.CreateCollection(ctx, tx, "coll-1", 1234, dbId) require.NoError(t, err) require.NoError(t, tx.Commit(ctx)) tx, err = tm.StartTx(ctx) require.NoError(t, err) - idxId, err := k.EncodeIndexName(ctx, tx, "idx-1", 1234, dbId, collId) + idxId, err := k.CreateIndex(ctx, tx, "idx-1", 1234, dbId, collId) require.NoError(t, err) require.NoError(t, tx.Commit(ctx)) @@ -233,7 +233,7 @@ func TestDictionaryEncodingDropped(t *testing.T) { tx, err = tm.StartTx(ctx) require.NoError(t, err) - err = k.EncodeIndexAsDropped(ctx, tx, "idx-1", 1234, dbId, collId, idxId) + err = k.DropIndex(ctx, tx, "idx-1", 1234, dbId, collId, idxId) require.NoError(t, err) require.NoError(t, tx.Commit(ctx)) @@ -245,7 +245,7 @@ func TestDictionaryEncodingDropped(t *testing.T) { require.Equal(t, v, InvalidId) }) t.Run("drop_collection_multiple", func(t *testing.T) { - k := NewDictionaryEncoder(&TestMDNameRegistry{ + k := NewMetadataDictionary(&TestMDNameRegistry{ ReserveSB: "test_reserved", EncodingSB: "test_encoding", }) @@ -261,13 +261,13 @@ func TestDictionaryEncodingDropped(t *testing.T) { tx, err = tm.StartTx(ctx) require.NoError(t, err) - dbId, err := k.EncodeDatabaseName(ctx, tx, "db-1", 1234) + dbId, err := k.CreateDatabase(ctx, tx, "db-1", 1234) require.NoError(t, err) require.NoError(t, tx.Commit(ctx)) tx, err = tm.StartTx(ctx) require.NoError(t, err) - collId, err := k.EncodeCollectionName(ctx, tx, "coll-1", 1234, dbId) + collId, err := k.CreateCollection(ctx, tx, "coll-1", 1234, dbId) require.NoError(t, err) require.NoError(t, tx.Commit(ctx)) @@ -280,7 +280,7 @@ func TestDictionaryEncodingDropped(t *testing.T) { tx, err = tm.StartTx(ctx) require.NoError(t, err) - err = k.EncodeCollectionAsDropped(ctx, tx, "coll-1", 1234, dbId, collId) + err = k.DropCollection(ctx, tx, "coll-1", 1234, dbId, collId) require.NoError(t, err) require.NoError(t, tx.Commit(ctx)) @@ -293,7 +293,7 @@ func TestDictionaryEncodingDropped(t *testing.T) { tx, err = tm.StartTx(ctx) require.NoError(t, err) - newCollId, err := k.EncodeCollectionName(ctx, tx, "coll-1", 1234, dbId) + newCollId, err := k.CreateCollection(ctx, tx, "coll-1", 1234, dbId) require.NoError(t, err) require.NoError(t, tx.Commit(ctx)) @@ -316,7 +316,7 @@ func TestDictionaryEncoding_Error(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - k := NewDictionaryEncoder(&TestMDNameRegistry{ + k := NewMetadataDictionary(&TestMDNameRegistry{ ReserveSB: "test_reserved", EncodingSB: "test_encoding", }) @@ -327,15 +327,15 @@ func TestDictionaryEncoding_Error(t *testing.T) { tm := transaction.NewManager(kvs) tx, err := tm.StartTx(ctx) require.NoError(t, err) - dbId, err := k.EncodeDatabaseName(ctx, tx, "db-1", 0) + dbId, err := k.CreateDatabase(ctx, tx, "db-1", 0) require.Error(t, api.Errorf(api.Code_INVALID_ARGUMENT, "invalid namespace id"), err) require.Equal(t, InvalidId, dbId) - collId, err := k.EncodeCollectionName(ctx, tx, "coll-1", 1234, 0) + collId, err := k.CreateCollection(ctx, tx, "coll-1", 1234, 0) require.Error(t, api.Errorf(api.Code_INVALID_ARGUMENT, "invalid database id"), err) require.Equal(t, InvalidId, collId) - indexId, err := k.EncodeIndexName(ctx, tx, "pkey", 1234, 1, 0) + indexId, err := k.CreateIndex(ctx, tx, "pkey", 1234, 1, 0) require.Error(t, api.Errorf(api.Code_INVALID_ARGUMENT, "invalid collection id"), err) require.Equal(t, InvalidId, indexId) require.NoError(t, tx.Rollback(context.TODO())) @@ -353,7 +353,7 @@ func TestDictionaryEncoding_GetMethods(t *testing.T) { tm := transaction.NewManager(kvs) t.Run("get_databases", func(t *testing.T) { - k := NewDictionaryEncoder(&TestMDNameRegistry{ + k := NewMetadataDictionary(&TestMDNameRegistry{ ReserveSB: "test_reserved", EncodingSB: "test_encoding", }) @@ -363,9 +363,9 @@ func TestDictionaryEncoding_GetMethods(t *testing.T) { tx, err := tm.StartTx(ctx) require.NoError(t, err) - dbId1, err := k.EncodeDatabaseName(ctx, tx, "db-1", 1) + dbId1, err := k.CreateDatabase(ctx, tx, "db-1", 1) require.NoError(t, err) - dbId2, err := k.EncodeDatabaseName(ctx, tx, "db-2", 1) + dbId2, err := k.CreateDatabase(ctx, tx, "db-2", 1) require.NoError(t, err) dbToId, err := k.GetDatabases(ctx, tx, 1) @@ -376,7 +376,7 @@ func TestDictionaryEncoding_GetMethods(t *testing.T) { require.Equal(t, dbToId["db-2"], dbId2) }) t.Run("get_collections", func(t *testing.T) { - k := NewDictionaryEncoder(&TestMDNameRegistry{ + k := NewMetadataDictionary(&TestMDNameRegistry{ ReserveSB: "test_reserved", EncodingSB: "test_encoding", }) @@ -386,12 +386,12 @@ func TestDictionaryEncoding_GetMethods(t *testing.T) { tx, err := tm.StartTx(ctx) require.NoError(t, err) - dbId, err := k.EncodeDatabaseName(ctx, tx, "db-1", 1) + dbId, err := k.CreateDatabase(ctx, tx, "db-1", 1) require.NoError(t, err) - cid1, err := k.EncodeCollectionName(ctx, tx, "coll-1", 1, dbId) + cid1, err := k.CreateCollection(ctx, tx, "coll-1", 1, dbId) require.NoError(t, err) - cid2, err := k.EncodeCollectionName(ctx, tx, "coll-2", 1, dbId) + cid2, err := k.CreateCollection(ctx, tx, "coll-2", 1, dbId) require.NoError(t, err) collToId, err := k.GetCollections(ctx, tx, 1, dbId) @@ -402,7 +402,7 @@ func TestDictionaryEncoding_GetMethods(t *testing.T) { require.Equal(t, collToId["coll-2"], cid2) }) t.Run("get_indexes", func(t *testing.T) { - k := NewDictionaryEncoder(&TestMDNameRegistry{ + k := NewMetadataDictionary(&TestMDNameRegistry{ ReserveSB: "test_reserved", EncodingSB: "test_encoding", }) @@ -412,13 +412,13 @@ func TestDictionaryEncoding_GetMethods(t *testing.T) { tx, err := tm.StartTx(ctx) require.NoError(t, err) - dbId, err := k.EncodeDatabaseName(ctx, tx, "db-1", 1) + dbId, err := k.CreateDatabase(ctx, tx, "db-1", 1) require.NoError(t, err) - cid1, err := k.EncodeCollectionName(ctx, tx, "coll-1", 1, dbId) + cid1, err := k.CreateCollection(ctx, tx, "coll-1", 1, dbId) require.NoError(t, err) - pkid, err := k.EncodeIndexName(ctx, tx, "pkey", 1, dbId, cid1) + pkid, err := k.CreateIndex(ctx, tx, "pkey", 1, dbId, cid1) require.NoError(t, err) idxToId, err := k.GetIndexes(ctx, tx, 1, dbId, cid1) require.NoError(t, err) @@ -468,7 +468,7 @@ func TestReservedNamespace(t *testing.T) { func TestDecode(t *testing.T) { k := kv.BuildKey(encVersion, UInt32ToByte(1234), dbKey, "db-1", keyEnd) - mp, err := NewDictionaryEncoder(&TestMDNameRegistry{ + mp, err := NewMetadataDictionary(&TestMDNameRegistry{ ReserveSB: "test_reserved", EncodingSB: "test_encoding", }).decode(context.TODO(), k) diff --git a/server/metadata/key_encoder.go b/server/metadata/key_encoder.go index a8791579a..497ac43c6 100644 --- a/server/metadata/key_encoder.go +++ b/server/metadata/key_encoder.go @@ -16,16 +16,14 @@ package metadata import ( "bytes" + api "github.com/tigrisdata/tigris/api/server/v1" + "github.com/tigrisdata/tigris/internal" "github.com/tigrisdata/tigris/keys" "github.com/tigrisdata/tigris/schema" "github.com/tigrisdata/tigris/server/metadata/encoding" ) -var ( - userTableKeyPrefix = []byte("data") -) - // Encoder is used to encode/decode values of the Key. type Encoder interface { // EncodeTableName returns encoded bytes which are formed by combining namespace, database, and collection. @@ -40,29 +38,24 @@ type Encoder interface { // information i.e. whether the index is pkey, etc. The remaining elements are values for this index. EncodeKey(encodedTable []byte, idx *schema.Index, idxParts []interface{}) (keys.Key, error) - // DecodeTableName is used to decode the key stored in FDB and extract namespace name, database name and collection name. - DecodeTableName(tableName []byte) (string, string, string, bool) + // DecodeTableName is used to decode the key stored in FDB and extract namespace name, database name and collection ids. + DecodeTableName(tableName []byte) (uint32, uint32, uint32, bool) DecodeIndexName(indexName []byte) uint32 } -// NewEncoder creates Dictionary encoder to encode keys. -func NewEncoder(mgr *TenantManager) Encoder { - return &DictKeyEncoder{ - mgr: mgr, - } +// NewEncoder creates Dictionary metaStore to encode keys. +func NewEncoder() Encoder { + return &DictKeyEncoder{} } type DictKeyEncoder struct { - mgr *TenantManager } +// EncodeTableName creates storage friendly table name from namespace, database and collection ids +// Database and collection objects can be omitted to get table name prefix. +// If the collection is ommitted then result name includes all the collections in the database +// If both database and collections are omitted then result name includes all databases in the namespace func (d *DictKeyEncoder) EncodeTableName(ns Namespace, db *Database, coll *schema.DefaultCollection) ([]byte, error) { - if db == nil { - return nil, api.Errorf(api.Code_INVALID_ARGUMENT, "database is missing") - } - if coll == nil { - return nil, api.Errorf(api.Code_INVALID_ARGUMENT, "collection is missing") - } return d.encodedTableName(ns, db, coll), nil } @@ -86,10 +79,14 @@ func (d *DictKeyEncoder) EncodeKey(encodedTable []byte, idx *schema.Index, idxPa func (d *DictKeyEncoder) encodedTableName(ns Namespace, db *Database, coll *schema.DefaultCollection) []byte { var appendTo []byte - appendTo = append(appendTo, userTableKeyPrefix...) + appendTo = append(appendTo, internal.UserTableKeyPrefix...) appendTo = append(appendTo, encoding.UInt32ToByte(ns.Id())...) - appendTo = append(appendTo, encoding.UInt32ToByte(db.id)...) - appendTo = append(appendTo, encoding.UInt32ToByte(coll.Id)...) + if db != nil { + appendTo = append(appendTo, encoding.UInt32ToByte(db.id)...) + } + if coll != nil { + appendTo = append(appendTo, encoding.UInt32ToByte(coll.Id)...) + } return appendTo } @@ -97,16 +94,16 @@ func (d *DictKeyEncoder) encodedIdxName(idx *schema.Index) []byte { return encoding.UInt32ToByte(idx.Id) } -func (d *DictKeyEncoder) DecodeTableName(tableName []byte) (string, string, string, bool) { - if len(tableName) < 16 || !bytes.Equal(tableName[0:4], userTableKeyPrefix) { - return "", "", "", false +func (d *DictKeyEncoder) DecodeTableName(tableName []byte) (uint32, uint32, uint32, bool) { + if len(tableName) < 16 || !bytes.Equal(tableName[0:4], internal.UserTableKeyPrefix) { + return 0, 0, 0, false } nsId := encoding.ByteToUInt32(tableName[4:8]) dbId := encoding.ByteToUInt32(tableName[8:12]) collId := encoding.ByteToUInt32(tableName[12:16]) - return d.mgr.GetTableNameFromId(nsId, dbId, collId) + return nsId, dbId, collId, true } func (d *DictKeyEncoder) DecodeIndexName(indexName []byte) uint32 { diff --git a/server/metadata/key_encoder_test.go b/server/metadata/key_encoder_test.go index 88715a5f9..5f8894a9a 100644 --- a/server/metadata/key_encoder_test.go +++ b/server/metadata/key_encoder_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/stretchr/testify/require" + "github.com/tigrisdata/tigris/internal" "github.com/tigrisdata/tigris/schema" "github.com/tigrisdata/tigris/server/metadata/encoding" ) @@ -54,10 +55,10 @@ func TestEncodeDecodeKey(t *testing.T) { }, } - k := NewEncoder(mgr) + k := NewEncoder() encodedTable, err := k.EncodeTableName(ns, db, coll) require.NoError(t, err) - require.Equal(t, userTableKeyPrefix, encodedTable[0:4]) + require.Equal(t, internal.UserTableKeyPrefix, encodedTable[0:4]) require.Equal(t, uint32(1), encoding.ByteToUInt32(encodedTable[4:8])) require.Equal(t, uint32(3), encoding.ByteToUInt32(encodedTable[8:12])) require.Equal(t, uint32(5), encoding.ByteToUInt32(encodedTable[12:16])) @@ -65,7 +66,12 @@ func TestEncodeDecodeKey(t *testing.T) { encodedIdx := k.EncodeIndexName(idx) require.Equal(t, uint32(10), encoding.ByteToUInt32(encodedIdx)) - tenantName, dbName, collName, ok := k.DecodeTableName(encodedTable) + tenantID, dbID, collID, ok := k.DecodeTableName(encodedTable) + require.True(t, ok) + + tenantName, dbName, collName, ok := mgr.GetTableNameFromIds(tenantID, dbID, collID) + require.True(t, ok) + require.Equal(t, ns.Name(), tenantName) require.Equal(t, db.name, dbName) require.Equal(t, coll.Name, collName) diff --git a/server/metadata/tenant.go b/server/metadata/tenant.go index dc8019524..925fa308e 100644 --- a/server/metadata/tenant.go +++ b/server/metadata/tenant.go @@ -18,8 +18,10 @@ import ( "bytes" "context" "fmt" + "math/rand" "reflect" "sync" + "time" jsoniter "github.com/json-iterator/go" "github.com/rs/zerolog/log" @@ -104,25 +106,29 @@ func (n *TenantNamespace) Id() uint32 { type TenantManager struct { sync.RWMutex - encoder *encoding.DictionaryEncoder + metaStore *encoding.MetadataDictionary schemaStore *encoding.SchemaSubspace kvStore kv.KeyValueStore + searchStore search.Store tenants map[string]*Tenant idToTenantMap map[uint32]string version Version versionH *VersionHandler mdNameRegistry encoding.MDNameRegistry + encoder Encoder } -func NewTenantManager(kvStore kv.KeyValueStore) *TenantManager { +func NewTenantManager(kvStore kv.KeyValueStore, searchStore search.Store) *TenantManager { mdNameRegistry := &encoding.DefaultMDNameRegistry{} - return newTenantManager(kvStore, mdNameRegistry) + return newTenantManager(kvStore, searchStore, mdNameRegistry) } -func newTenantManager(kvStore kv.KeyValueStore, mdNameRegistry encoding.MDNameRegistry) *TenantManager { +func newTenantManager(kvStore kv.KeyValueStore, searchStore search.Store, mdNameRegistry encoding.MDNameRegistry) *TenantManager { return &TenantManager{ kvStore: kvStore, - encoder: encoding.NewDictionaryEncoder(mdNameRegistry), + searchStore: searchStore, + encoder: NewEncoder(), + metaStore: encoding.NewMetadataDictionary(mdNameRegistry), schemaStore: encoding.NewSchemaStore(mdNameRegistry), tenants: make(map[string]*Tenant), idToTenantMap: make(map[uint32]string), @@ -131,6 +137,13 @@ func newTenantManager(kvStore kv.KeyValueStore, mdNameRegistry encoding.MDNameRe } } +func (m *TenantManager) EnsureDefaultNamespace(txMgr *transaction.Manager) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := m.CreateOrGetTenant(ctx, txMgr, NewDefaultNamespace()) + return err +} + // CreateOrGetTenant is a thread safe implementation of creating a new tenant. It returns the tenant if it already exists. // This is mainly returning the tenant to avoid calling "Get" again after creating the tenant. This method is expensive // as it reloads the existing tenants from the disk if it sees the tenant is not present in the cache. @@ -170,11 +183,15 @@ func (m *TenantManager) CreateOrGetTenant(ctx context.Context, txMgr *transactio return m.createOrGetTenantInternal(ctx, tx, namespace) } +func (m *TenantManager) GetEncoder() Encoder { + return m.encoder +} + // CreateTenant is a thread safe implementation of creating a new tenant. It returns the error if it already exists. func (m *TenantManager) CreateTenant(ctx context.Context, tx transaction.Tx, namespace Namespace) (Namespace, error) { m.Lock() defer m.Unlock() - namespaces, err := m.encoder.GetNamespaces(ctx, tx) + namespaces, err := m.metaStore.GetNamespaces(ctx, tx) if err != nil { return nil, err } @@ -187,7 +204,7 @@ func (m *TenantManager) CreateTenant(ctx context.Context, tx transaction.Tx, nam return nil, api.Errorf(api.Code_CONFLICT, "namespace with same id already exists with name '%s'", name) } } - if err := m.encoder.ReserveNamespace(ctx, tx, namespace.Name(), namespace.Id()); ulog.E(err) { + if err := m.metaStore.ReserveNamespace(ctx, tx, namespace.Name(), namespace.Id()); ulog.E(err) { return nil, err } if err := m.versionH.Increment(ctx, tx); ulog.E(err) { @@ -195,6 +212,7 @@ func (m *TenantManager) CreateTenant(ctx context.Context, tx transaction.Tx, nam } return namespace, nil } + func (m *TenantManager) getTenantFromCache(namespaceName string) (tenant *Tenant) { m.RLock() defer m.RUnlock() @@ -207,9 +225,8 @@ func (m *TenantManager) getTenantFromCache(namespaceName string) (tenant *Tenant // GetTenant is responsible for returning the tenant from the cache. If the tenant is not available in the cache then // this method will attempt to load it from the disk and will update the tenant manager cache accordingly. func (m *TenantManager) GetTenant(ctx context.Context, namespaceName string, txMgr *transaction.Manager) (tenant *Tenant, err error) { - tenant = m.getTenantFromCache(namespaceName) - if tenant != nil { - return tenant, nil + if tenant = m.getTenantFromCache(namespaceName); tenant != nil { + return } m.Lock() @@ -232,27 +249,27 @@ func (m *TenantManager) GetTenant(ctx context.Context, namespaceName string, txM m.idToTenantMap[tenant.namespace.Id()] = tenant.namespace.Name() } } else { - log.Warn().Msg("Could not get namespaces") + log.Err(err).Str("ns", namespaceName).Msg("Could not get namespace") _ = tx.Rollback(ctx) } }() var namespaces map[string]uint32 - if namespaces, err = m.encoder.GetNamespaces(ctx, tx); err != nil { + if namespaces, err = m.metaStore.GetNamespaces(ctx, tx); err != nil { return nil, err } id, ok := namespaces[namespaceName] if !ok { - return nil, nil + return nil, fmt.Errorf("namespace not found: %s", namespaceName) } currentVersion, err := m.versionH.Read(ctx, tx) - if ulog.E(err) { + if err != nil { return nil, err } namespace := NewTenantNamespace(namespaceName, id) - tenant = NewTenant(namespace, m.kvStore, m.encoder, m.schemaStore, m.versionH, currentVersion) + tenant = NewTenant(namespace, m.kvStore, m.searchStore, m.metaStore, m.schemaStore, m.encoder, m.versionH, currentVersion) if err = tenant.reload(ctx, tx, currentVersion); err != nil { return nil, err } @@ -262,7 +279,7 @@ func (m *TenantManager) GetTenant(ctx context.Context, namespaceName string, txM func (m *TenantManager) ListNamespaces(ctx context.Context, tx transaction.Tx) ([]Namespace, error) { m.RLock() defer m.RUnlock() - namespaces, err := m.encoder.GetNamespaces(ctx, tx) + namespaces, err := m.metaStore.GetNamespaces(ctx, tx) if err != nil { _ = tx.Rollback(ctx) log.Warn().Err(err).Msg("Could not list namespaces") @@ -276,7 +293,7 @@ func (m *TenantManager) ListNamespaces(ctx context.Context, tx transaction.Tx) ( } func (m *TenantManager) createOrGetTenantInternal(ctx context.Context, tx transaction.Tx, namespace Namespace) (*Tenant, error) { - namespaces, err := m.encoder.GetNamespaces(ctx, tx) + namespaces, err := m.metaStore.GetNamespaces(ctx, tx) if err != nil { return nil, err } @@ -288,7 +305,7 @@ func (m *TenantManager) createOrGetTenantInternal(ctx context.Context, tx transa return nil, err } - tenant := NewTenant(namespace, m.kvStore, m.encoder, m.schemaStore, m.versionH, currentVersion) + tenant := NewTenant(namespace, m.kvStore, m.searchStore, m.metaStore, m.schemaStore, m.encoder, m.versionH, currentVersion) tenant.Lock() err = tenant.reload(ctx, tx, currentVersion) tenant.Unlock() @@ -302,15 +319,15 @@ func (m *TenantManager) createOrGetTenantInternal(ctx context.Context, tx transa return nil, err } - if err := m.encoder.ReserveNamespace(ctx, tx, namespace.Name(), namespace.Id()); ulog.E(err) { + if err := m.metaStore.ReserveNamespace(ctx, tx, namespace.Name(), namespace.Id()); ulog.E(err) { return nil, err } - return NewTenant(namespace, m.kvStore, m.encoder, m.schemaStore, m.versionH, nil), nil + return NewTenant(namespace, m.kvStore, m.searchStore, m.metaStore, m.schemaStore, m.encoder, m.versionH, nil), nil } -// GetTableNameFromId returns tenant name, database name, collection name corresponding to their encoded ids. -func (m *TenantManager) GetTableNameFromId(tenantId uint32, dbId uint32, collId uint32) (string, string, string, bool) { +// GetTableNameFromIds returns tenant name, database name, collection name corresponding to their encoded ids. +func (m *TenantManager) GetTableNameFromIds(tenantId uint32, dbId uint32, collId uint32) (string, string, string, bool) { m.RLock() defer m.RUnlock() @@ -342,6 +359,38 @@ func (m *TenantManager) GetTableNameFromId(tenantId uint32, dbId uint32, collId return tenantName, dbName, collName, ok } +// GetDatabaseAndCollectionId returns the id of db and c in the default namespace. This is just a temporary API for +// the streams to know if database and collection exists at the start of streaming and their corresponding IDs. +func (m *TenantManager) GetDatabaseAndCollectionId(db string, c string) (uint32, uint32) { + m.RLock() + defer m.RUnlock() + + tenant, ok := m.tenants[DefaultNamespaceName] + if !ok { + return 0, 0 + } + database, ok := tenant.databases[db] + if !ok { + return 0, 0 + } + + coll, ok := database.collections[c] + if !ok { + return 0, 0 + } + + return database.id, coll.id +} + +func (m *TenantManager) DecodeTableName(tableName []byte) (string, string, string, bool) { + n, d, c, ok := m.encoder.DecodeTableName(tableName) + if !ok { + return "", "", "", false + } + + return m.GetTableNameFromIds(n, d, c) +} + // Reload reads all the namespaces exists in the disk and build the in-memory map of the manager to track the tenants. // As this is an expensive call, the reloading happens during start time for now. It is possible that reloading // fails during start time then we rely on lazily reloading cache during serving user requests. @@ -364,7 +413,7 @@ func (m *TenantManager) Reload(ctx context.Context, tx transaction.Tx) error { } func (m *TenantManager) reload(ctx context.Context, tx transaction.Tx, currentVersion Version) error { - namespaces, err := m.encoder.GetNamespaces(ctx, tx) + namespaces, err := m.metaStore.GetNamespaces(ctx, tx) if err != nil { return err } @@ -372,7 +421,7 @@ func (m *TenantManager) reload(ctx context.Context, tx transaction.Tx, currentVe for namespace, id := range namespaces { if _, ok := m.tenants[namespace]; !ok { - m.tenants[namespace] = NewTenant(NewTenantNamespace(namespace, id), m.kvStore, m.encoder, m.schemaStore, m.versionH, currentVersion) + m.tenants[namespace] = NewTenant(NewTenantNamespace(namespace, id), m.kvStore, m.searchStore, m.metaStore, m.schemaStore, m.encoder, m.versionH, currentVersion) m.idToTenantMap[id] = namespace } } @@ -395,8 +444,10 @@ type Tenant struct { sync.RWMutex kvStore kv.KeyValueStore - encoder *encoding.DictionaryEncoder + searchStore search.Store schemaStore *encoding.SchemaSubspace + metaStore *encoding.MetadataDictionary + Encoder Encoder databases map[string]*Database idToDatabaseMap map[uint32]string namespace Namespace @@ -404,16 +455,18 @@ type Tenant struct { versionH *VersionHandler } -func NewTenant(namespace Namespace, kvStore kv.KeyValueStore, encoder *encoding.DictionaryEncoder, schemaStore *encoding.SchemaSubspace, versionH *VersionHandler, currentVersion Version) *Tenant { +func NewTenant(namespace Namespace, kvStore kv.KeyValueStore, searchStore search.Store, dict *encoding.MetadataDictionary, schemaStore *encoding.SchemaSubspace, encoder Encoder, versionH *VersionHandler, currentVersion Version) *Tenant { return &Tenant{ kvStore: kvStore, + searchStore: searchStore, namespace: namespace, - encoder: encoder, + metaStore: dict, schemaStore: schemaStore, databases: make(map[string]*Database), idToDatabaseMap: make(map[uint32]string), versionH: versionH, version: currentVersion, + Encoder: encoder, } } @@ -425,7 +478,7 @@ func (tenant *Tenant) reload(ctx context.Context, tx transaction.Tx, currentVers tenant.databases = make(map[string]*Database) tenant.idToDatabaseMap = make(map[uint32]string) - dbNameToId, err := tenant.encoder.GetDatabases(ctx, tx, tenant.namespace.Id()) + dbNameToId, err := tenant.metaStore.GetDatabases(ctx, tx, tenant.namespace.Id()) if err != nil { return err } @@ -509,13 +562,13 @@ func (tenant *Tenant) CreateDatabase(ctx context.Context, tx transaction.Tx, dbN // otherwise, proceed to create the database if there are concurrent requests on different workers then one of // them will fail with duplicate entry and only one will succeed. - _, err := tenant.encoder.EncodeDatabaseName(ctx, tx, dbName, tenant.namespace.Id()) + _, err := tenant.metaStore.CreateDatabase(ctx, tx, dbName, tenant.namespace.Id()) return false, err } // DropDatabase is responsible for first dropping a dictionary encoding of the database and then adding a corresponding // dropped encoding in the table. Drop returns "false" if database doesn't exist so that caller can reason about. -func (tenant *Tenant) DropDatabase(ctx context.Context, tx transaction.Tx, dbName string, searchStore search.Store, rowKeyEncoder Encoder) (bool, error) { +func (tenant *Tenant) DropDatabase(ctx context.Context, tx transaction.Tx, dbName string) (bool, error) { tenant.Lock() defer tenant.Unlock() @@ -527,12 +580,12 @@ func (tenant *Tenant) DropDatabase(ctx context.Context, tx transaction.Tx, dbNam // if there are concurrent requests on different workers then one of them will fail with duplicate entry and only // one will succeed. - if err := tenant.encoder.EncodeDatabaseAsDropped(ctx, tx, dbName, tenant.namespace.Id(), db.id); err != nil { + if err := tenant.metaStore.DropDatabase(ctx, tx, dbName, tenant.namespace.Id(), db.id); err != nil { return true, err } for _, c := range db.collections { - if err := tenant.dropCollection(ctx, tx, db, c.collection.Name, searchStore, rowKeyEncoder); err != nil { + if err := tenant.dropCollection(ctx, tx, db, c.collection.Name); err != nil { return true, err } } @@ -572,13 +625,13 @@ func (tenant *Tenant) ListDatabases(_ context.Context, _ transaction.Tx) []strin func (tenant *Tenant) reloadDatabase(ctx context.Context, tx transaction.Tx, dbName string, dbId uint32) (*Database, error) { database := NewDatabase(dbId, dbName) - collNameToId, err := tenant.encoder.GetCollections(ctx, tx, tenant.namespace.Id(), database.id) + collNameToId, err := tenant.metaStore.GetCollections(ctx, tx, tenant.namespace.Id(), database.id) if err != nil { return nil, err } for coll, id := range collNameToId { - idxNameToId, err := tenant.encoder.GetIndexes(ctx, tx, tenant.namespace.Id(), database.id, id) + idxNameToId, err := tenant.metaStore.GetIndexes(ctx, tx, tenant.namespace.Id(), database.id, id) if err != nil { database.needFixingCollections[coll] = struct{}{} log.Debug().Err(err).Str("collection", coll).Msg("skipping loading collection") @@ -618,7 +671,7 @@ func (tenant *Tenant) GetCollection(db string, collection string) *schema.Defaul } // CreateCollection is to create a collection inside tenant namespace. -func (tenant *Tenant) CreateCollection(ctx context.Context, tx transaction.Tx, database *Database, schFactory *schema.Factory, searchStore search.Store) error { +func (tenant *Tenant) CreateCollection(ctx context.Context, tx transaction.Tx, database *Database, schFactory *schema.Factory) error { tenant.Lock() defer tenant.Unlock() @@ -632,10 +685,10 @@ func (tenant *Tenant) CreateCollection(ctx context.Context, tx transaction.Tx, d // shortcut to just check if schema is eq then return early return err } - return tenant.updateCollection(ctx, tx, database, c, schFactory, searchStore) + return tenant.updateCollection(ctx, tx, database, c, schFactory, tenant.searchStore) } - collectionId, err := tenant.encoder.EncodeCollectionName(ctx, tx, schFactory.Name, tenant.namespace.Id(), database.id) + collectionId, err := tenant.metaStore.CreateCollection(ctx, tx, schFactory.Name, tenant.namespace.Id(), database.id) if err != nil { return err } @@ -644,7 +697,7 @@ func (tenant *Tenant) CreateCollection(ctx context.Context, tx transaction.Tx, d indexes := schFactory.Indexes.GetIndexes() idxNameToId := make(map[string]uint32) for _, i := range indexes { - id, err := tenant.encoder.EncodeIndexName(ctx, tx, i.Name, tenant.namespace.Id(), database.id, collectionId) + id, err := tenant.metaStore.CreateIndex(ctx, tx, i.Name, tenant.namespace.Id(), database.id, collectionId) if err != nil { return err } @@ -663,7 +716,7 @@ func (tenant *Tenant) CreateCollection(ctx context.Context, tx transaction.Tx, d database.collections[schFactory.Name] = NewCollectionHolder(collectionId, schFactory.Name, collection, idxNameToId) if config.DefaultConfig.Search.WriteEnabled { - if err := searchStore.CreateCollection(ctx, collection.Search); err != nil { + if err := tenant.searchStore.CreateCollection(ctx, collection.Search); err != nil { if err != search.ErrDuplicateEntity { return err } @@ -683,7 +736,7 @@ func (tenant *Tenant) updateCollection(ctx context.Context, tx transaction.Tx, d for _, idx := range newIndexes { // these are the new indexes present in the new collection - id, err := tenant.encoder.EncodeIndexName(ctx, tx, idx.Name, tenant.namespace.Id(), database.id, c.id) + id, err := tenant.metaStore.CreateIndex(ctx, tx, idx.Name, tenant.namespace.Id(), database.id, c.id) if err != nil { return err } @@ -727,11 +780,11 @@ func (tenant *Tenant) updateCollection(ctx context.Context, tx transaction.Tx, d // DropCollection is to drop a collection and its associated indexes. It removes the "created" entry from the encoding // subspace and adds a "dropped" entry for the same collection key. -func (tenant *Tenant) DropCollection(ctx context.Context, tx transaction.Tx, db *Database, collectionName string, searchStore search.Store, rowKeyEncoder Encoder) error { +func (tenant *Tenant) DropCollection(ctx context.Context, tx transaction.Tx, db *Database, collectionName string) error { tenant.Lock() defer tenant.Unlock() - err := tenant.dropCollection(ctx, tx, db, collectionName, searchStore, rowKeyEncoder) + err := tenant.dropCollection(ctx, tx, db, collectionName) if err != nil { return err } @@ -743,7 +796,7 @@ func (tenant *Tenant) DropCollection(ctx context.Context, tx transaction.Tx, db return err } -func (tenant *Tenant) dropCollection(ctx context.Context, tx transaction.Tx, db *Database, collectionName string, searchStore search.Store, rowKeyEncoder Encoder) error { +func (tenant *Tenant) dropCollection(ctx context.Context, tx transaction.Tx, db *Database, collectionName string) error { if db == nil { return api.Errorf(api.Code_NOT_FOUND, "database missing") } @@ -753,12 +806,12 @@ func (tenant *Tenant) dropCollection(ctx context.Context, tx transaction.Tx, db return api.Errorf(api.Code_NOT_FOUND, "collection doesn't exists '%s'", collectionName) } - if err := tenant.encoder.EncodeCollectionAsDropped(ctx, tx, cHolder.name, tenant.namespace.Id(), db.id, cHolder.id); err != nil { + if err := tenant.metaStore.DropCollection(ctx, tx, cHolder.name, tenant.namespace.Id(), db.id, cHolder.id); err != nil { return err } for idxName, idxId := range cHolder.idxNameToId { - if err := tenant.encoder.EncodeIndexAsDropped(ctx, tx, idxName, tenant.namespace.Id(), db.id, cHolder.id, idxId); err != nil { + if err := tenant.metaStore.DropIndex(ctx, tx, idxName, tenant.namespace.Id(), db.id, cHolder.id, idxId); err != nil { return err } } @@ -766,8 +819,9 @@ func (tenant *Tenant) dropCollection(ctx context.Context, tx transaction.Tx, db return err } + // TODO: Move actual deletion out of the mutex if config.DefaultConfig.Server.FDBDelete { - tableName, err := rowKeyEncoder.EncodeTableName(tenant.namespace, db, cHolder.collection) + tableName, err := tenant.Encoder.EncodeTableName(tenant.namespace, db, cHolder.collection) if err != nil { return err } @@ -778,7 +832,7 @@ func (tenant *Tenant) dropCollection(ctx context.Context, tx transaction.Tx, db } if config.DefaultConfig.Search.WriteEnabled { - if err := searchStore.DropCollection(ctx, cHolder.collection.SearchCollectionName()); err != nil { + if err := tenant.searchStore.DropCollection(ctx, cHolder.collection.SearchCollectionName()); err != nil { if err != search.ErrNotFound { return err } @@ -796,6 +850,32 @@ func (tenant *Tenant) String() string { return fmt.Sprintf("id: %d, name: %s", tenant.namespace.Id(), tenant.namespace.Name()) } +// Size returns approximate data size on disk for all the collections in the namespace +func (tenant *Tenant) Size(ctx context.Context) (int64, error) { + tenant.Lock() + nsName, _ := tenant.Encoder.EncodeTableName(tenant.namespace, nil, nil) + tenant.Unlock() + + return tenant.kvStore.TableSize(ctx, nsName) +} + +func (tenant *Tenant) DatabaseSize(ctx context.Context, db *Database) (int64, error) { + tenant.Lock() + nsName, _ := tenant.Encoder.EncodeTableName(tenant.namespace, db, nil) + tenant.Unlock() + + return tenant.kvStore.TableSize(ctx, nsName) +} + +func (tenant *Tenant) CollectionSize(ctx context.Context, db *Database, coll *schema.DefaultCollection) (int64, error) { + tenant.Lock() + + nsName, _ := tenant.Encoder.EncodeTableName(tenant.namespace, db, coll) + tenant.Unlock() + + return tenant.kvStore.TableSize(ctx, nsName) +} + // Database is to manage the collections for this database. Check the Clone method before changing this struct. type Database struct { sync.RWMutex @@ -962,3 +1042,20 @@ func IsSchemaEq(s1, s2 []byte) (bool, error) { } return reflect.DeepEqual(j2, j), nil } + +// NewTestTenantMgr creates new TenantManager for tests +func NewTestTenantMgr(kvStore kv.KeyValueStore) (*TenantManager, context.Context, context.CancelFunc) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + + m := newTenantManager(kvStore, &search.NoopStore{}, &encoding.TestMDNameRegistry{ + ReserveSB: fmt.Sprintf("test_tenant_reserve_%x", rand.Uint64()), + EncodingSB: fmt.Sprintf("test_tenant_encoding_%x", rand.Uint64()), + SchemaSB: fmt.Sprintf("test_tenant_schema_%x", rand.Uint64()), + }) + + _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) + _ = kvStore.DropTable(ctx, m.mdNameRegistry.EncodingSubspaceName()) + _ = kvStore.DropTable(ctx, m.mdNameRegistry.SchemaSubspaceName()) + + return m, ctx, cancel +} diff --git a/server/metadata/tenant_test.go b/server/metadata/tenant_test.go index b69269bcb..95a154036 100644 --- a/server/metadata/tenant_test.go +++ b/server/metadata/tenant_test.go @@ -16,41 +16,30 @@ package metadata import ( "context" + "fmt" "os" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" api "github.com/tigrisdata/tigris/api/server/v1" + "github.com/tigrisdata/tigris/internal" "github.com/tigrisdata/tigris/schema" "github.com/tigrisdata/tigris/server/config" - "github.com/tigrisdata/tigris/server/metadata/encoding" "github.com/tigrisdata/tigris/server/transaction" "github.com/tigrisdata/tigris/store/kv" - "github.com/tigrisdata/tigris/store/search" ulog "github.com/tigrisdata/tigris/util/log" ) -func TestTenantManager_CreateOrGetTenant(t *testing.T) { - fdbCfg, err := config.GetTestFDBConfig("../..") - require.NoError(t, err) - - kvStore, err := kv.NewKeyValueStore(fdbCfg) - require.NoError(t, err) +var kvStore kv.KeyValueStore +func TestTenantManager_CreateOrGetTenant(t *testing.T) { tm := transaction.NewManager(kvStore) t.Run("create_tenant", func(t *testing.T) { - m := newTenantManager(kvStore, &encoding.TestMDNameRegistry{ - ReserveSB: "test_tenant_reserve", - EncodingSB: "test_tenant_encoding", - SchemaSB: "test_tenant_schema", - }) - - ctx := context.TODO() - _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) - _ = kvStore.DropTable(ctx, m.mdNameRegistry.EncodingSubspaceName()) - _ = kvStore.DropTable(ctx, m.mdNameRegistry.SchemaSubspaceName()) + m, ctx, cancel := NewTestTenantMgr(kvStore) + defer cancel() - _, err = m.CreateOrGetTenant(ctx, tm, &TenantNamespace{"ns-test1", 2}) + _, err := m.CreateOrGetTenant(ctx, tm, &TenantNamespace{"ns-test1", 2}) require.NoError(t, err) tenant := m.tenants["ns-test1"] @@ -62,16 +51,10 @@ func TestTenantManager_CreateOrGetTenant(t *testing.T) { }) t.Run("create_multiple_tenants", func(t *testing.T) { - m := newTenantManager(kvStore, &encoding.TestMDNameRegistry{ - ReserveSB: "test_tenant_reserve", - EncodingSB: "test_tenant_encoding", - SchemaSB: "test_tenant_schema", - }) + m, ctx, cancel := NewTestTenantMgr(kvStore) + defer cancel() - ctx := context.TODO() - _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) - - _, err = m.CreateOrGetTenant(ctx, tm, &TenantNamespace{"ns-test1", 2}) + _, err := m.CreateOrGetTenant(ctx, tm, &TenantNamespace{"ns-test1", 2}) require.NoError(t, err) _, err = m.CreateOrGetTenant(ctx, tm, &TenantNamespace{"ns-test2", 3}) @@ -94,35 +77,23 @@ func TestTenantManager_CreateOrGetTenant(t *testing.T) { _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) }) t.Run("create_duplicate_tenant_error", func(t *testing.T) { - m := newTenantManager(kvStore, &encoding.TestMDNameRegistry{ - ReserveSB: "test_tenant_reserve", - EncodingSB: "test_tenant_encoding", - SchemaSB: "test_tenant_schema", - }) + m, ctx, cancel := NewTestTenantMgr(kvStore) + defer cancel() - ctx := context.TODO() - _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) - - _, err = m.CreateOrGetTenant(ctx, tm, &TenantNamespace{"ns-test1", 2}) + _, err := m.CreateOrGetTenant(ctx, tm, &TenantNamespace{"ns-test1", 2}) require.NoError(t, err) // should fail now - _, err = m.CreateOrGetTenant(context.TODO(), tm, &TenantNamespace{"ns-test1", 3}) + _, err = m.CreateOrGetTenant(ctx, tm, &TenantNamespace{"ns-test1", 3}) require.Equal(t, "id is already assigned to 'ns-test1'", err.(*api.TigrisError).Error()) _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) }) t.Run("create_duplicate_tenant_id_error", func(t *testing.T) { - m := newTenantManager(kvStore, &encoding.TestMDNameRegistry{ - ReserveSB: "test_tenant_reserve", - EncodingSB: "test_tenant_encoding", - SchemaSB: "test_tenant_schema", - }) + m, ctx, cancel := NewTestTenantMgr(kvStore) + defer cancel() - ctx := context.TODO() - _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) - - _, err = m.CreateOrGetTenant(ctx, tm, &TenantNamespace{"ns-test1", 2}) + _, err := m.CreateOrGetTenant(ctx, tm, &TenantNamespace{"ns-test1", 2}) require.NoError(t, err) require.Equal(t, 1, len(m.idToTenantMap)) @@ -137,53 +108,34 @@ func TestTenantManager_CreateOrGetTenant(t *testing.T) { } func TestTenantManager_CreateTenant(t *testing.T) { - fdbCfg, err := config.GetTestFDBConfig("../..") - require.NoError(t, err) - - kvStore, err := kv.NewKeyValueStore(fdbCfg) - require.NoError(t, err) - tm := transaction.NewManager(kvStore) t.Run("create_tenant", func(t *testing.T) { - m := newTenantManager(kvStore, &encoding.TestMDNameRegistry{ - ReserveSB: "test_tenant_reserve", - EncodingSB: "test_tenant_encoding", - SchemaSB: "test_tenant_schema", - }) + m, ctx, cancel := NewTestTenantMgr(kvStore) + defer cancel() - ctx := context.TODO() - _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) - _ = kvStore.DropTable(ctx, m.mdNameRegistry.EncodingSubspaceName()) - _ = kvStore.DropTable(ctx, m.mdNameRegistry.SchemaSubspaceName()) tx, e := tm.StartTx(ctx) require.NoError(t, e) _, err := m.CreateTenant(ctx, tx, &TenantNamespace{"ns-test1", 2}) require.NoError(t, err) - namespaces, err := m.encoder.GetNamespaces(ctx, tx) + namespaces, err := m.metaStore.GetNamespaces(ctx, tx) require.NoError(t, err) id := namespaces["ns-test1"] require.Equal(t, uint32(2), id) _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) }) t.Run("create_multiple_tenants", func(t *testing.T) { - m := newTenantManager(kvStore, &encoding.TestMDNameRegistry{ - ReserveSB: "test_tenant_reserve", - EncodingSB: "test_tenant_encoding", - SchemaSB: "test_tenant_schema", - }) - - ctx := context.TODO() - _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) + m, ctx, cancel := NewTestTenantMgr(kvStore) + defer cancel() tx, e := tm.StartTx(ctx) require.NoError(t, e) - _, err = m.CreateTenant(ctx, tx, &TenantNamespace{"ns-test1", 2}) + _, err := m.CreateTenant(ctx, tx, &TenantNamespace{"ns-test1", 2}) require.NoError(t, err) _, err = m.CreateTenant(ctx, tx, &TenantNamespace{"ns-test2", 3}) require.NoError(t, err) - namespaces, err := m.encoder.GetNamespaces(ctx, tx) + namespaces, err := m.metaStore.GetNamespaces(ctx, tx) require.NoError(t, err) id := namespaces["ns-test1"] @@ -194,37 +146,27 @@ func TestTenantManager_CreateTenant(t *testing.T) { _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) }) t.Run("create_duplicate_tenant_error", func(t *testing.T) { - m := newTenantManager(kvStore, &encoding.TestMDNameRegistry{ - ReserveSB: "test_tenant_reserve", - EncodingSB: "test_tenant_encoding", - SchemaSB: "test_tenant_schema", - }) + m, ctx, cancel := NewTestTenantMgr(kvStore) + defer cancel() - ctx := context.TODO() - _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) tx, e := tm.StartTx(ctx) require.NoError(t, e) - _, err = m.CreateTenant(ctx, tx, &TenantNamespace{"ns-test1", 2}) + _, err := m.CreateTenant(ctx, tx, &TenantNamespace{"ns-test1", 2}) require.NoError(t, err) // should fail now - _, err = m.CreateTenant(context.TODO(), tx, &TenantNamespace{"ns-test1", 3}) + _, err = m.CreateTenant(ctx, tx, &TenantNamespace{"ns-test1", 3}) require.Equal(t, "namespace with same name already exists with id '2'", err.(*api.TigrisError).Error()) _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) }) t.Run("create_duplicate_tenant_id_error", func(t *testing.T) { - m := newTenantManager(kvStore, &encoding.TestMDNameRegistry{ - ReserveSB: "test_tenant_reserve", - EncodingSB: "test_tenant_encoding", - SchemaSB: "test_tenant_schema", - }) + m, ctx, cancel := NewTestTenantMgr(kvStore) + defer cancel() - ctx := context.TODO() - _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) tx, e := tm.StartTx(ctx) require.NoError(t, e) - _, err = m.CreateTenant(ctx, tx, &TenantNamespace{"ns-test1", 2}) + _, err := m.CreateTenant(ctx, tx, &TenantNamespace{"ns-test1", 2}) require.NoError(t, err) // should fail now @@ -235,29 +177,16 @@ func TestTenantManager_CreateTenant(t *testing.T) { } func TestTenantManager_CreateDatabases(t *testing.T) { - fdbCfg, err := config.GetTestFDBConfig("../..") - require.NoError(t, err) - - kvStore, err := kv.NewKeyValueStore(fdbCfg) - require.NoError(t, err) - tm := transaction.NewManager(kvStore) t.Run("create_databases", func(t *testing.T) { - m := newTenantManager(kvStore, &encoding.TestMDNameRegistry{ - ReserveSB: "test_tenant_reserve", - EncodingSB: "test_tenant_encoding", - SchemaSB: "test_tenant_schema", - }) - - ctx := context.TODO() - _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) - _ = kvStore.DropTable(ctx, m.mdNameRegistry.EncodingSubspaceName()) + m, ctx, cancel := NewTestTenantMgr(kvStore) + defer cancel() - _, err = m.CreateOrGetTenant(context.TODO(), tm, &TenantNamespace{"ns-test1", 2}) + _, err := m.CreateOrGetTenant(ctx, tm, &TenantNamespace{"ns-test1", 2}) require.NoError(t, err) tenant := m.tenants["ns-test1"] - tx, err := tm.StartTx(context.TODO()) + tx, err := tm.StartTx(ctx) require.NoError(t, err) _, err = tenant.CreateDatabase(ctx, tx, "tenant_db1") require.NoError(t, err) @@ -273,7 +202,7 @@ func TestTenantManager_CreateDatabases(t *testing.T) { db2, err := tenant.GetDatabase(ctx, tx, "tenant_db2") require.NoError(t, err) require.Equal(t, "tenant_db2", db2.name) - require.NoError(t, tx.Commit(context.TODO())) + require.NoError(t, tx.Commit(ctx)) require.Equal(t, "tenant_db2", tenant.idToDatabaseMap[db2.id]) _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) @@ -282,30 +211,16 @@ func TestTenantManager_CreateDatabases(t *testing.T) { } func TestTenantManager_CreateCollections(t *testing.T) { - fdbCfg, err := config.GetTestFDBConfig("../..") - require.NoError(t, err) - - kvStore, err := kv.NewKeyValueStore(fdbCfg) - require.NoError(t, err) - tm := transaction.NewManager(kvStore) t.Run("create_collections", func(t *testing.T) { - m := newTenantManager(kvStore, &encoding.TestMDNameRegistry{ - ReserveSB: "test_tenant_reserve", - EncodingSB: "test_tenant_encoding", - SchemaSB: "test_tenant_schema", - }) + m, ctx, cancel := NewTestTenantMgr(kvStore) + defer cancel() - ctx := context.TODO() - _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) - _ = kvStore.DropTable(ctx, m.mdNameRegistry.EncodingSubspaceName()) - _ = kvStore.DropTable(ctx, m.mdNameRegistry.SchemaSubspaceName()) - - _, err = m.CreateOrGetTenant(context.TODO(), tm, &TenantNamespace{"ns-test1", 2}) + _, err := m.CreateOrGetTenant(ctx, tm, &TenantNamespace{"ns-test1", 2}) require.NoError(t, err) tenant := m.tenants["ns-test1"] - tx, err := tm.StartTx(context.TODO()) + tx, err := tm.StartTx(ctx) require.NoError(t, err) _, err = tenant.CreateDatabase(ctx, tx, "tenant_db1") require.NoError(t, err) @@ -345,7 +260,7 @@ func TestTenantManager_CreateCollections(t *testing.T) { factory, err := schema.Build("test_collection", jsSchema) require.NoError(t, err) - require.NoError(t, tenant.CreateCollection(context.TODO(), tx, db2, factory, &search.NoopStore{})) + require.NoError(t, tenant.CreateCollection(ctx, tx, db2, factory)) require.NoError(t, tenant.reload(ctx, tx, nil)) @@ -356,7 +271,7 @@ func TestTenantManager_CreateCollections(t *testing.T) { require.Equal(t, "test_collection", db2.idToCollectionMap[collection.Id]) require.Equal(t, 1, len(db2.idToCollectionMap)) - require.NoError(t, tx.Commit(context.TODO())) + require.NoError(t, tx.Commit(ctx)) _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) _ = kvStore.DropTable(ctx, m.mdNameRegistry.EncodingSubspaceName()) @@ -365,26 +280,12 @@ func TestTenantManager_CreateCollections(t *testing.T) { } func TestTenantManager_DropCollection(t *testing.T) { - fdbCfg, err := config.GetTestFDBConfig("../..") - require.NoError(t, err) - - kvStore, err := kv.NewKeyValueStore(fdbCfg) - require.NoError(t, err) - tm := transaction.NewManager(kvStore) t.Run("drop_collection", func(t *testing.T) { - m := newTenantManager(kvStore, &encoding.TestMDNameRegistry{ - ReserveSB: "test_tenant_reserve", - EncodingSB: "test_tenant_encoding", - SchemaSB: "test_tenant_schema", - }) + m, ctx, cancel := NewTestTenantMgr(kvStore) + defer cancel() - ctx := context.TODO() - _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) - _ = kvStore.DropTable(ctx, m.mdNameRegistry.EncodingSubspaceName()) - _ = kvStore.DropTable(ctx, m.mdNameRegistry.SchemaSubspaceName()) - - _, err = m.CreateOrGetTenant(ctx, tm, &TenantNamespace{"ns-test1", 2}) + _, err := m.CreateOrGetTenant(ctx, tm, &TenantNamespace{"ns-test1", 2}) require.NoError(t, err) tenant := m.tenants["ns-test1"] @@ -421,7 +322,7 @@ func TestTenantManager_DropCollection(t *testing.T) { factory, err := schema.Build("test_collection", jsSchema) require.NoError(t, err) - require.NoError(t, tenant.CreateCollection(ctx, tx, db2, factory, &search.NoopStore{})) + require.NoError(t, tenant.CreateCollection(ctx, tx, db2, factory)) require.NoError(t, tenant.reload(ctx, tx, nil)) require.NoError(t, tx.Commit(ctx)) @@ -433,7 +334,7 @@ func TestTenantManager_DropCollection(t *testing.T) { tx, err = tm.StartTx(ctx) require.NoError(t, err) - require.NoError(t, tenant.DropCollection(ctx, tx, db2, "test_collection", &search.NoopStore{}, NewEncoder(m))) + require.NoError(t, tenant.DropCollection(ctx, tx, db2, "test_collection")) require.NoError(t, tx.Commit(ctx)) _, err = tm.StartTx(ctx) @@ -448,7 +349,184 @@ func TestTenantManager_DropCollection(t *testing.T) { }) } +func TestTenantManager_DataSize(t *testing.T) { + tm := transaction.NewManager(kvStore) + m, ctx, cancel := NewTestTenantMgr(kvStore) + defer cancel() + + _, err := m.CreateOrGetTenant(context.TODO(), tm, &TenantNamespace{"ns-test1", 2}) + require.NoError(t, err) + + _, err = m.CreateOrGetTenant(context.TODO(), tm, &TenantNamespace{"ns-test2", 3}) + require.NoError(t, err) + + tenant := m.tenants["ns-test1"] + tx, err := tm.StartTx(context.TODO()) + require.NoError(t, err) + + _, err = tenant.CreateDatabase(ctx, tx, "tenant_db1") + require.NoError(t, err) + _, err = tenant.CreateDatabase(ctx, tx, "tenant_db2") + require.NoError(t, err) + + tenant2 := m.tenants["ns-test2"] + + _, err = tenant2.CreateDatabase(ctx, tx, "tenant_db1") + require.NoError(t, err) + _, err = tenant2.CreateDatabase(ctx, tx, "tenant_db2") + require.NoError(t, err) + + require.NoError(t, tenant.reload(ctx, tx, nil)) + require.NoError(t, tenant2.reload(ctx, tx, nil)) + + jsSchema := []byte(`{ + "title": "test_collection", + "properties": { + "K1": { + "type": "string" + }, + "K2": { + "type": "integer" + }, + "D1": { + "type": "string", + "maxLength": 128 + } + }, + "primary_key": ["K1", "K2"] + }`) + + factory, err := schema.Build("test_collection", jsSchema) + require.NoError(t, err) + + db1, err := tenant.GetDatabase(ctx, tx, "tenant_db1") + require.NoError(t, err) + db2, err := tenant.GetDatabase(ctx, tx, "tenant_db2") + require.NoError(t, err) + + require.NoError(t, tenant.CreateCollection(ctx, tx, db1, factory)) + require.NoError(t, err) + require.NoError(t, tenant.CreateCollection(ctx, tx, db2, factory)) + require.NoError(t, err) + + // create tenant2 dbs and collections + db21, err := tenant2.GetDatabase(ctx, tx, "tenant_db1") + require.NoError(t, err) + db22, err := tenant2.GetDatabase(ctx, tx, "tenant_db2") + require.NoError(t, err) + + require.NoError(t, tenant2.CreateCollection(ctx, tx, db21, factory)) + require.NoError(t, err) + require.NoError(t, tenant2.CreateCollection(ctx, tx, db22, factory)) + require.NoError(t, err) + + require.NoError(t, tenant.reload(ctx, tx, nil)) + require.NoError(t, tenant2.reload(ctx, tx, nil)) + + require.NoError(t, tx.Commit(context.TODO())) + + coll1 := db2.GetCollection("test_collection") + docSize := 10 * 1024 + table, err := m.encoder.EncodeTableName(tenant.GetNamespace(), db1, coll1) + require.NoError(t, err) + + err = tenant.kvStore.DropTable(ctx, table) + require.NoError(t, err) + + for i := 0; i < 100; i++ { + err = tenant.kvStore.Insert(ctx, table, kv.BuildKey(fmt.Sprintf("aaa%d", i)), &internal.TableData{RawData: make([]byte, docSize)}) + require.NoError(t, err) + } + + coll21 := db21.GetCollection("test_collection") + table21, err := m.encoder.EncodeTableName(tenant2.GetNamespace(), db21, coll21) + require.NoError(t, err) + + err = tenant2.kvStore.DropTable(ctx, table21) + require.NoError(t, err) + + for i := 0; i < 150; i++ { + err = tenant2.kvStore.Insert(ctx, table21, kv.BuildKey(fmt.Sprintf("aaa%d", i)), &internal.TableData{RawData: make([]byte, docSize)}) + require.NoError(t, err) + } + + coll22 := db22.GetCollection("test_collection") + table22, err := m.encoder.EncodeTableName(tenant2.GetNamespace(), db22, coll22) + require.NoError(t, err) + + err = tenant2.kvStore.DropTable(ctx, table22) + require.NoError(t, err) + + for i := 0; i < 50; i++ { + err = tenant2.kvStore.Insert(ctx, table22, kv.BuildKey(fmt.Sprintf("aaa%d", i)), &internal.TableData{RawData: make([]byte, docSize)}) + require.NoError(t, err) + } + + sz, err := tenant.Size(ctx) + require.NoError(t, err) + assert.Equal(t, int64(983500), sz) + + sz, err = tenant.DatabaseSize(ctx, db1) + require.NoError(t, err) + assert.Equal(t, int64(983500), sz) + + sz, err = tenant.CollectionSize(ctx, db1, coll1) + require.NoError(t, err) + assert.Equal(t, int64(983500), sz) + + // db2 is empty + sz, err = tenant.DatabaseSize(ctx, db2) + require.NoError(t, err) + assert.Equal(t, int64(0), sz) + + // Tenant 2 + // db21 + sz, err = tenant2.Size(ctx) + require.NoError(t, err) + assert.Equal(t, int64(2186250), sz) // sum of db21 and db22 + + sz, err = tenant2.DatabaseSize(ctx, db21) + require.NoError(t, err) + assert.Equal(t, int64(1694750), sz) + + sz, err = tenant2.CollectionSize(ctx, db21, coll21) + require.NoError(t, err) + assert.Equal(t, int64(1694750), sz) + + // db22 + sz, err = tenant2.DatabaseSize(ctx, db22) + require.NoError(t, err) + assert.Equal(t, int64(491500), sz) + + sz, err = tenant2.CollectionSize(ctx, db22, coll22) + require.NoError(t, err) + assert.Equal(t, int64(491500), sz) + + // cleanup + err = tenant2.kvStore.DropTable(ctx, table) + require.NoError(t, err) + err = tenant2.kvStore.DropTable(ctx, table21) + require.NoError(t, err) + err = tenant2.kvStore.DropTable(ctx, table22) + require.NoError(t, err) + + _ = kvStore.DropTable(ctx, m.mdNameRegistry.ReservedSubspaceName()) + _ = kvStore.DropTable(ctx, m.mdNameRegistry.EncodingSubspaceName()) + _ = kvStore.DropTable(ctx, m.mdNameRegistry.SchemaSubspaceName()) +} + func TestMain(m *testing.M) { ulog.Configure(ulog.LogConfig{Level: "disabled"}) + + fdbCfg, err := config.GetTestFDBConfig("../..") + if err != nil { + panic(fmt.Sprintf("failed to init FDB config: %v", err)) + } + + kvStore, err = kv.NewKeyValueStore(fdbCfg) + if err != nil { + panic(fmt.Sprintf("failed to init FDB KV %v", err)) + } + os.Exit(m.Run()) } diff --git a/server/metrics/requests.go b/server/metrics/requests.go index 073e1a26c..9cc48aa04 100644 --- a/server/metrics/requests.go +++ b/server/metrics/requests.go @@ -15,68 +15,93 @@ package metrics import ( + "context" "fmt" "strings" "github.com/tigrisdata/tigris/server/config" + "github.com/tigrisdata/tigris/server/request" "google.golang.org/grpc" ) const ( - AdminServiceName = "tigrisdata.admin.v1.Admin" - SystemTigrisTenantName = "system" + AdminServiceName = "tigrisdata.admin.v1.Admin" + SystemTigrisTenantName = "system" + DefaultReportedTigrisTenant = "unknown" ) type RequestEndpointMetadata struct { - serviceName string - methodInfo grpc.MethodInfo + serviceName string + methodInfo grpc.MethodInfo + namespaceName string } -func newRequestEndpointMetadata(serviceName string, methodInfo grpc.MethodInfo) RequestEndpointMetadata { - return RequestEndpointMetadata{serviceName: serviceName, methodInfo: methodInfo} +func getNamespaceName(ctx context.Context) string { + if namespace, _ := request.GetNamespace(ctx); namespace != "" { + return namespace + } + return DefaultReportedTigrisTenant +} + +func newRequestEndpointMetadata(ctx context.Context, serviceName string, methodInfo grpc.MethodInfo) RequestEndpointMetadata { + return RequestEndpointMetadata{serviceName: serviceName, methodInfo: methodInfo, namespaceName: getNamespaceName(ctx)} } -func (g *RequestEndpointMetadata) GetPreInitializedTags() map[string]string { - if g.serviceName == AdminServiceName { +func (r *RequestEndpointMetadata) GetMethodName() string { + return strings.Split(r.methodInfo.Name, "/")[2] +} + +func (r *RequestEndpointMetadata) GetServiceType() string { + if r.methodInfo.IsServerStream { + return "stream" + } else { + return "unary" + } +} + +func (r *RequestEndpointMetadata) GetTags() map[string]string { + if r.serviceName == AdminServiceName { return map[string]string{ - "method": g.methodInfo.Name, - "grpc_service": g.serviceName, - "tigris_tenant": SystemTigrisTenantName, + "grpc_method": r.methodInfo.Name, + "grpc_service": r.serviceName, + "tigris_tenant": r.namespaceName, + "grpc_service_type": r.GetServiceType(), } } else { return map[string]string{ - "method": g.methodInfo.Name, - "grpc_service": g.serviceName, - "tigris_tenant": DefaultReportedTigrisTenant, + "grpc_method": r.methodInfo.Name, + "grpc_service": r.serviceName, + "tigris_tenant": r.namespaceName, + "grpc_service_type": r.GetServiceType(), } } } -func (g *RequestEndpointMetadata) GetSpecificErrorTags(source string, code string) map[string]string { - if g.serviceName == AdminServiceName { +func (r *RequestEndpointMetadata) GetSpecificErrorTags(source string, code string) map[string]string { + if r.serviceName == AdminServiceName { return map[string]string{ - "method": g.methodInfo.Name, - "grpc_service": g.serviceName, - "source": source, - "code": code, + "grpc_method": r.methodInfo.Name, + "grpc_service": r.serviceName, + "error_source": source, + "error_code": code, "tigris_tenant": SystemTigrisTenantName, } } else { return map[string]string{ - "method": g.methodInfo.Name, - "grpc_service": g.serviceName, - "source": source, - "code": code, + "grpc_method": r.methodInfo.Name, + "grpc_service": r.serviceName, + "error_source": source, + "error_code": code, "tigris_tenant": DefaultReportedTigrisTenant, } } } -func (g *RequestEndpointMetadata) getFullMethod() string { - return fmt.Sprintf("/%s/%s", g.serviceName, g.methodInfo.Name) +func (r *RequestEndpointMetadata) getFullMethod() string { + return fmt.Sprintf("/%s/%s", r.serviceName, r.methodInfo.Name) } -func GetGrpcEndPointMetadataFromFullMethod(fullMethod string, methodType string) RequestEndpointMetadata { +func GetGrpcEndPointMetadataFromFullMethod(ctx context.Context, fullMethod string, methodType string) RequestEndpointMetadata { var methodInfo grpc.MethodInfo methodList := strings.Split(fullMethod, "/") svcName := methodList[1] @@ -94,12 +119,7 @@ func GetGrpcEndPointMetadataFromFullMethod(fullMethod string, methodType string) IsServerStream: true, } } - return newRequestEndpointMetadata(svcName, methodInfo) -} - -func GetPreinitializedTagsFromFullMethod(fullMethod string, methodType string) map[string]string { - metaData := GetGrpcEndPointMetadataFromFullMethod(fullMethod, methodType) - return metaData.GetPreInitializedTags() + return newRequestEndpointMetadata(ctx, svcName, methodInfo) } func InitializeRequestScopes() { diff --git a/server/metrics/requests_test.go b/server/metrics/requests_test.go index e79b79562..b8fbbc595 100644 --- a/server/metrics/requests_test.go +++ b/server/metrics/requests_test.go @@ -15,6 +15,7 @@ package metrics import ( + "context" "fmt" "testing" @@ -44,29 +45,33 @@ func TestGRPCMetrics(t *testing.T) { IsClientStream: false, } - unaryEndPointMetadata := newRequestEndpointMetadata(svcName, unaryMethodInfo) - streamingEndpointMetadata := newRequestEndpointMetadata(svcName, streamingMethodInfo) + ctx := context.Background() + + unaryEndPointMetadata := newRequestEndpointMetadata(ctx, svcName, unaryMethodInfo) + streamingEndpointMetadata := newRequestEndpointMetadata(ctx, svcName, streamingMethodInfo) t.Run("Test GetGrpcEndPointMetadataFromFullMethod", func(t *testing.T) { - unaryMetadata := GetGrpcEndPointMetadataFromFullMethod(unaryEndPointMetadata.getFullMethod(), "unary") + unaryMetadata := GetGrpcEndPointMetadataFromFullMethod(ctx, unaryEndPointMetadata.getFullMethod(), "unary") assert.Equal(t, unaryMetadata, unaryEndPointMetadata) - streamMetadata := GetGrpcEndPointMetadataFromFullMethod(streamingEndpointMetadata.getFullMethod(), "stream") + streamMetadata := GetGrpcEndPointMetadataFromFullMethod(ctx, streamingEndpointMetadata.getFullMethod(), "stream") assert.Equal(t, streamMetadata, streamingEndpointMetadata) }) t.Run("Test GetPreinitializedTagsFromFullMethod", func(t *testing.T) { - unaryTags := unaryEndPointMetadata.GetPreInitializedTags() + unaryTags := unaryEndPointMetadata.GetTags() assert.Equal(t, unaryTags, map[string]string{ - "method": unaryMethodInfo.Name, - "grpc_service": "tigrisdata.v1.Tigris", - "tigris_tenant": DefaultReportedTigrisTenant, + "grpc_method": unaryMethodInfo.Name, + "grpc_service": "tigrisdata.v1.Tigris", + "grpc_service_type": "unary", + "tigris_tenant": DefaultReportedTigrisTenant, }) - streamTags := streamingEndpointMetadata.GetPreInitializedTags() + streamTags := streamingEndpointMetadata.GetTags() assert.Equal(t, streamTags, map[string]string{ - "method": streamingMethodInfo.Name, - "grpc_service": "tigrisdata.v1.Tigris", - "tigris_tenant": DefaultReportedTigrisTenant, + "grpc_method": streamingMethodInfo.Name, + "grpc_service": "tigrisdata.v1.Tigris", + "grpc_service_type": "stream", + "tigris_tenant": DefaultReportedTigrisTenant, }) }) @@ -88,10 +93,10 @@ func TestGRPCMetrics(t *testing.T) { }) t.Run("Test unary method preinitialized tags", func(t *testing.T) { - tags := unaryEndPointMetadata.GetPreInitializedTags() + tags := unaryEndPointMetadata.GetTags() for tagName, tagValue := range tags { switch tagName { - case "method": + case "grpc_method": assert.Equal(t, tagValue, "TestUnaryMethod") case "grpc_service": assert.Equal(t, tagValue, "tigrisdata.v1.Tigris") @@ -100,10 +105,10 @@ func TestGRPCMetrics(t *testing.T) { }) t.Run("Test streaming method preinitialized tags", func(t *testing.T) { - tags := streamingEndpointMetadata.GetPreInitializedTags() + tags := streamingEndpointMetadata.GetTags() for tagName, tagValue := range tags { switch tagName { - case "method": + case "grpc_method": assert.Equal(t, tagValue, "TestStreamingMethod") case "grpc_service": assert.Equal(t, tagValue, "tigrisdata.v1.Tigris") diff --git a/server/metrics/tenant.go b/server/metrics/tenant.go index 0a0da0eb0..609fb721d 100644 --- a/server/metrics/tenant.go +++ b/server/metrics/tenant.go @@ -16,14 +16,11 @@ package metrics import ( "context" + "github.com/tigrisdata/tigris/server/request" ulog "github.com/tigrisdata/tigris/util/log" ) -const ( - DefaultReportedTigrisTenant string = "unknown" -) - func addTigrisTenantToTags(ctx context.Context, tags map[string]string) map[string]string { namespace, err := request.GetNamespace(ctx) if ulog.E(err) { diff --git a/server/metrics/tracing.go b/server/metrics/tracing.go new file mode 100644 index 000000000..203339205 --- /dev/null +++ b/server/metrics/tracing.go @@ -0,0 +1,72 @@ +// Copyright 2022 Tigris Data, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "context" + + "github.com/tigrisdata/tigris/server/config" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" +) + +const ( + GrpcTracingServiceName = "tigris.grpc" + KvTracingServiceName = "tigris.fdb.kv" + TxManagerTracingServiceName = "tigris.tx.manager" +) + +type SpanMeta struct { + serviceName string + resourceName string + spanType string + tags map[string]string +} + +func NewSpanMeta(serviceName string, resourceName string, spanType string, tags map[string]string) *SpanMeta { + return &SpanMeta{serviceName: serviceName, resourceName: resourceName, spanType: spanType, tags: tags} +} + +func (s *SpanMeta) GetSpanOptions() []tracer.StartSpanOption { + return []tracer.StartSpanOption{ + tracer.ServiceName(s.serviceName), + tracer.ResourceName(s.resourceName), + tracer.SpanType(s.spanType), + tracer.Measured(), + } +} + +func (s *SpanMeta) StartTracing(ctx context.Context, childOnly bool) (context.Context, func()) { + if !config.DefaultConfig.Tracing.Enabled { + return ctx, func() {} + } + spanOpts := s.GetSpanOptions() + parentSpan, exists := tracer.SpanFromContext(ctx) + if exists { + // This is a child span, parents need to be marked + spanOpts = append(spanOpts, tracer.ChildOf(parentSpan.Context())) + } + if childOnly && !exists { + // There is no parent span, no need to start tracing here + return ctx, func() {} + } + span := tracer.StartSpan(s.resourceName, spanOpts...) + for k, v := range s.tags { + span.SetTag(k, v) + } + ctx = tracer.ContextWithSpan(ctx, span) + return ctx, func() { + span.Finish() + } +} diff --git a/server/midddleware/metrics.go b/server/midddleware/metrics.go index 5b8a3f86e..34df1c682 100644 --- a/server/midddleware/metrics.go +++ b/server/midddleware/metrics.go @@ -19,13 +19,10 @@ import ( "errors" "strconv" - "github.com/tigrisdata/tigris/server/request" - ulog "github.com/tigrisdata/tigris/util/log" - "github.com/uber-go/tally" - "github.com/apple/foundationdb/bindings/go/src/fdb" api "github.com/tigrisdata/tigris/api/server/v1" "github.com/tigrisdata/tigris/server/metrics" + "github.com/uber-go/tally" "google.golang.org/grpc" ) @@ -33,53 +30,26 @@ import ( // loaded if the metrics are not enabled. It is more efficient this way, but if we want dynamic reloading, // we may want to change this in the future. -func getOkMethodTags(ctx context.Context, methodName string, methodType string) map[string]string { - tags := metrics.GetPreinitializedTagsFromFullMethod(methodName, methodType) - namespace, err := request.GetNamespace(ctx) - if ulog.E(err) && err == request.ErrNamespaceNotFound { - tags["tigris_tenant"] = metrics.DefaultReportedTigrisTenant - } else { - tags["tigris_tenant"] = namespace - } - return tags -} - -func getErrorMethodTags(ctx context.Context, fullMethod string, methodType string, errSource string, errCode string) map[string]string { - metaData := metrics.GetGrpcEndPointMetadataFromFullMethod(fullMethod, methodType) - tags := metaData.GetSpecificErrorTags(errSource, errCode) - namespace, err := request.GetNamespace(ctx) - if ulog.E(err) && err == request.ErrNamespaceNotFound { - tags["tigris_tenant"] = metrics.DefaultReportedTigrisTenant - } else { - tags["tigris_tenant"] = namespace - } - return tags -} - -func countOkMessage(ctx context.Context, fullMethod string, methodType string) { +func countOkMessage(grpcMeta metrics.RequestEndpointMetadata) { // tigris_server_requests_ok - tags := getOkMethodTags(ctx, fullMethod, methodType) - metrics.Requests.Tagged(tags).Counter("ok").Inc(1) + metrics.Requests.Tagged(grpcMeta.GetTags()).Counter("ok").Inc(1) } -func countUnknownErrorMessage(ctx context.Context, fullMethod string, methodType string) { +func countUnknownErrorMessage(grpcMeta metrics.RequestEndpointMetadata) { // tigris_server_requests_error - tags := getOkMethodTags(ctx, fullMethod, methodType) - metrics.ErrorRequests.Tagged(tags).Counter("unknown").Inc(1) + metrics.ErrorRequests.Tagged(grpcMeta.GetTags()).Counter("unknown").Inc(1) } -func countSpecificErrorMessage(ctx context.Context, fullMethod string, methodType, errSource string, errCode string) { +func countSpecificErrorMessage(grpcMeta metrics.RequestEndpointMetadata, errSource string, errCode string) { // tigris_server_requests_error // For specific errors the tags are not pre-initialized because it has the error code in it - tags := getErrorMethodTags(ctx, fullMethod, methodType, errSource, errCode) - metrics.ErrorRequests.Tagged(tags).Counter("specific").Inc(1) + metrics.ErrorRequests.Tagged(grpcMeta.GetSpecificErrorTags(errSource, errCode)).Counter("specific").Inc(1) } func metricsUnaryServerInterceptorResponseTime() func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { - methodType := "unary" - tags := getOkMethodTags(ctx, info.FullMethod, methodType) - defer metrics.RequestsRespTime.Tagged(tags).Histogram("histogram", tally.DefaultBuckets).Start().Stop() + grpcMeta := metrics.GetGrpcEndPointMetadataFromFullMethod(ctx, info.FullMethod, "unary") + defer metrics.RequestsRespTime.Tagged(grpcMeta.GetTags()).Histogram("histogram", tally.DefaultBuckets).Start().Stop() resp, err := handler(ctx, req) return resp, err } @@ -88,19 +58,19 @@ func metricsUnaryServerInterceptorResponseTime() func(ctx context.Context, req i func metricsUnaryServerInterceptorCounters() func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { resp, err := handler(ctx, req) - methodType := "unary" + grpcMeta := metrics.GetGrpcEndPointMetadataFromFullMethod(ctx, info.FullMethod, "unary") if err != nil { var terr *api.TigrisError var ferr fdb.Error if errors.As(err, &terr) { - countSpecificErrorMessage(ctx, info.FullMethod, methodType, "tigris_server", terr.Code.String()) + countSpecificErrorMessage(grpcMeta, "tigris_server", terr.Code.String()) } else if errors.As(err, &ferr) { - countSpecificErrorMessage(ctx, info.FullMethod, methodType, "fdb_server", strconv.Itoa(ferr.Code)) + countSpecificErrorMessage(grpcMeta, "fdb_server", strconv.Itoa(ferr.Code)) } else { - countUnknownErrorMessage(ctx, info.FullMethod, methodType) + countUnknownErrorMessage(grpcMeta) } } else { - countOkMessage(ctx, info.FullMethod, methodType) + countOkMessage(grpcMeta) } return resp, err } @@ -108,10 +78,8 @@ func metricsUnaryServerInterceptorCounters() func(ctx context.Context, req inter func metricsStreamServerInterceptorResponseTime() grpc.StreamServerInterceptor { return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { - methodType := "stream" - ctx := stream.Context() - tags := getOkMethodTags(ctx, info.FullMethod, methodType) - defer metrics.RequestsRespTime.Tagged(tags).Histogram("histogram", tally.DefaultBuckets).Start().Stop() + grpcMeta := metrics.GetGrpcEndPointMetadataFromFullMethod(stream.Context(), info.FullMethod, "stream") + defer metrics.RequestsRespTime.Tagged(grpcMeta.GetTags()).Histogram("histogram", tally.DefaultBuckets).Start().Stop() wrapper := &recvWrapper{stream} err := handler(srv, wrapper) return err @@ -120,22 +88,21 @@ func metricsStreamServerInterceptorResponseTime() grpc.StreamServerInterceptor { func metricsStreamServerInterceptorCounter() grpc.StreamServerInterceptor { return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + grpcMeta := metrics.GetGrpcEndPointMetadataFromFullMethod(stream.Context(), info.FullMethod, "stream") wrapper := &recvWrapper{stream} - ctx := stream.Context() err := handler(srv, wrapper) - methodType := "stream" if err != nil { var terr *api.TigrisError var ferr fdb.Error if errors.As(err, &terr) { - countSpecificErrorMessage(ctx, info.FullMethod, methodType, "tigris_server", terr.Code.String()) + countSpecificErrorMessage(grpcMeta, "tigris_server", terr.Code.String()) } else if errors.As(err, &ferr) { - countSpecificErrorMessage(ctx, info.FullMethod, methodType, "fdb_server", strconv.Itoa(ferr.Code)) + countSpecificErrorMessage(grpcMeta, "fdb_server", strconv.Itoa(ferr.Code)) } else { - countUnknownErrorMessage(ctx, info.FullMethod, methodType) + countUnknownErrorMessage(grpcMeta) } } else { - countOkMessage(ctx, info.FullMethod, methodType) + countOkMessage(grpcMeta) } return err } diff --git a/server/midddleware/metrics_test.go b/server/midddleware/metrics_test.go index 0c9da5eb7..4fd40fc2a 100644 --- a/server/midddleware/metrics_test.go +++ b/server/midddleware/metrics_test.go @@ -23,14 +23,11 @@ import ( func TestGrpcMetrics(t *testing.T) { metrics.InitializeMetrics() - methodType := "unary" - fullMethodName := "/tigrisdata.v1.Tigris/TestMethod" - - ctx := context.Background() + grpcMeta := metrics.GetGrpcEndPointMetadataFromFullMethod(context.Background(), "/tigrisdata.v1.Tigris/TestMethod", "unary") t.Run("Test tigris server counters", func(t *testing.T) { - countUnknownErrorMessage(ctx, fullMethodName, methodType) - countOkMessage(ctx, fullMethodName, methodType) - countSpecificErrorMessage(ctx, fullMethodName, methodType, "test_source", "test_code") + countUnknownErrorMessage(grpcMeta) + countOkMessage(grpcMeta) + countSpecificErrorMessage(grpcMeta, "test_source", "test_code") }) } diff --git a/server/midddleware/middleware.go b/server/midddleware/middleware.go index 361a739af..a823fd7cc 100644 --- a/server/midddleware/middleware.go +++ b/server/midddleware/middleware.go @@ -22,7 +22,6 @@ import ( grpc_opentracing "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing" grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/auth" grpc_logging "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" - grpc_ratelimit "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/ratelimit" grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery" "github.com/rs/zerolog" "github.com/rs/zerolog/log" @@ -34,7 +33,6 @@ import ( "github.com/tigrisdata/tigris/server/transaction" "github.com/tigrisdata/tigris/util" "google.golang.org/grpc" - grpctrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/google.golang.org/grpc" ) func Get(config *config.Config, tenantMgr *metadata.TenantManager, txMgr *transaction.Manager) (grpc.UnaryServerInterceptor, grpc.StreamServerInterceptor) { @@ -68,12 +66,12 @@ func Get(config *config.Config, tenantMgr *metadata.TenantManager, txMgr *transa // The order of the interceptors matter with optional elements in them streamInterceptors := []grpc.StreamServerInterceptor{ forwarderStreamServerInterceptor(), - grpc_ratelimit.StreamServerInterceptor(&RateLimiter{}), grpc_auth.StreamServerInterceptor(authFunction), namespaceInitializer.NamespaceSetterStreamServerInterceptor(), - grpctrace.StreamServerInterceptor(grpctrace.WithServiceName(util.Service)), - grpc_logging.StreamServerInterceptor(grpc_zerolog.InterceptorLogger(sampledTaggedLogger), []grpc_logging.Option{}...), - validatorStreamServerInterceptor(), + } + + if config.Tracing.Enabled { + streamInterceptors = append(streamInterceptors, traceStream()) } if config.Metrics.Grpc.Enabled && config.Metrics.Grpc.ResponseTime { @@ -85,6 +83,9 @@ func Get(config *config.Config, tenantMgr *metadata.TenantManager, txMgr *transa } streamInterceptors = append(streamInterceptors, []grpc.StreamServerInterceptor{ + quotaStreamServerInterceptor(), + grpc_logging.StreamServerInterceptor(grpc_zerolog.InterceptorLogger(sampledTaggedLogger), []grpc_logging.Option{}...), + validatorStreamServerInterceptor(), grpc_opentracing.StreamServerInterceptor(), grpc_recovery.StreamServerInterceptor(), headersStreamServerInterceptor(), @@ -99,14 +100,12 @@ func Get(config *config.Config, tenantMgr *metadata.TenantManager, txMgr *transa // The order of the interceptors matter with optional elements in them unaryInterceptors := []grpc.UnaryServerInterceptor{ forwarderUnaryServerInterceptor(), - pprofUnaryServerInterceptor(), - grpc_ratelimit.UnaryServerInterceptor(&RateLimiter{}), grpc_auth.UnaryServerInterceptor(authFunction), namespaceInitializer.NamespaceSetterUnaryServerInterceptor(), - grpctrace.UnaryServerInterceptor(grpctrace.WithServiceName(util.Service)), - grpc_logging.UnaryServerInterceptor(grpc_zerolog.InterceptorLogger(sampledTaggedLogger)), - validatorUnaryServerInterceptor(), - timeoutUnaryServerInterceptor(DefaultTimeout), + } + + if config.Tracing.Enabled { + unaryInterceptors = append(unaryInterceptors, traceUnary()) } if config.Metrics.Grpc.Enabled && config.Metrics.Grpc.ResponseTime { @@ -118,6 +117,12 @@ func Get(config *config.Config, tenantMgr *metadata.TenantManager, txMgr *transa } unaryInterceptors = append(unaryInterceptors, []grpc.UnaryServerInterceptor{ + //grpctrace.UnaryServerInterceptor(grpctrace.WithServiceName(util.Service)), + pprofUnaryServerInterceptor(), + quotaUnaryServerInterceptor(), + grpc_logging.UnaryServerInterceptor(grpc_zerolog.InterceptorLogger(sampledTaggedLogger)), + validatorUnaryServerInterceptor(), + timeoutUnaryServerInterceptor(DefaultTimeout), grpc_opentracing.UnaryServerInterceptor(), grpc_recovery.UnaryServerInterceptor(), headersUnaryServerInterceptor(), diff --git a/server/midddleware/quota.go b/server/midddleware/quota.go new file mode 100644 index 000000000..385075b09 --- /dev/null +++ b/server/midddleware/quota.go @@ -0,0 +1,30 @@ +package middleware + +import ( + "context" + + "github.com/tigrisdata/tigris/server/quota" + "github.com/tigrisdata/tigris/server/request" + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" +) + +func quotaUnaryServerInterceptor() grpc.UnaryServerInterceptor { + return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + ns, _ := request.GetNamespace(ctx) + if err := quota.Allow(ctx, ns, proto.Size(req.(proto.Message))); err != nil { + return nil, err + } + return handler(ctx, req) + } +} + +func quotaStreamServerInterceptor() grpc.StreamServerInterceptor { + return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + ns, _ := request.GetNamespace(stream.Context()) + if err := quota.Allow(stream.Context(), ns, 0); err != nil { + return err + } + return handler(srv, stream) + } +} diff --git a/server/midddleware/rate_limiter.go b/server/midddleware/rate_limiter.go deleted file mode 100644 index d59781506..000000000 --- a/server/midddleware/rate_limiter.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2022 Tigris Data, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package middleware - -import ( - "context" - "time" -) - -const ( - AllowedConnections int = 256 - BacklogConnectionsLimit int = 256 - BacklogWindow time.Duration = 1 * time.Second -) - -type RateLimiter struct { -} - -func (r *RateLimiter) Limit(_ context.Context) error { - return nil -} diff --git a/server/midddleware/tracing.go b/server/midddleware/tracing.go new file mode 100644 index 000000000..b0587f1de --- /dev/null +++ b/server/midddleware/tracing.go @@ -0,0 +1,50 @@ +// Copyright 2022 Tigris Data, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package middleware + +import ( + "context" + + middleware "github.com/grpc-ecosystem/go-grpc-middleware" + "github.com/tigrisdata/tigris/server/metrics" + "google.golang.org/grpc" +) + +func traceUnary() func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + var finisher func() + grpcMeta := metrics.GetGrpcEndPointMetadataFromFullMethod(ctx, info.FullMethod, "unary") + spanMeta := metrics.NewSpanMeta(metrics.GrpcTracingServiceName, info.FullMethod, "unary_rpc", grpcMeta.GetTags()) + ctx, finisher = spanMeta.StartTracing(ctx, false) + defer finisher() + resp, err := handler(ctx, req) + return resp, err + } +} + +func traceStream() grpc.StreamServerInterceptor { + return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + var finisher func() + wrapped := middleware.WrapServerStream(stream) + wrapped.WrappedContext = stream.Context() + grpcMeta := metrics.GetGrpcEndPointMetadataFromFullMethod(wrapped.WrappedContext, info.FullMethod, "stream") + spanMeta := metrics.NewSpanMeta(metrics.GrpcTracingServiceName, info.FullMethod, "stream_rpc", grpcMeta.GetTags()) + wrapped.WrappedContext, finisher = spanMeta.StartTracing(wrapped.WrappedContext, false) + defer finisher() + wrapper := &recvWrapper{wrapped} + err := handler(srv, wrapper) + return err + } +} diff --git a/server/quota/quota.go b/server/quota/quota.go new file mode 100644 index 000000000..2f27595aa --- /dev/null +++ b/server/quota/quota.go @@ -0,0 +1,120 @@ +package quota + +import ( + "context" + "sync" + "time" + + api "github.com/tigrisdata/tigris/api/server/v1" + "github.com/tigrisdata/tigris/server/config" + "github.com/tigrisdata/tigris/server/metadata" + "github.com/tigrisdata/tigris/server/transaction" + "go.uber.org/atomic" + "golang.org/x/time/rate" +) + +var ( + sizeLimitUpdateInterval int64 = 5 // seconds + + ErrRateExceeded = api.Errorf(api.Code_RESOURCE_EXHAUSTED, "request rate limit exceeded") + ErrThroughputExceeded = api.Errorf(api.Code_RESOURCE_EXHAUSTED, "request throughput limit exceeded") + ErrStorageSizeExceeded = api.Errorf(api.Code_RESOURCE_EXHAUSTED, "data size limit exceeded") +) + +type State struct { + Rate *rate.Limiter + WriteThroughput *rate.Limiter + Size atomic.Int64 + SizeUpdateAt atomic.Int64 + SizeLock sync.Mutex +} + +type Manager struct { + tenantQuota sync.Map + cfg *config.QuotaConfig + tenantMgr *metadata.TenantManager + txMgr *transaction.Manager +} + +var mgr Manager + +func Init(t *metadata.TenantManager, tx *transaction.Manager, c *config.QuotaConfig) { + mgr = *newManager(t, tx, c) +} + +func Allow(ctx context.Context, namespace string, reqSize int) error { + if !mgr.cfg.Enabled { + return nil + } + return mgr.check(ctx, namespace, reqSize) +} + +func newManager(t *metadata.TenantManager, tx *transaction.Manager, c *config.QuotaConfig) *Manager { + mgr.cfg = c + mgr.tenantMgr = t + mgr.txMgr = tx + + return &Manager{cfg: c, tenantMgr: t, txMgr: tx} +} + +func (m *Manager) check(ctx context.Context, namespace string, size int) error { + is, ok := m.tenantQuota.Load(namespace) + if !ok { + // Create new state if didn't exist before + is = &State{ + Rate: rate.NewLimiter(rate.Limit(m.cfg.RateLimit), 10), + WriteThroughput: rate.NewLimiter(rate.Limit(m.cfg.WriteThroughputLimit), m.cfg.WriteThroughputLimit), + } + m.tenantQuota.Store(namespace, is) + } + + s := is.(*State) + + if !s.Rate.Allow() { + return ErrRateExceeded + } + + if !s.WriteThroughput.AllowN(time.Now(), size) { + return ErrThroughputExceeded + } + + return m.checkStorageSize(ctx, namespace, s, size) +} + +func (m *Manager) checkStorageSize(ctx context.Context, namespace string, s *State, size int) error { + sz := s.Size.Load() + + if time.Now().Unix() < s.SizeUpdateAt.Load()+sizeLimitUpdateInterval { + if sz+int64(size) >= m.cfg.DataSizeLimit { + return ErrStorageSizeExceeded + } + return nil + } + + s.SizeLock.Lock() + defer s.SizeLock.Unlock() + + if time.Now().Unix() >= s.SizeUpdateAt.Load()+sizeLimitUpdateInterval { + s.SizeUpdateAt.Store(time.Now().Unix()) + + t, err := m.tenantMgr.GetTenant(ctx, namespace, m.txMgr) + if err != nil { + return err + } + + dsz, err := t.Size(ctx) + if err != nil { + return err + } + + s.Size.Store(dsz) + } + + sz = s.Size.Load() + + if sz+int64(size) >= m.cfg.DataSizeLimit { + return ErrStorageSizeExceeded + } + + return nil +} diff --git a/server/quota/quota_test.go b/server/quota/quota_test.go new file mode 100644 index 000000000..00808f043 --- /dev/null +++ b/server/quota/quota_test.go @@ -0,0 +1,118 @@ +package quota + +import ( + "context" + "fmt" + "math/rand" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/tigrisdata/tigris/internal" + "github.com/tigrisdata/tigris/schema" + "github.com/tigrisdata/tigris/server/config" + "github.com/tigrisdata/tigris/server/metadata" + "github.com/tigrisdata/tigris/server/transaction" + "github.com/tigrisdata/tigris/store/kv" + ulog "github.com/tigrisdata/tigris/util/log" +) + +var kvStore kv.KeyValueStore + +func TestQuotaManager(t *testing.T) { + txMgr := transaction.NewManager(kvStore) + tenantMgr, ctx, cancel := metadata.NewTestTenantMgr(kvStore) + defer cancel() + + m := newManager(tenantMgr, txMgr, &config.QuotaConfig{Enabled: true, + RateLimit: 6, + WriteThroughputLimit: 100, + DataSizeLimit: 10000, + }) + + ns := fmt.Sprintf("ns-test-tenantQuota-1-%x", rand.Uint64()) + id := rand.Uint32() + + tenant, err := tenantMgr.CreateOrGetTenant(ctx, txMgr, metadata.NewTenantNamespace(ns, id)) + require.NoError(t, err) + + tx, err := txMgr.StartTx(context.TODO()) + require.NoError(t, err) + + _, err = tenant.CreateDatabase(ctx, tx, "tenant_db1") + require.NoError(t, err) + + jsSchema := []byte(`{ + "title": "test_collection", + "properties": { + "K1": { "type": "string" }, + "K2": { "type": "integer" }, + "D1": { "type": "string", "maxLength": 128 } + }, + "primary_key": ["K1", "K2"] + }`) + + factory, err := schema.Build("test_collection", jsSchema) + require.NoError(t, err) + + err = tenant.ReloadUsingOutsideVersion(ctx, tx, []byte("aaa"), "") + require.NoError(t, err) + + db1, err := tenant.GetDatabase(ctx, tx, "tenant_db1") + require.NoError(t, err) + + require.NoError(t, tenant.CreateCollection(ctx, tx, db1, factory)) + + require.NoError(t, tx.Commit(context.TODO())) + + coll1 := db1.GetCollection("test_collection") + docSize := 10 * 1024 + table, err := metadata.NewEncoder().EncodeTableName(tenant.GetNamespace(), db1, coll1) + require.NoError(t, err) + + require.NoError(t, kvStore.DropTable(ctx, table)) + + require.NoError(t, m.check(ctx, ns, 10)) + require.NoError(t, m.check(ctx, ns, 20)) + + require.Equal(t, ErrThroughputExceeded, m.check(ctx, ns, 500)) + + for i := 0; i < 10; i++ { + err = kvStore.Insert(ctx, table, kv.BuildKey(fmt.Sprintf("aaa%d", i)), &internal.TableData{RawData: make([]byte, docSize)}) + require.NoError(t, err) + } + + sizeLimitUpdateInterval = 0 + require.Equal(t, ErrStorageSizeExceeded, m.check(ctx, ns, 0)) + sizeLimitUpdateInterval = 1 + require.Equal(t, ErrStorageSizeExceeded, m.check(ctx, ns, 0)) + + require.NoError(t, kvStore.DropTable(ctx, table)) + + sizeLimitUpdateInterval = 0 + + require.NoError(t, m.check(ctx, ns, 0)) + + for err != ErrRateExceeded { + err = m.check(ctx, ns, 0) + } +} + +func TestMain(m *testing.M) { + rand.Seed(time.Now().Unix()) + + ulog.Configure(ulog.LogConfig{Level: "disabled", Format: "console"}) + + fdbCfg, err := config.GetTestFDBConfig("../..") + if err != nil { + panic(fmt.Sprintf("failed to init FDB config: %v", err)) + } + + kvStore, err = kv.NewKeyValueStore(fdbCfg) + if err != nil { + panic(fmt.Sprintf("failed to init FDB KV %v", err)) + } + + os.Exit(m.Run()) +} diff --git a/server/services/v1/api.go b/server/services/v1/api.go index 660e2e208..fa544cfa2 100644 --- a/server/services/v1/api.go +++ b/server/services/v1/api.go @@ -55,7 +55,6 @@ type apiService struct { kvStore kv.KeyValueStore txMgr *transaction.Manager - encoder metadata.Encoder tenantMgr *metadata.TenantManager cdcMgr *cdc.Manager sessions *SessionManager @@ -70,6 +69,8 @@ func newApiService(kv kv.KeyValueStore, searchStore search.Store, tenantMgr *met txMgr: txMgr, versionH: &metadata.VersionHandler{}, searchStore: searchStore, + cdcMgr: cdc.NewManager(), + tenantMgr: tenantMgr, } ctx := context.TODO() @@ -82,13 +83,20 @@ func newApiService(kv kv.KeyValueStore, searchStore search.Store, tenantMgr *met // ToDo: no need to panic, probably handle through async thread. log.Err(err).Msgf("error starting server: reloading tenants failed") } - _ = tx.Commit(ctx) + ulog.E(tx.Commit(ctx)) + + var txListeners []TxListener + if config.DefaultConfig.Cdc.Enabled { + txListeners = append(txListeners, u.cdcMgr) + } + if config.DefaultConfig.Search.WriteEnabled { + // just for testing so that we can disable it if needed + txListeners = append(txListeners, NewSearchIndexer(searchStore, tenantMgr)) + } + + u.sessions = NewSessionManager(u.txMgr, u.tenantMgr, u.versionH, txListeners) + u.runnerFactory = NewQueryRunnerFactory(u.txMgr, u.cdcMgr, u.searchStore) - u.tenantMgr = tenantMgr - u.encoder = metadata.NewEncoder(u.tenantMgr) - u.cdcMgr = cdc.NewManager() - u.sessions = NewSessionManager(u.txMgr, u.tenantMgr, u.versionH, u.cdcMgr, u.searchStore, u.encoder) - u.runnerFactory = NewQueryRunnerFactory(u.txMgr, u.encoder, u.cdcMgr, u.searchStore) return u } @@ -403,6 +411,11 @@ func (s *apiService) Events(r *api.EventsRequest, stream api.Tigris_EventsServer if !config.DefaultConfig.Cdc.Enabled { return api.Errorf(api.Code_METHOD_NOT_ALLOWED, "change streams is disabled for this collection") } + + if len(r.Collection) == 0 { + return api.Errorf(api.Code_INVALID_ARGUMENT, "collection name is missing") + } + publisher := s.cdcMgr.GetPublisher(r.GetDb()) streamer, err := publisher.NewStreamer(s.kvStore) if err != nil { @@ -410,15 +423,28 @@ func (s *apiService) Events(r *api.EventsRequest, stream api.Tigris_EventsServer } defer streamer.Close() + reqDatabaseId, reqCollectionId := uint32(0), uint32(0) for tx := range streamer.Txs { for _, op := range tx.Ops { - _, _, collection, ok := s.encoder.DecodeTableName(op.Table) + if reqDatabaseId == 0 || reqCollectionId == 0 { + if reqDatabaseId, reqCollectionId = s.tenantMgr.GetDatabaseAndCollectionId(r.GetDb(), r.Collection); reqDatabaseId == 0 || reqCollectionId == 0 { + // neither is ready yet + continue + } + } + + _, dbId, cId, ok := s.tenantMgr.GetEncoder().DecodeTableName(op.Table) if !ok { - log.Err(err).Str("table", string(op.Table)).Msg("failed to decode collection name") - return api.Errorf(api.Code_INTERNAL, "failed to decode collection name") + log.Err(err).Str("table", string(op.Table)).Msg("unexpected key in event streams") + return api.Errorf(api.Code_INTERNAL, "unexpected key in event streams") + } + + if dbId != reqDatabaseId || cId != reqCollectionId { + // this probably means database/collection is dropped and this entry is some old entry in the log, ignore it + continue } - if r.Collection == "" || r.Collection == collection { + if op.Op != kv.DeleteEvent && op.Op != kv.DeleteRangeEvent { td, err := internal.Decode(op.Data) if err != nil { log.Err(err).Str("data", string(op.Data)).Msg("failed to decode data") @@ -427,7 +453,7 @@ func (s *apiService) Events(r *api.EventsRequest, stream api.Tigris_EventsServer event := &api.StreamEvent{ TxId: tx.Id, - Collection: collection, + Collection: r.Collection, Op: op.Op, Key: op.Key, Lkey: op.LKey, diff --git a/server/services/v1/query_runner.go b/server/services/v1/query_runner.go index 4faf5a278..edfc44ddf 100644 --- a/server/services/v1/query_runner.go +++ b/server/services/v1/query_runner.go @@ -50,10 +50,10 @@ type QueryRunnerFactory struct { } // NewQueryRunnerFactory returns QueryRunnerFactory object -func NewQueryRunnerFactory(txMgr *transaction.Manager, encoder metadata.Encoder, cdcMgr *cdc.Manager, searchStore search.Store) *QueryRunnerFactory { +func NewQueryRunnerFactory(txMgr *transaction.Manager, cdcMgr *cdc.Manager, searchStore search.Store) *QueryRunnerFactory { return &QueryRunnerFactory{ txMgr: txMgr, - encoder: encoder, + encoder: metadata.NewEncoder(), cdcMgr: cdcMgr, searchStore: searchStore, } @@ -536,6 +536,11 @@ func (runner *SearchQueryRunner) Run(ctx context.Context, tx transaction.Tx, ten return nil, ctx, err } + fieldSelection, err := runner.getFieldSelection(collection.GetFields()) + if err != nil { + return nil, ctx, err + } + pageSize := int(runner.req.PageSize) if pageSize == 0 { pageSize = defaultPerPage @@ -548,6 +553,7 @@ func (runner *SearchQueryRunner) Run(ctx context.Context, tx transaction.Tx, ten Facets(facets). PageSize(pageSize). Filter(wrappedF). + ReadFields(fieldSelection). Build() var rowReader *SearchRowReader @@ -665,7 +671,6 @@ func (runner *SearchQueryRunner) getFacetFields(collFields []*schema.Field) (qse } found = true break - } if !found { return qsearch.Facets{}, api.Errorf(api.Code_INVALID_ARGUMENT, "`%s` is not a schema field", ff.Name) @@ -675,6 +680,43 @@ func (runner *SearchQueryRunner) getFacetFields(collFields []*schema.Field) (qse return facets, nil } +func (runner *SearchQueryRunner) getFieldSelection(collFields []*schema.Field) (*read.FieldFactory, error) { + var selectionFields []string + + // Only one of include/exclude. Honor inclusion over exclusion + if len(runner.req.IncludeFields) > 0 { + selectionFields = runner.req.IncludeFields + } else if len(runner.req.ExcludeFields) > 0 { + selectionFields = runner.req.ExcludeFields + } else { + return nil, nil + } + + factory := &read.FieldFactory{ + Include: map[string]read.Field{}, + Exclude: map[string]read.Field{}, + } + + for _, sf := range selectionFields { + found := false + for _, cf := range collFields { + if sf == cf.FieldName { + found = true + } + } + if !found { + return nil, api.Errorf(api.Code_INVALID_ARGUMENT, "`%s` is not a schema field", sf) + } + + factory.AddField(&read.SimpleField{ + Name: sf, + Incl: len(runner.req.IncludeFields) > 0, + }) + } + + return factory, nil +} + type CollectionQueryRunner struct { *BaseQueryRunner @@ -713,7 +755,7 @@ func (runner *CollectionQueryRunner) Run(ctx context.Context, tx transaction.Tx, tx.Context().StageDatabase(db) } - if err = tenant.DropCollection(ctx, tx, db, runner.dropReq.GetCollection(), runner.searchStore, runner.encoder); err != nil { + if err = tenant.DropCollection(ctx, tx, db, runner.dropReq.GetCollection()); err != nil { return nil, ctx, err } @@ -742,7 +784,7 @@ func (runner *CollectionQueryRunner) Run(ctx context.Context, tx transaction.Tx, tx.Context().StageDatabase(db) } - if err = tenant.CreateCollection(ctx, tx, db, schFactory, runner.searchStore); err != nil { + if err = tenant.CreateCollection(ctx, tx, db, schFactory); err != nil { if err == kv.ErrDuplicateKey { // this simply means, concurrently CreateCollection is called, return nil, ctx, api.Errorf(api.Code_ABORTED, "concurrent create collection request, aborting") @@ -781,11 +823,18 @@ func (runner *CollectionQueryRunner) Run(ctx context.Context, tx transaction.Tx, if err != nil { return nil, ctx, err } + + size, err := tenant.CollectionSize(ctx, db, coll) + if err != nil { + return nil, ctx, err + } + return &Response{ Response: &api.DescribeCollectionResponse{ Collection: coll.Name, Metadata: &api.CollectionMetadata{}, Schema: coll.Schema, + Size: size, }, }, ctx, nil } @@ -820,7 +869,7 @@ func (runner *DatabaseQueryRunner) SetDescribeDatabaseReq(describe *api.Describe func (runner *DatabaseQueryRunner) Run(ctx context.Context, tx transaction.Tx, tenant *metadata.Tenant) (*Response, context.Context, error) { if runner.drop != nil { - exist, err := tenant.DropDatabase(ctx, tx, runner.drop.GetDb(), runner.searchStore, runner.encoder) + exist, err := tenant.DropDatabase(ctx, tx, runner.drop.GetDb()) if err != nil { return nil, ctx, err } @@ -867,18 +916,29 @@ func (runner *DatabaseQueryRunner) Run(ctx context.Context, tx transaction.Tx, t var collections = make([]*api.CollectionDescription, len(collectionList)) for i, c := range collectionList { + size, err := tenant.CollectionSize(ctx, db, c) + if err != nil { + return nil, ctx, err + } collections[i] = &api.CollectionDescription{ Collection: c.GetName(), Metadata: &api.CollectionMetadata{}, Schema: c.Schema, + Size: size, } } + size, err := tenant.DatabaseSize(ctx, db) + if err != nil { + return nil, ctx, err + } + return &Response{ Response: &api.DescribeDatabaseResponse{ Db: db.Name(), Metadata: &api.DatabaseMetadata{}, Collections: collections, + Size: size, }, }, ctx, nil } diff --git a/server/services/v1/query_runner_test.go b/server/services/v1/query_runner_test.go new file mode 100644 index 000000000..31615d97e --- /dev/null +++ b/server/services/v1/query_runner_test.go @@ -0,0 +1,97 @@ +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + api "github.com/tigrisdata/tigris/api/server/v1" + "github.com/tigrisdata/tigris/schema" +) + +func TestSearchQueryRunner_getFieldSelection(t *testing.T) { + t.Run("only include fields are provided", func(t *testing.T) { + collFields := []*schema.Field{ + {FieldName: "field_1"}, + {FieldName: "field_2"}, + } + + runner := &SearchQueryRunner{ + req: &api.SearchRequest{ + IncludeFields: []string{"field_1", "field_2"}, + }, + } + + factory, err := runner.getFieldSelection(collFields) + + assert.Nil(t, err) + assert.NotNil(t, factory) + assert.Empty(t, factory.Exclude) + assert.Len(t, factory.Include, 2) + assert.Contains(t, factory.Include, "field_1") + assert.Contains(t, factory.Include, "field_2") + }) + + t.Run("only exclude fields are provided", func(t *testing.T) { + collFields := []*schema.Field{ + {FieldName: "field_1"}, + {FieldName: "field_2"}, + } + + runner := &SearchQueryRunner{ + req: &api.SearchRequest{ + ExcludeFields: []string{"field_1", "field_2"}, + }, + } + + factory, err := runner.getFieldSelection(collFields) + + assert.Nil(t, err) + assert.NotNil(t, factory) + assert.Empty(t, factory.Include) + assert.Len(t, factory.Exclude, 2) + assert.Contains(t, factory.Exclude, "field_1") + assert.Contains(t, factory.Exclude, "field_2") + }) + + t.Run("no fields to include or exclude", func(t *testing.T) { + collFields := []*schema.Field{ + {FieldName: "field_1"}, + {FieldName: "field_2"}, + } + runner := &SearchQueryRunner{req: &api.SearchRequest{}} + + factory, err := runner.getFieldSelection(collFields) + + assert.Nil(t, err) + assert.Nil(t, factory) + }) + + t.Run("no schema fields are defined", func(t *testing.T) { + var collFields []*schema.Field + runner := &SearchQueryRunner{req: &api.SearchRequest{}} + + factory, err := runner.getFieldSelection(collFields) + + assert.Nil(t, err) + assert.Nil(t, factory) + }) + + t.Run("selection fields are not in schema", func(t *testing.T) { + collFields := []*schema.Field{ + {FieldName: "field_1"}, + {FieldName: "field_2"}, + } + + runner := &SearchQueryRunner{ + req: &api.SearchRequest{ + ExcludeFields: []string{"field_2", "field_3"}, + }, + } + + factory, err := runner.getFieldSelection(collFields) + + assert.Nil(t, factory) + assert.NotNil(t, err) + assert.ErrorContains(t, err, "`field_3` is not a schema field") + }) +} diff --git a/server/services/v1/search_indexer.go b/server/services/v1/search_indexer.go index e11da51ec..974218b92 100644 --- a/server/services/v1/search_indexer.go +++ b/server/services/v1/search_indexer.go @@ -47,20 +47,21 @@ const ( type SearchIndexer struct { searchStore search.Store - encoder metadata.Encoder + tenantMgr *metadata.TenantManager } -func NewSearchIndexer(searchStore search.Store, encoder metadata.Encoder) *SearchIndexer { +func NewSearchIndexer(searchStore search.Store, tenantMgr *metadata.TenantManager) *SearchIndexer { return &SearchIndexer{ searchStore: searchStore, - encoder: encoder, + tenantMgr: tenantMgr, } } func (i *SearchIndexer) OnPostCommit(ctx context.Context, tenant *metadata.Tenant, eventListener kv.EventListener) error { for _, event := range eventListener.GetEvents() { var err error - _, db, coll, ok := i.encoder.DecodeTableName(event.Table) + + _, db, coll, ok := i.tenantMgr.DecodeTableName(event.Table) if !ok { continue } diff --git a/server/services/v1/search_reader.go b/server/services/v1/search_reader.go index 32fbee990..c57763d77 100644 --- a/server/services/v1/search_reader.go +++ b/server/services/v1/search_reader.go @@ -20,9 +20,11 @@ import ( jsoniter "github.com/json-iterator/go" api "github.com/tigrisdata/tigris/api/server/v1" "github.com/tigrisdata/tigris/query/filter" + "github.com/tigrisdata/tigris/query/read" qsearch "github.com/tigrisdata/tigris/query/search" "github.com/tigrisdata/tigris/schema" "github.com/tigrisdata/tigris/store/search" + ulog "github.com/tigrisdata/tigris/util/log" tsApi "github.com/typesense/typesense-go/typesense/api" ) @@ -38,6 +40,7 @@ type page struct { hits *HitsResponse wrappedF *filter.WrappedFilter collection *schema.DefaultCollection + readFields *read.FieldFactory } func newPage(collection *schema.DefaultCollection, query *qsearch.Query) *page { @@ -47,6 +50,7 @@ func newPage(collection *schema.DefaultCollection, query *qsearch.Query) *page { cap: query.PageSize, wrappedF: query.WrappedF, collection: collection, + readFields: query.ReadFields, } } @@ -89,11 +93,25 @@ func (p *page) readRow(row *Row) bool { continue } - // set the raw data now after marshaling it - if row.Data.RawData, p.err = jsoniter.Marshal(doc); p.err != nil { + var rawData []byte + + // marshal the doc as bytes + rawData, p.err = jsoniter.Marshal(doc) + if p.err != nil { return false } + // apply field selection + if p.readFields != nil { + newValue, err := p.readFields.Apply(rawData) + if ulog.E(err) { + return false + } + row.Data.RawData = newValue + } else { + row.Data.RawData = rawData + } + return true } diff --git a/server/services/v1/sessions.go b/server/services/v1/sessions.go index f57fc5a76..40b969f72 100644 --- a/server/services/v1/sessions.go +++ b/server/services/v1/sessions.go @@ -22,14 +22,11 @@ import ( "github.com/rs/zerolog/log" api "github.com/tigrisdata/tigris/api/server/v1" - "github.com/tigrisdata/tigris/server/cdc" - "github.com/tigrisdata/tigris/server/config" "github.com/tigrisdata/tigris/server/metadata" middleware "github.com/tigrisdata/tigris/server/midddleware" "github.com/tigrisdata/tigris/server/request" "github.com/tigrisdata/tigris/server/transaction" "github.com/tigrisdata/tigris/store/kv" - "github.com/tigrisdata/tigris/store/search" ulog "github.com/tigrisdata/tigris/util/log" ) @@ -44,27 +41,16 @@ type SessionManager struct { tenantMgr *metadata.TenantManager versionH *metadata.VersionHandler tracker *sessionTracker - searchStore search.Store txListeners []TxListener } -func NewSessionManager(txMgr *transaction.Manager, tenantMgr *metadata.TenantManager, versionH *metadata.VersionHandler, cdc *cdc.Manager, searchStore search.Store, encoder metadata.Encoder) *SessionManager { - var txListeners []TxListener - if config.DefaultConfig.Cdc.Enabled { - txListeners = append(txListeners, cdc) - } - if config.DefaultConfig.Search.WriteEnabled { - // just for testing so that we can disable it if needed - txListeners = append(txListeners, NewSearchIndexer(searchStore, encoder)) - } - +func NewSessionManager(txMgr *transaction.Manager, tenantMgr *metadata.TenantManager, versionH *metadata.VersionHandler, listeners []TxListener) *SessionManager { return &SessionManager{ txMgr: txMgr, tenantMgr: tenantMgr, versionH: versionH, tracker: newSessionTracker(), - txListeners: txListeners, - searchStore: searchStore, + txListeners: listeners, } } @@ -76,15 +62,7 @@ func (sessMgr *SessionManager) Create(ctx context.Context, reloadVerOutside bool if err != nil { return nil, err } - var tenant *metadata.Tenant - if namespaceForThisSession == metadata.DefaultNamespaceName { - tenant, err = sessMgr.tenantMgr.CreateOrGetTenant(ctx, sessMgr.txMgr, metadata.NewDefaultNamespace()) - } else { - tenant, err = sessMgr.tenantMgr.GetTenant(ctx, namespaceForThisSession, sessMgr.txMgr) - } - if tenant == nil { - return nil, api.Errorf(api.Code_NOT_FOUND, "Tenant %s not found", namespaceForThisSession) - } + tenant, err := sessMgr.tenantMgr.GetTenant(ctx, namespaceForThisSession, sessMgr.txMgr) if err != nil { log.Warn().Err(err).Msgf("Could not find tenant, this must not happen with right authn/authz configured") return nil, api.Errorf(api.Code_NOT_FOUND, "Tenant %s not found", namespaceForThisSession) diff --git a/server/transaction/manager.go b/server/transaction/manager.go index e563cff7e..c81b3601f 100644 --- a/server/transaction/manager.go +++ b/server/transaction/manager.go @@ -23,8 +23,11 @@ import ( "github.com/tigrisdata/tigris/internal" "github.com/tigrisdata/tigris/keys" "github.com/tigrisdata/tigris/schema" + "github.com/tigrisdata/tigris/server/config" + "github.com/tigrisdata/tigris/server/metrics" "github.com/tigrisdata/tigris/server/types" "github.com/tigrisdata/tigris/store/kv" + ulog "github.com/tigrisdata/tigris/util/log" ) var ( @@ -50,6 +53,7 @@ type Tx interface { Rollback(ctx context.Context) error SetVersionstampedValue(ctx context.Context, key []byte, value []byte) error SetVersionstampedKey(ctx context.Context, key []byte, value []byte) error + start(ctx context.Context) error } type StagedDB interface { @@ -73,6 +77,7 @@ func (c *SessionCtx) GetStagedDatabase() StagedDB { // Manager is used to track all the sessions and provide all the functionality related to transactions. Once created // this will create a session tracker for tracking the sessions. + type Manager struct { kvStore kv.KeyValueStore } @@ -85,7 +90,13 @@ func NewManager(kvStore kv.KeyValueStore) *Manager { // StartTx always starts a new session and tracks the session based on the input parameter. func (m *Manager) StartTx(ctx context.Context) (Tx, error) { - session, err := newTxSession(m.kvStore) + var session Tx + var err error + if config.DefaultConfig.Tracing.Enabled { + session, err = newTxSessionWithMetrics(m.kvStore) + } else { + session, err = newTxSession(m.kvStore) + } if err != nil { return nil, api.Errorf(api.Code_INTERNAL, "issue creating a session %v", err) } @@ -118,6 +129,10 @@ type TxSession struct { txCtx *api.TransactionCtx } +type TxSessionWithMetrics struct { + t *TxSession +} + func newTxSession(kv kv.KeyValueStore) (*TxSession, error) { if kv == nil { return nil, api.Errorf(api.Code_INTERNAL, "session needs non-nil kv object") @@ -130,10 +145,36 @@ func newTxSession(kv kv.KeyValueStore) (*TxSession, error) { }, nil } +func newTxSessionWithMetrics(kv kv.KeyValueStore) (*TxSessionWithMetrics, error) { + if kv == nil { + return nil, api.Errorf(api.Code_INTERNAL, "session needs non-nil kv object") + } + return &TxSessionWithMetrics{&TxSession{ + context: &SessionCtx{}, + kvStore: kv, + state: sessionCreated, + txCtx: generateTransactionCtx(), + }}, nil +} + +func (m *TxSessionWithMetrics) measure(ctx context.Context, name string, f func(ctx context.Context) error) { + var finishTracing func() + tags := metrics.GetFdbTags(ctx, name) + spanMeta := metrics.NewSpanMeta(metrics.TxManagerTracingServiceName, name, "tx_manager", tags) + ctx, finishTracing = spanMeta.StartTracing(ctx, true) + defer finishTracing() + // TODO: better error handling (error in trace, etc) + ulog.E(f(ctx)) +} + func (s *TxSession) GetTxCtx() *api.TransactionCtx { return s.txCtx } +func (m *TxSessionWithMetrics) GetTxCtx() *api.TransactionCtx { + return m.t.txCtx +} + func (s *TxSession) start(ctx context.Context) error { s.Lock() defer s.Unlock() @@ -151,6 +192,14 @@ func (s *TxSession) start(ctx context.Context) error { return nil } +func (m *TxSessionWithMetrics) start(ctx context.Context) (err error) { + m.measure(ctx, "start", func(ctx context.Context) error { + err = m.t.start(ctx) + return err + }) + return +} + func (s *TxSession) validateSession() error { if s.state == sessionEnded { return ErrSessionIsGone @@ -173,6 +222,14 @@ func (s *TxSession) Insert(ctx context.Context, key keys.Key, data *internal.Tab return s.kTx.Insert(ctx, key.Table(), kv.BuildKey(key.IndexParts()...), data) } +func (m *TxSessionWithMetrics) Insert(ctx context.Context, key keys.Key, data *internal.TableData) (err error) { + m.measure(ctx, "Insert", func(ctx context.Context) error { + err = m.t.Insert(ctx, key, data) + return err + }) + return +} + func (s *TxSession) Replace(ctx context.Context, key keys.Key, data *internal.TableData) error { s.Lock() defer s.Unlock() @@ -184,6 +241,14 @@ func (s *TxSession) Replace(ctx context.Context, key keys.Key, data *internal.Ta return s.kTx.Replace(ctx, key.Table(), kv.BuildKey(key.IndexParts()...), data) } +func (m *TxSessionWithMetrics) Replace(ctx context.Context, key keys.Key, data *internal.TableData) (err error) { + m.measure(ctx, "Replace", func(ctx context.Context) error { + err = m.t.Replace(ctx, key, data) + return err + }) + return +} + func (s *TxSession) Update(ctx context.Context, key keys.Key, apply func(*internal.TableData) (*internal.TableData, error)) (int32, error) { s.Lock() defer s.Unlock() @@ -195,6 +260,14 @@ func (s *TxSession) Update(ctx context.Context, key keys.Key, apply func(*intern return s.kTx.Update(ctx, key.Table(), kv.BuildKey(key.IndexParts()...), apply) } +func (m *TxSessionWithMetrics) Update(ctx context.Context, key keys.Key, apply func(*internal.TableData) (*internal.TableData, error)) (encoded int32, err error) { + m.measure(ctx, "Update", func(ctx context.Context) error { + encoded, err = m.t.Update(ctx, key, apply) + return err + }) + return +} + func (s *TxSession) Delete(ctx context.Context, key keys.Key) error { s.Lock() defer s.Unlock() @@ -206,6 +279,14 @@ func (s *TxSession) Delete(ctx context.Context, key keys.Key) error { return s.kTx.Delete(ctx, key.Table(), kv.BuildKey(key.IndexParts()...)) } +func (m *TxSessionWithMetrics) Delete(ctx context.Context, key keys.Key) (err error) { + m.measure(ctx, "Delete", func(ctx context.Context) error { + err = m.t.Delete(ctx, key) + return err + }) + return +} + func (s *TxSession) Read(ctx context.Context, key keys.Key) (kv.Iterator, error) { s.Lock() defer s.Unlock() @@ -217,6 +298,14 @@ func (s *TxSession) Read(ctx context.Context, key keys.Key) (kv.Iterator, error) return s.kTx.Read(ctx, key.Table(), kv.BuildKey(key.IndexParts()...)) } +func (m *TxSessionWithMetrics) Read(ctx context.Context, key keys.Key) (it kv.Iterator, err error) { + m.measure(ctx, "Read", func(ctx context.Context) error { + it, err = m.t.Read(ctx, key) + return err + }) + return +} + func (s *TxSession) SetVersionstampedValue(ctx context.Context, key []byte, value []byte) error { s.Lock() defer s.Unlock() @@ -228,6 +317,14 @@ func (s *TxSession) SetVersionstampedValue(ctx context.Context, key []byte, valu return s.kTx.SetVersionstampedValue(ctx, key, value) } +func (m *TxSessionWithMetrics) SetVersionstampedValue(ctx context.Context, key []byte, value []byte) (err error) { + m.measure(ctx, "SetVersionstampedValue", func(ctx context.Context) error { + err = m.t.SetVersionstampedValue(ctx, key, value) + return err + }) + return +} + func (s *TxSession) SetVersionstampedKey(ctx context.Context, key []byte, value []byte) error { s.Lock() defer s.Unlock() @@ -239,6 +336,14 @@ func (s *TxSession) SetVersionstampedKey(ctx context.Context, key []byte, value return s.kTx.SetVersionstampedKey(ctx, key, value) } +func (m *TxSessionWithMetrics) SetVersionstampedKey(ctx context.Context, key []byte, value []byte) (err error) { + m.measure(ctx, "SetVersionstampedKey", func(ctx context.Context) error { + err = m.t.SetVersionstampedKey(ctx, key, value) + return err + }) + return +} + func (s *TxSession) Get(ctx context.Context, key []byte) ([]byte, error) { s.Lock() defer s.Unlock() @@ -250,6 +355,14 @@ func (s *TxSession) Get(ctx context.Context, key []byte) ([]byte, error) { return s.kTx.Get(ctx, key) } +func (m *TxSessionWithMetrics) Get(ctx context.Context, key []byte) (val []byte, err error) { + m.measure(ctx, "Get", func(ctx context.Context) error { + val, err = m.t.Get(ctx, key) + return err + }) + return +} + func (s *TxSession) Commit(ctx context.Context) error { s.Lock() defer s.Unlock() @@ -262,6 +375,14 @@ func (s *TxSession) Commit(ctx context.Context) error { return err } +func (m *TxSessionWithMetrics) Commit(ctx context.Context) (err error) { + m.measure(ctx, "Commit", func(ctx context.Context) error { + err = m.t.Commit(ctx) + return err + }) + return +} + func (s *TxSession) Rollback(ctx context.Context) error { s.Lock() defer s.Unlock() @@ -274,10 +395,22 @@ func (s *TxSession) Rollback(ctx context.Context) error { return err } +func (m *TxSessionWithMetrics) Rollback(ctx context.Context) (err error) { + m.measure(ctx, "Rollback", func(ctx context.Context) error { + err = m.t.Rollback(ctx) + return err + }) + return +} + func (s *TxSession) Context() *SessionCtx { return s.context } +func (m *TxSessionWithMetrics) Context() *SessionCtx { + return m.t.context +} + func generateTransactionCtx() *api.TransactionCtx { return &api.TransactionCtx{ Id: uuid.New().String(), diff --git a/store/kv/fdb.go b/store/kv/fdb.go index 9620b95af..f58ca9c2e 100644 --- a/store/kv/fdb.go +++ b/store/kv/fdb.go @@ -229,6 +229,26 @@ func (d *fdbkv) DropTable(ctx context.Context, name []byte) error { return nil } +// TableSize calculates approximate table size in bytes +// It also works with the prefix of the table name, +// allowing to calculate sizes of multiple table with the same prefix +func (d *fdbkv) TableSize(ctx context.Context, name []byte) (int64, error) { + s := subspace.FromBytes(name) + + var sz int64 + _, err := d.txWithRetry(ctx, func(tr fdb.Transaction) (interface{}, error) { + var err error + sz, err = tr.GetEstimatedRangeSizeBytes(s).Get() + return nil, err + }) + + if err != nil { + log.Err(err).Str("name", string(name)).Int64("size", sz).Msg("table size") + } + + return sz, err +} + func (d *fdbkv) Batch() (baseTx, error) { tx, err := d.db.CreateTransaction() if ulog.E(err) { diff --git a/store/kv/kv.go b/store/kv/kv.go index c7f23d9d0..a51ffd692 100644 --- a/store/kv/kv.go +++ b/store/kv/kv.go @@ -60,6 +60,7 @@ type KeyValueStore interface { CreateTable(ctx context.Context, name []byte) error DropTable(ctx context.Context, name []byte) error GetInternalDatabase() (interface{}, error) // TODO: CDC remove workaround + TableSize(ctx context.Context, name []byte) (int64, error) } type Iterator interface { @@ -98,6 +99,10 @@ func NewKeyValueStoreWithMetrics(cfg *config.FoundationDBConfig) (KeyValueStore, func measureLow(ctx context.Context, name string, f func() error) { // Low level measurement wrapper that is called by the measure functions on the appropriate receiver tags := metrics.GetFdbTags(ctx, name) + spanMeta := metrics.NewSpanMeta(metrics.KvTracingServiceName, name, "fdb_kv", tags) + var finishTracing func() + ctx, finishTracing = spanMeta.StartTracing(ctx, true) + defer finishTracing() if config.DefaultConfig.Metrics.Fdb.ResponseTime { metrics.FdbRequests.Tagged(tags).Histogram("histogram", tally.DefaultBuckets) defer metrics.FdbRequests.Tagged(tags).Histogram("histogram", tally.DefaultBuckets).Start().Stop() @@ -158,6 +163,14 @@ func (m *KeyValueStoreImplWithMetrics) DropTable(ctx context.Context, name []byt return } +func (m *KeyValueStoreImplWithMetrics) TableSize(ctx context.Context, name []byte) (size int64, err error) { + m.measure(ctx, "TableSize", func() error { + size, err = m.kv.TableSize(ctx, name) + return err + }) + return +} + func (m *KeyValueStoreImplWithMetrics) SetVersionstampedValue(ctx context.Context, key []byte, value []byte) (err error) { m.measure(ctx, "SetVersionstampedValue", func() error { err = m.kv.SetVersionstampedValue(ctx, key, value) diff --git a/store/kv/listener.go b/store/kv/listener.go index e935a298d..246aa22d4 100644 --- a/store/kv/listener.go +++ b/store/kv/listener.go @@ -15,7 +15,10 @@ package kv import ( + "bytes" "context" + + "github.com/tigrisdata/tigris/internal" ) const ( @@ -58,7 +61,15 @@ type DefaultListener struct { Events []*Event } +func (l *DefaultListener) skip(table []byte) bool { + return !bytes.Equal(table[0:4], internal.UserTableKeyPrefix) +} + func (l *DefaultListener) OnSet(op string, table []byte, key []byte, data []byte) { + if l.skip(table) { + return + } + l.Events = append(l.Events, &Event{ Op: op, Table: table, @@ -67,6 +78,10 @@ func (l *DefaultListener) OnSet(op string, table []byte, key []byte, data []byte }) } func (l *DefaultListener) OnClearRange(op string, table []byte, lKey []byte, rKey []byte) { + if l.skip(table) { + return + } + l.Events = append(l.Events, &Event{ Op: op, Table: table, diff --git a/store/kv/noop_store.go b/store/kv/noop_store.go index 12401de4f..85373580c 100644 --- a/store/kv/noop_store.go +++ b/store/kv/noop_store.go @@ -2,6 +2,7 @@ package kv import ( "context" + "github.com/tigrisdata/tigris/internal" ) @@ -24,10 +25,11 @@ type NoopKVStore struct { *NoopKV } -func (n *NoopKVStore) BeginTx(ctx context.Context) (Tx, error) { return &NoopTx{}, nil } -func (n *NoopKVStore) CreateTable(ctx context.Context, name []byte) error { return nil } -func (n *NoopKVStore) DropTable(ctx context.Context, name []byte) error { return nil } -func (n *NoopKVStore) GetInternalDatabase() (interface{}, error) { return nil, nil } +func (n *NoopKVStore) BeginTx(_ context.Context) (Tx, error) { return &NoopTx{}, nil } +func (n *NoopKVStore) CreateTable(_ context.Context, _ []byte) error { return nil } +func (n *NoopKVStore) DropTable(_ context.Context, _ []byte) error { return nil } +func (n *NoopKVStore) GetInternalDatabase() (interface{}, error) { return nil, nil } +func (n *NoopKVStore) TableSize(_ context.Context, _ []byte) (int64, error) { return 0, nil } type NoopKV struct{} diff --git a/store/object/object.go b/store/object/object.go deleted file mode 100644 index 1a9477745..000000000 --- a/store/object/object.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2022 Tigris Data, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package object diff --git a/store/object/s3.go b/store/object/s3.go deleted file mode 100644 index 1a9477745..000000000 --- a/store/object/s3.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2022 Tigris Data, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package object diff --git a/store/search/ts.go b/store/search/ts.go index b3d748f43..54d3d80f8 100644 --- a/store/search/ts.go +++ b/store/search/ts.go @@ -18,6 +18,9 @@ import ( "context" "encoding/json" "fmt" + "io" + "net/http" + jsoniter "github.com/json-iterator/go" qsearch "github.com/tigrisdata/tigris/query/search" "github.com/tigrisdata/tigris/server/config" @@ -26,8 +29,6 @@ import ( "github.com/typesense/typesense-go/typesense" tsApi "github.com/typesense/typesense-go/typesense/api" "github.com/uber-go/tally" - "io" - "net/http" ) type storeImpl struct { @@ -38,8 +39,30 @@ type storeImplWithMetrics struct { s Store } +func (m *storeImplWithMetrics) measure(ctx context.Context, name string, f func(ctx context.Context) error) { + // Low level measurement wrapper that is called by the measure functions on the appropriate receiver + tags := metrics.GetSearchTags(ctx, name) + var finishTrace func() + spanMeta := metrics.NewSpanMeta("tigris.search", name, "search", tags) + ctx, finishTrace = spanMeta.StartTracing(ctx, true) + defer finishTrace() + if config.DefaultConfig.Metrics.Search.ResponseTime { + metrics.SearchRequests.Tagged(tags).Histogram("histogram", tally.DefaultBuckets) + defer metrics.SearchRequests.Tagged(tags).Histogram("histogram", tally.DefaultBuckets).Start().Stop() + } + err := f(ctx) + if err == nil { + // Request was ok + metrics.SearchRequests.Tagged(tags).Counter("ok").Inc(1) + return + } + if config.DefaultConfig.Metrics.Search.Counters { + metrics.SearchErrorRequests.Tagged(tags).Counter("unknown").Inc(1) + } +} + func (m *storeImplWithMetrics) CreateCollection(ctx context.Context, schema *tsApi.CollectionSchema) (err error) { - m.measure(ctx, "CreateCollection", func() error { + m.measure(ctx, "CreateCollection", func(ctx context.Context) error { err = m.s.CreateCollection(ctx, schema) return err }) @@ -47,7 +70,7 @@ func (m *storeImplWithMetrics) CreateCollection(ctx context.Context, schema *tsA } func (m *storeImplWithMetrics) UpdateCollection(ctx context.Context, name string, schema *tsApi.CollectionUpdateSchema) (err error) { - m.measure(ctx, "UpdateCollection", func() error { + m.measure(ctx, "UpdateCollection", func(ctx context.Context) error { err = m.s.UpdateCollection(ctx, name, schema) return err }) @@ -55,7 +78,7 @@ func (m *storeImplWithMetrics) UpdateCollection(ctx context.Context, name string } func (m *storeImplWithMetrics) DropCollection(ctx context.Context, table string) (err error) { - m.measure(ctx, "DropCollection", func() error { + m.measure(ctx, "DropCollection", func(ctx context.Context) error { err = m.s.DropCollection(ctx, table) return err }) @@ -63,7 +86,7 @@ func (m *storeImplWithMetrics) DropCollection(ctx context.Context, table string) } func (m *storeImplWithMetrics) IndexDocuments(ctx context.Context, table string, documents io.Reader, options IndexDocumentsOptions) (err error) { - m.measure(ctx, "IndexDocuments", func() error { + m.measure(ctx, "IndexDocuments", func(ctx context.Context) error { err = m.s.IndexDocuments(ctx, table, documents, options) return err }) @@ -71,7 +94,7 @@ func (m *storeImplWithMetrics) IndexDocuments(ctx context.Context, table string, } func (m *storeImplWithMetrics) DeleteDocuments(ctx context.Context, table string, key string) (err error) { - m.measure(ctx, "DeleteDocuments", func() error { + m.measure(ctx, "DeleteDocuments", func(ctx context.Context) error { err = m.s.DeleteDocuments(ctx, table, key) return err }) @@ -79,31 +102,13 @@ func (m *storeImplWithMetrics) DeleteDocuments(ctx context.Context, table string } func (m *storeImplWithMetrics) Search(ctx context.Context, table string, query *qsearch.Query, pageNo int) (result []tsApi.SearchResult, err error) { - m.measure(ctx, "Search", func() error { + m.measure(ctx, "Search", func(ctx context.Context) error { result, err = m.s.Search(ctx, table, query, pageNo) return err }) return } -func (m *storeImplWithMetrics) measure(ctx context.Context, name string, f func() error) { - // Low level measurement wrapper that is called by the measure functions on the appropriate receiver - tags := metrics.GetSearchTags(ctx, name) - if config.DefaultConfig.Metrics.Search.ResponseTime { - metrics.SearchRequests.Tagged(tags).Histogram("histogram", tally.DefaultBuckets) - defer metrics.SearchRequests.Tagged(tags).Histogram("histogram", tally.DefaultBuckets).Start().Stop() - } - err := f() - if err == nil { - // Request was ok - metrics.SearchRequests.Tagged(tags).Counter("ok").Inc(1) - return - } - if config.DefaultConfig.Metrics.Search.Counters { - metrics.SearchErrorRequests.Tagged(tags).Counter("unknown").Inc(1) - } -} - type IndexDocumentsOptions struct { Action string BatchSize int diff --git a/test/docker/docker-compose.yml b/test/docker/docker-compose.yml index e22c81354..04204ae26 100644 --- a/test/docker/docker-compose.yml +++ b/test/docker/docker-compose.yml @@ -41,6 +41,8 @@ services: - TIGRIS_ENVIRONMENT=test - TIGRIS_SERVER_SEARCH_AUTH_KEY=ts_test_key - TIGRIS_SERVER_SEARCH_HOST=tigris_search + - TIGRIS_SERVER_LOG_FORMAT=console + - TIGRIS_SERVER_CDC_ENABLED=true build: context: ../../ dockerfile: docker/Dockerfile @@ -64,6 +66,8 @@ services: - TIGRIS_ENVIRONMENT=test - TIGRIS_SERVER_SEARCH_AUTH_KEY=ts_test_key - TIGRIS_SERVER_SEARCH_HOST=tigris_search + - TIGRIS_SERVER_LOG_FORMAT=console + - TIGRIS_SERVER_CDC_ENABLED=true build: context: ../../ dockerfile: docker/Dockerfile diff --git a/test/v1/client/client_test.go b/test/v1/client/client_test.go index 371412e3c..a40fb083d 100644 --- a/test/v1/client/client_test.go +++ b/test/v1/client/client_test.go @@ -45,8 +45,9 @@ func TestClientCollectionBasic(t *testing.T) { h, p := getTestServerHostPort() var db *tigris.Database var err error + cfg := &config.Database{Driver: config.Driver{URL: fmt.Sprintf("%v:%d", h, p)}} for { - db, err = tigris.OpenDatabase(ctx, &config.Database{Driver: config.Driver{URL: fmt.Sprintf("%v:%d", h, p)}}, "db111222", &Coll1{}, &Coll2{}) + db, err = tigris.OpenDatabase(ctx, cfg, "db111222", &Coll1{}, &Coll2{}) if err != nil && err.Error() == "transaction not committed due to conflict with another transaction" { continue } @@ -54,7 +55,7 @@ func TestClientCollectionBasic(t *testing.T) { break } defer func() { - require.NoError(t, db.Drop(ctx)) + require.NoError(t, tigris.DropDatabase(ctx, cfg, "db111222")) }() c := tigris.GetCollection[Coll1](db) @@ -123,8 +124,9 @@ func TestClientCollectionTx(t *testing.T) { h, p := getTestServerHostPort() var err error var db *tigris.Database + cfg := &config.Database{Driver: config.Driver{URL: fmt.Sprintf("%v:%d", h, p)}} for { - db, err = tigris.OpenDatabase(ctx, &config.Database{Driver: config.Driver{URL: fmt.Sprintf("%v:%d", h, p)}}, "db111333", &Coll1{}) + db, err = tigris.OpenDatabase(ctx, cfg, "db111333", &Coll1{}) if err != nil && err.Error() == "transaction not committed due to conflict with another transaction" { continue } @@ -133,7 +135,7 @@ func TestClientCollectionTx(t *testing.T) { } defer func() { - _ = db.Drop(ctx) + _ = tigris.DropDatabase(ctx, cfg, "db111333") }() err = db.Tx(ctx, func(ctx context.Context) error { diff --git a/test/v1/server/admin_test.go b/test/v1/server/admin_test.go index a826d7fb7..82b9cd45d 100644 --- a/test/v1/server/admin_test.go +++ b/test/v1/server/admin_test.go @@ -2,60 +2,55 @@ package server import ( "fmt" + "math/rand" "net/http" "testing" - "github.com/stretchr/testify/suite" + "github.com/stretchr/testify/assert" "github.com/tigrisdata/tigris/test/config" "gopkg.in/gavv/httpexpect.v1" ) type AdminTestMap map[string]interface{} -type AdminSuite struct { - suite.Suite +func TestCreateNamespace(t *testing.T) { + name := fmt.Sprintf("namespace-a-%x", rand.Int63()) + id := rand.Int31() + resp := createNamespace(t, name, id) + resp.Status(http.StatusOK). + JSON(). + Object(). + ValueEqual("message", fmt.Sprintf("Namespace created, with id=%d, and name=%s", id, name)) } -func (s *AdminSuite) TestCreateNamespace() { - s.Run("create_namespace", func() { - resp := createNamespace(s.T(), "namespace-a", 2) - resp.Status(http.StatusOK). - JSON(). - Object(). - ValueEqual("message", "Namespace created, with id=2, and name=namespace-a") - }) -} -func adminExpectLow(s httpexpect.LoggerReporter, url string) *httpexpect.Expect { +func adminExpect(s httpexpect.LoggerReporter) *httpexpect.Expect { return httpexpect.WithConfig(httpexpect.Config{ - BaseURL: url, + BaseURL: config.GetBaseURL(), Reporter: httpexpect.NewAssertReporter(s), }) } -func adminExpect(s httpexpect.LoggerReporter) *httpexpect.Expect { - return adminExpectLow(s, config.GetBaseURL()) -} - -func (s *AdminSuite) TestListNamespaces() { - s.Run("list_namespaces", func() { - _ = createNamespace(s.T(), "namespace-b", 3) - resp := listNamespaces(s.T()) - namespaces := resp.Status(http.StatusOK). - JSON(). - Object(). - Value("namespaces"). - Array(). - Raw() - var found = false - for _, namespace := range namespaces { - if converted, ok := namespace.(map[string]interface{}); ok { - if converted["name"] == "namespace-b" { - found = true - } +func TestListNamespaces(t *testing.T) { + name := fmt.Sprintf("namespace-b-%x", rand.Int63()) + id := rand.Int31() + _ = createNamespace(t, name, id) + resp := listNamespaces(t) + namespaces := resp.Status(http.StatusOK). + JSON(). + Object(). + Value("namespaces"). + Array(). + Raw() + var found = false + for _, namespace := range namespaces { + if converted, ok := namespace.(map[string]interface{}); ok { + if converted["name"] == name { + found = true + assert.Equal(t, float64(id), converted["id"]) } } - s.True(found) - }) + } + assert.True(t, found) } func createNamespace(t *testing.T, namespaceName string, namespaceId int32) *httpexpect.Response { @@ -75,6 +70,7 @@ func listNamespaces(t *testing.T) *httpexpect.Response { func getCreateNamespaceURL(namespaceName string) string { return fmt.Sprintf("/admin/v1/namespaces/%s/create", namespaceName) } + func listNamespaceUrl() string { return "/admin/v1/namespaces/list" } diff --git a/test/v1/server/collection_test.go b/test/v1/server/collection_test.go index c1f0bebb6..80337b876 100644 --- a/test/v1/server/collection_test.go +++ b/test/v1/server/collection_test.go @@ -232,7 +232,8 @@ func (s *CollectionSuite) TestDescribeCollection() { resp.Status(http.StatusOK). JSON(). Object(). - ValueEqual("collection", "test_collection") + ValueEqual("collection", "test_collection"). + ValueEqual("size", 0) // cleanup dropCollection(s.T(), s.database, "test_collection") diff --git a/test/v1/server/database_test.go b/test/v1/server/database_test.go index 1163cc0bf..983d73af6 100644 --- a/test/v1/server/database_test.go +++ b/test/v1/server/database_test.go @@ -21,42 +21,39 @@ import ( "net/http" "testing" - "github.com/stretchr/testify/suite" api "github.com/tigrisdata/tigris/api/server/v1" "gopkg.in/gavv/httpexpect.v1" ) -type DatabaseSuite struct { - suite.Suite -} - func getDatabaseURL(databaseName string, methodName string) string { return fmt.Sprintf("/api/v1/databases/%s/%s", databaseName, methodName) } -func (s *DatabaseSuite) TestCreateDatabase() { - resp := createDatabase(s.T(), "test_db") +func TestCreateDatabase(t *testing.T) { + resp := createDatabase(t, "test_db") resp.Status(http.StatusOK). JSON(). Object(). ValueEqual("message", "database created successfully") } -func (s *DatabaseSuite) TestDescribeDatabase() { - resp := describeDatabase(s.T(), "test_db") +func TestDescribeDatabase(t *testing.T) { + createCollection(t, "test_db", "test_collection", testCreateSchema).Status(http.StatusOK) + resp := describeDatabase(t, "test_db") resp.Status(http.StatusOK). JSON(). Object(). - ValueEqual("db", "test_db") + ValueEqual("db", "test_db"). + ValueEqual("size", 0) } -func (s *DatabaseSuite) TestDropDatabase_NotFound() { - resp := dropDatabase(s.T(), "test_drop_db_not_found") +func TestDropDatabase_NotFound(t *testing.T) { + resp := dropDatabase(t, "test_drop_db_not_found") testError(resp, http.StatusNotFound, api.Code_NOT_FOUND, "database doesn't exist 'test_drop_db_not_found'") } -func (s *DatabaseSuite) TestDropDatabase() { - resp := dropDatabase(s.T(), "test_db") +func TestDropDatabase(t *testing.T) { + resp := dropDatabase(t, "test_db") resp.Status(http.StatusOK). JSON(). Object(). diff --git a/test/v1/server/integration_test.go b/test/v1/server/integration_test.go index 9cc813e36..7fcfffec8 100644 --- a/test/v1/server/integration_test.go +++ b/test/v1/server/integration_test.go @@ -21,6 +21,7 @@ import ( "math/rand" "os" "testing" + "time" "github.com/stretchr/testify/suite" api "github.com/tigrisdata/tigris/api/server/v1" @@ -29,8 +30,6 @@ import ( func SetupSuites() []suite.TestingSuite { var suites []suite.TestingSuite - suites = append(suites, &DatabaseSuite{}) - suites = append(suites, &CollectionSuite{ database: fmt.Sprintf("integration_db1_%x", rand.Uint64()), }) @@ -40,7 +39,6 @@ func SetupSuites() []suite.TestingSuite { collection: "test_collection", }) - suites = append(suites, &AdminSuite{}) return suites } @@ -54,6 +52,7 @@ func TestServer(t *testing.T) { } func TestMain(m *testing.M) { + rand.Seed(time.Now().Unix()) os.Exit(m.Run()) } diff --git a/util/log/log.go b/util/log/log.go index 0b850b5ac..36e104729 100644 --- a/util/log/log.go +++ b/util/log/log.go @@ -65,6 +65,20 @@ func Configure(config LogConfig) { } } +// E is a helper function to shortcut condition checking and logging +// in the case of error +// Used like this: +// +// if E(err) { +// return err +// } +// +// to replace: +// +// if err != nil { +// log.Msgf(err.Error()) +// return err +// } func E(err error) bool { if err == nil { return false @@ -75,6 +89,17 @@ func E(err error) bool { return true } +// CE is a helper to shortcut error creation and logging +// Used like this: +// +// return CE("msg, value %v", value) +// +// to replace: +// +// err := fmt.Errorf("msg, value %v", value) +// log.Msgf("msg, value %v", value) +// return err +// func CE(format string, args ...interface{}) error { err := fmt.Errorf(format, args...)