Merge pull request #388 from tigrisdata/main

api/server/v1/marshaler.go

@@ -118,9 +118,14 @@ func (x *SearchRequest) UnmarshalJSON(data []byte) error {
 		case "sort":
 			// delaying the sort deserialization
 			x.Sort = value
-		case "fields":
-			// not decoding it here and let it decode during fields parsing
-			x.Fields = value
+		case "include_fields":
+			if err := jsoniter.Unmarshal(value, &x.IncludeFields); err != nil {
+				return err
+			}
+		case "exclude_fields":
+			if err := jsoniter.Unmarshal(value, &x.ExcludeFields); err != nil {
+				return err
+			}
 		case "page_size":
 			if err := jsoniter.Unmarshal(value, &x.PageSize); err != nil {
 				return err
@@ -306,13 +311,15 @@ type collDesc struct {
 	Collection string              `json:"collection"`
 	Metadata   *CollectionMetadata `json:"metadata"`
 	Schema     json.RawMessage     `json:"schema"`
+	Size       int64               `json:"size"`
 }
 
 func (x *DescribeCollectionResponse) MarshalJSON() ([]byte, error) {
 	return json.Marshal(&collDesc{
 		Collection: x.Collection,
 		Metadata:   x.Metadata,
 		Schema:     x.Schema,
+		Size:       x.Size,
 	})
 }
 
@@ -321,16 +328,19 @@ func (x *DescribeDatabaseResponse) MarshalJSON() ([]byte, error) {
 		Db          string            `json:"db"`
 		Metadata    *DatabaseMetadata `json:"metadata"`
 		Collections []*collDesc       `json:"collections"`
+		Size        int64             `json:"size"`
 	}{
 		Db:       x.Db,
 		Metadata: x.Metadata,
+		Size:     x.Size,
 	}
 
 	for _, v := range x.Collections {
 		resp.Collections = append(resp.Collections, &collDesc{
 			Collection: v.Collection,
 			Metadata:   v.Metadata,
 			Schema:     v.Schema,
+			Size:       v.Size,
 		})
 	}
 

api/server/v1/marshaler_test.go

@@ -33,7 +33,7 @@ func TestJSONEncoding(t *testing.T) {
 	t.Run("unmarshal SearchRequest", func(t *testing.T) {
 		inputDoc := []byte(`{"q":"my search text","search_fields":["first_name","last_name"],
 							"filter":{"last_name":"Steve"},"facet":{"facet stat":0},
-							"sort":[{"salary":"$asc"}],"fields":["employment","history"]}`)
+							"sort":[{"salary":"$asc"}],"include_fields":["employment","history"]}`)
 
 		req := &SearchRequest{}
 		err := json.Unmarshal(inputDoc, req)
@@ -43,7 +43,7 @@ func TestJSONEncoding(t *testing.T) {
 		require.Equal(t, []byte(`{"last_name":"Steve"}`), req.GetFilter())
 		require.Equal(t, []byte(`{"facet stat":0}`), req.GetFacet())
 		require.Equal(t, []byte(`[{"salary":"$asc"}]`), req.GetSort())
-		require.Equal(t, []byte(`["employment","history"]`), req.GetFields())
+		require.Equal(t, []string{"employment", "history"}, req.GetIncludeFields())
 	})
 
 	t.Run("marshal SearchResponse", func(t *testing.T) {

api/server/v1/validator.go

@@ -103,6 +103,10 @@ func (x *SearchRequest) Validate() error {
 		return err
 	}
 
+	if len(x.IncludeFields) > 0 && len(x.ExcludeFields) > 0 {
+		return Errorf(Code_INVALID_ARGUMENT, "Cannot use both `include_fields` and `exclude_fields` together")
+	}
+
 	if err := isValidPaginationParam("page", int(x.Page)); err != nil {
 		return err
 	}

docker/Dockerfile.local

@@ -72,6 +72,7 @@ RUN echo "#!/bin/bash\n \
 	fdbserver --listen-address 127.0.0.1:4500 --public-address 127.0.0.1:4500 --datadir /var/lib/foundationdb/data --logdir /var/lib/foundationdb/logs --locality-zoneid tigris --locality-machineid tigris & \n\
 	export TIGRIS_SERVER_SEARCH_AUTH_KEY=ts_dev_key \n \
 	export TIGRIS_SERVER_SEARCH_HOST=localhost \n \
+	export TIGRIS_SERVER_CDC_ENABLED=true \n \
 	fdbcli --exec 'configure new single memory' \n \
 	/server/service\n" >/server/service.sh
 

go.mod

@@ -7,10 +7,8 @@ require (
 	github.com/auth0/go-jwt-middleware/v2 v2.0.1
 	github.com/buger/jsonparser v1.1.1
 	github.com/davecgh/go-spew v1.1.1
-	github.com/deepmap/oapi-codegen v1.11.0
 	github.com/fsnotify/fsnotify v1.5.4
 	github.com/fullstorydev/grpchan v1.1.1
-	github.com/getkin/kin-openapi v0.97.0
 	github.com/go-chi/chi/v5 v5.0.7
 	github.com/go-chi/cors v1.2.1
 	github.com/golang/protobuf v1.5.2
@@ -19,7 +17,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.2.0.20211207221722-a5b9e0b0458c
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20211207221722-a5b9e0b0458c
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.3
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/json-iterator/go v1.1.12
 	github.com/m3db/prometheus_client_golang v1.12.8
@@ -30,41 +28,42 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.12.0
 	github.com/stretchr/testify v1.7.2
-	github.com/tigrisdata/tigris-client-go v1.0.0-alpha.18
+	github.com/tigrisdata/tigris-client-go v1.0.0-alpha.20
 	github.com/typesense/typesense-go v0.5.0
 	github.com/uber-go/tally v3.5.0+incompatible
 	github.com/ugorji/go/codec v1.2.7
 	github.com/valyala/bytebufferpool v1.0.0
-	google.golang.org/genproto v0.0.0-20220615141314-f1464d18c36b
-	google.golang.org/grpc v1.47.0
+	google.golang.org/genproto v0.0.0-20220725144611-272f38e5d71b
+	google.golang.org/grpc v1.48.0
 	google.golang.org/protobuf v1.28.0
-	gopkg.in/DataDog/dd-trace-go.v1 v1.38.1
+	gopkg.in/DataDog/dd-trace-go.v1 v1.40.1
 	gopkg.in/gavv/httpexpect.v1 v1.1.3
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
-	cloud.google.com/go/compute v1.6.1 // indirect
-	github.com/DataDog/datadog-agent/pkg/obfuscate v0.0.0-20211129110424-6491aa3bf583 // indirect
-	github.com/DataDog/datadog-go v4.8.2+incompatible // indirect
-	github.com/DataDog/datadog-go/v5 v5.0.2 // indirect
+	cloud.google.com/go/compute v1.7.0 // indirect
+	github.com/DataDog/datadog-agent/pkg/obfuscate v0.38.0 // indirect
+	github.com/DataDog/datadog-go/v5 v5.1.1 // indirect
 	github.com/DataDog/gostackparse v0.5.0 // indirect
-	github.com/DataDog/sketches-go v1.0.0 // indirect
-	github.com/Microsoft/go-winio v0.5.1 // indirect
+	github.com/DataDog/sketches-go v1.4.1 // indirect
+	github.com/Microsoft/go-winio v0.5.2 // indirect
 	github.com/ajg/form v1.5.1 // indirect
 	github.com/andybalholm/brotli v1.0.4 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/deepmap/oapi-codegen v1.11.0 // indirect
 	github.com/dgraph-io/ristretto v0.1.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 // indirect
 	github.com/gertd/go-pluralize v0.2.1 // indirect
+	github.com/getkin/kin-openapi v0.97.0 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/swag v0.21.1 // indirect
 	github.com/golang/glog v1.0.0 // indirect
 	github.com/google/go-querystring v1.0.0 // indirect
-	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
+	github.com/google/pprof v0.0.0-20220608213341-c488b8fa1db3 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/iancoleman/strcase v0.2.0 // indirect
@@ -90,15 +89,15 @@ require (
 	github.com/pelletier/go-toml/v2 v2.0.2 // indirect
 	github.com/philhofer/fwd v1.1.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/sergi/go-diff v1.0.0 // indirect
 	github.com/smartystreets/goconvey v1.7.2 // indirect
 	github.com/sony/gobreaker v0.5.0 // indirect
-	github.com/spf13/afero v1.8.2 // indirect
+	github.com/spf13/afero v1.9.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/subosito/gotenv v1.4.0 // indirect
-	github.com/tinylib/msgp v1.1.2 // indirect
+	github.com/tinylib/msgp v1.1.6 // indirect
 	github.com/twmb/murmur3 v1.1.6 // indirect
 	github.com/valyala/fasthttp v1.34.0 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
@@ -109,12 +108,12 @@ require (
 	github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect
 	github.com/yudai/pp v2.0.1+incompatible // indirect
 	go.uber.org/atomic v1.9.0 // indirect
-	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
-	golang.org/x/net v0.0.0-20220615171555-694bf12d69de // indirect
-	golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb // indirect
-	golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect
+	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
+	golang.org/x/net v0.0.0-20220726230323-06994584191e // indirect
+	golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c // indirect
+	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
+	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/grpc/examples v0.0.0-20220215234149-ec717cad7395 // indirect