Merge pull request #317 from tigrisdata/main

Alpha release

Makefile

@@ -60,7 +60,12 @@ local_test: generate
 	go test $(TEST_PARAM) ./...
 
 run: clean generate
-	$(DOCKER_COMPOSE) up --build --detach tigris_server
+	$(DOCKER_COMPOSE) up --build --detach tigris_server2
+
+local_run: server
+	$(DOCKER_COMPOSE) up --no-build --detach tigris_search tigris_fdb
+	fdbcli -C ./test/config/fdb.cluster --exec "configure new single memory"
+	TIGRIS_ENVIRONMENT=dev ./server/service
 
 # Runs tigris server and foundationdb, plus additional tools for it like:
 # - prometheus and grafana for monitoring

README.md

@@ -61,24 +61,27 @@ make run
 This would bring dependencies and server up in the docker containers with all
 your changes.
 
-### Running tests in docker containers
+Alternatively, you can run `make run_full` to bring up monitoring tools as well.
+  * Graphana: http://localhost:3000
+  * Prometheus: http://localhost:9090
+
+### Running tests
+
+#### Run in the docker container
 
 Tests are executed using `make test`. This runs both unit and integration
-tests. The FoundationDB container auto-generates the cluster file which is
-shared between containers using a docker volume.
+tests in the docker containers.
 
-The docker volume `fdbdata` is mounted at:
+#### Run in the IDE
 
-* /var/fdb/fdb.cluster in tigris_fdb container
-* /etc/foundationdb/fdb.cluster in tigris_test and tigris_server containers,
-  which is the default location where the client will search for the cluster
-  file
+Run `make run` to bring the server up in the docker container.
+Now you can run individual tests in the IDE of your choice.
+Entire test suite can be run using `make local_test`.
 
-You can run the test on your local machine as follows:
+#### Debugging the server in the IDE
 
-```shell
-make osx_test
-```
+Run `make local_run` to start Tigris server on the host.
+Now you can attach to the process and debug from the IDE.
 
 # License
 

api/server/v1/marshaler.go

@@ -53,52 +53,6 @@ func (c *CustomMarshaler) Marshal(v interface{}) ([]byte, error) {
 	return c.JSONBuiltin.Marshal(v)
 }
 
-// MarshalJSON on read response avoid any encoding/decoding on x.Data. With this approach we are not doing any extra
-// marshaling/unmarshalling in returning the data from the database. The document returned from the database is stored
-// in x.Data and will return as-is.
-//
-// Note: This also means any changes in ReadResponse proto needs to make sure that we add that here and similarly
-// the openAPI specs needs to be specified Data as object instead of bytes.
-func (x *ReadResponse) MarshalJSON() ([]byte, error) {
-	resp := struct {
-		Data        json.RawMessage `json:"data,omitempty"`
-		Metadata    Metadata        `json:"metadata,omitempty"`
-		ResumeToken []byte          `json:"resume_token,omitempty"`
-	}{
-		Data:        x.Data,
-		Metadata:    CreateMDFromResponseMD(x.Metadata),
-		ResumeToken: x.ResumeToken,
-	}
-	return json.Marshal(resp)
-}
-
-type Metadata struct {
-	CreatedAt *time.Time `json:"created_at,omitempty"`
-	UpdatedAt *time.Time `json:"updated_at,omitempty"`
-	DeletedAt *time.Time `json:"deleted_at,omitempty"`
-}
-
-func CreateMDFromResponseMD(x *ResponseMetadata) Metadata {
-	var md Metadata
-	if x == nil {
-		return md
-	}
-	if x.CreatedAt != nil {
-		tm := x.CreatedAt.AsTime()
-		md.CreatedAt = &tm
-	}
-	if x.UpdatedAt != nil {
-		tm := x.UpdatedAt.AsTime()
-		md.UpdatedAt = &tm
-	}
-	if x.DeletedAt != nil {
-		tm := x.DeletedAt.AsTime()
-		md.DeletedAt = &tm
-	}
-
-	return md
-}
-
 // UnmarshalJSON on ReadRequest avoids unmarshalling filter and instead this way we can write a custom struct to do
 // the unmarshalling and will be avoiding any extra allocation/copying.
 func (x *ReadRequest) UnmarshalJSON(data []byte) error {
@@ -131,6 +85,55 @@ func (x *ReadRequest) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
+// UnmarshalJSON for SearchRequest
+func (x *SearchRequest) UnmarshalJSON(data []byte) error {
+	var mp map[string]jsoniter.RawMessage
+	if err := jsoniter.Unmarshal(data, &mp); err != nil {
+		return nil
+	}
+	for key, value := range mp {
+		switch key {
+		case "db":
+			if err := jsoniter.Unmarshal(value, &x.Db); err != nil {
+				return err
+			}
+		case "collection":
+			if err := jsoniter.Unmarshal(value, &x.Collection); err != nil {
+				return err
+			}
+		case "search_fields":
+			if err := jsoniter.Unmarshal(value, &x.SearchFields); err != nil {
+				return err
+			}
+		case "q":
+			if err := jsoniter.Unmarshal(value, &x.Q); err != nil {
+				return err
+			}
+		case "filter":
+			// not decoding it here and let it decode during filter parsing
+			x.Filter = value
+		case "facet":
+			// delaying the facet deserialization to dedicated handler
+			x.Facet = value
+		case "sort":
+			// delaying the sort deserialization
+			x.Sort = value
+		case "fields":
+			// not decoding it here and let it decode during fields parsing
+			x.Fields = value
+		case "page_size":
+			if err := jsoniter.Unmarshal(value, &x.PageSize); err != nil {
+				return err
+			}
+		case "page":
+			if err := jsoniter.Unmarshal(value, &x.Page); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 // UnmarshalJSON on InsertRequest avoids unmarshalling user document. We only need to extract primary/index keys from
 // the document and want to store the document as-is in the database. This way there is no extra cost of serialization/deserialization
 // and also less error-prone because we are not touching the user document. The req handler needs to extract out
@@ -334,7 +337,7 @@ func (x *DescribeDatabaseResponse) MarshalJSON() ([]byte, error) {
 	return json.Marshal(&resp)
 }
 
-func (x *StreamResponse) MarshalJSON() ([]byte, error) {
+func (x *EventsResponse) MarshalJSON() ([]byte, error) {
 	type event struct {
 		TxId       []byte          `json:"tx_id"`
 		Collection string          `json:"collection"`
@@ -394,3 +397,96 @@ func (x *DeleteResponse) MarshalJSON() ([]byte, error) {
 func (x *UpdateResponse) MarshalJSON() ([]byte, error) {
 	return json.Marshal(&dmlResponse{Metadata: CreateMDFromResponseMD(x.Metadata), Status: x.Status, ModifiedCount: x.ModifiedCount})
 }
+
+// MarshalJSON on read response avoid any encoding/decoding on x.Data. With this approach we are not doing any extra
+// marshaling/unmarshalling in returning the data from the database. The document returned from the database is stored
+// in x.Data and will return as-is.
+//
+// Note: This also means any changes in ReadResponse proto needs to make sure that we add that here and similarly
+// the openAPI specs needs to be specified Data as object instead of bytes.
+func (x *ReadResponse) MarshalJSON() ([]byte, error) {
+	resp := struct {
+		Data        json.RawMessage `json:"data,omitempty"`
+		Metadata    Metadata        `json:"metadata,omitempty"`
+		ResumeToken []byte          `json:"resume_token,omitempty"`
+	}{
+		Data:        x.Data,
+		Metadata:    CreateMDFromResponseMD(x.Metadata),
+		ResumeToken: x.ResumeToken,
+	}
+	return json.Marshal(resp)
+}
+
+func (x *SearchResponse) MarshalJSON() ([]byte, error) {
+	resp := struct {
+		Hits   []*SearchHit            `json:"hits,omitempty"`
+		Facets map[string]*SearchFacet `json:"facets,omitempty"`
+		Meta   *SearchMetadata         `json:"meta,omitempty"`
+	}{
+		Hits:   x.Hits,
+		Facets: x.Facets,
+		Meta:   x.Meta,
+	}
+	return json.Marshal(resp)
+}
+
+func (x *SearchHit) MarshalJSON() ([]byte, error) {
+	resp := struct {
+		Data     json.RawMessage   `json:"data,omitempty"`
+		Metadata SearchHitMetadata `json:"metadata,omitempty"`
+	}{
+		Data:     x.Data,
+		Metadata: CreateMDFromSearchMD(x.Metadata),
+	}
+	return json.Marshal(resp)
+}
+
+type SearchHitMetadata struct {
+	CreatedAt *time.Time `json:"created_at,omitempty"`
+	UpdatedAt *time.Time `json:"updated_at,omitempty"`
+	DeletedAt *time.Time `json:"deleted_at,omitempty"`
+}
+
+type Metadata struct {
+	CreatedAt *time.Time `json:"created_at,omitempty"`
+	UpdatedAt *time.Time `json:"updated_at,omitempty"`
+	DeletedAt *time.Time `json:"deleted_at,omitempty"`
+}
+
+func CreateMDFromResponseMD(x *ResponseMetadata) Metadata {
+	var md Metadata
+	if x == nil {
+		return md
+	}
+	if x.CreatedAt != nil {
+		tm := x.CreatedAt.AsTime()
+		md.CreatedAt = &tm
+	}
+	if x.UpdatedAt != nil {
+		tm := x.UpdatedAt.AsTime()
+		md.UpdatedAt = &tm
+	}
+	if x.DeletedAt != nil {
+		tm := x.DeletedAt.AsTime()
+		md.DeletedAt = &tm
+	}
+
+	return md
+}
+
+func CreateMDFromSearchMD(x *SearchHitMeta) SearchHitMetadata {
+	var md SearchHitMetadata
+	if x == nil {
+		return md
+	}
+	if x.CreatedAt != nil {
+		tm := x.CreatedAt.AsTime()
+		md.CreatedAt = &tm
+	}
+	if x.UpdatedAt != nil {
+		tm := x.UpdatedAt.AsTime()
+		md.UpdatedAt = &tm
+	}
+
+	return md
+}

api/server/v1/marshaler_test.go

@@ -21,12 +21,58 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestDocument(t *testing.T) {
+func TestJSONEncoding(t *testing.T) {
 	// ToDo: add marshaler tests
 	inputDoc := []byte(`{"pkey_int": 1, "int_value": 2, "str_value": "foo"}`)
 	b, err := json.Marshal(inputDoc)
 	require.NoError(t, err)
 
 	var bb []byte
 	require.NoError(t, json.Unmarshal(b, &bb))
+
+	t.Run("unmarshal SearchRequest", func(t *testing.T) {
+		inputDoc := []byte(`{"q":"my search text","search_fields":["first_name","last_name"],
+							"filter":{"last_name":"Steve"},"facet":{"facet stat":0},
+							"sort":[{"salary":"$asc"}],"fields":["employment","history"]}`)
+
+		req := &SearchRequest{}
+		err := json.Unmarshal(inputDoc, req)
+		require.NoError(t, err)
+		require.Equal(t, "my search text", req.GetQ())
+		require.Equal(t, []string{"first_name", "last_name"}, req.GetSearchFields())
+		require.Equal(t, []byte(`{"last_name":"Steve"}`), req.GetFilter())
+		require.Equal(t, []byte(`{"facet stat":0}`), req.GetFacet())
+		require.Equal(t, []byte(`[{"salary":"$asc"}]`), req.GetSort())
+		require.Equal(t, []byte(`["employment","history"]`), req.GetFields())
+	})
+
+	t.Run("marshal SearchResponse", func(t *testing.T) {
+		resp := &SearchResponse{
+			Hits: []*SearchHit{{
+				Data:     nil,
+				Metadata: &SearchHitMeta{},
+			}},
+			Facets: map[string]*SearchFacet{
+				"myField": {
+					Counts: []*FacetCount{{
+						Count: 32,
+						Value: "adidas",
+					}},
+					Stats: &FacetStats{
+						Count: 50,
+						Avg:   40,
+					},
+				},
+			},
+			Meta: &SearchMetadata{
+				Found: 1234,
+				Page: &Page{
+					Current: 2,
+					PerPage: 10,
+				},
+			}}
+		r, err := json.Marshal(resp)
+		require.NoError(t, err)
+		require.Equal(t, []byte(`{"hits":[{"metadata":{}}],"facets":{"myField":{"counts":[{"count":32,"value":"adidas"}],"stats":{"avg":40,"count":50}}},"meta":{"found":1234,"page":{"current":2,"per_page":10}}}`), r)
+	})
 }

api/server/v1/tx.go

@@ -49,7 +49,6 @@ func GetTransaction(ctx context.Context, req proto.Message) *TransactionCtx {
 func GetTransactionLegacy(req proto.Message) *TransactionCtx {
 	switch r := req.(type) {
 	case *InsertRequest:
-		r.GetOptions().ProtoReflect()
 		if r.GetOptions() == nil || r.GetOptions().GetWriteOptions() == nil {
 			return nil
 		}

api/server/v1/validator.go

@@ -98,6 +98,14 @@ func (x *ReadRequest) Validate() error {
 	return nil
 }
 
+func (x *SearchRequest) Validate() error {
+	if err := isValidCollectionAndDatabase(x.Collection, x.Db); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (x *CreateOrUpdateCollectionRequest) Validate() error {
 	if err := isValidCollectionAndDatabase(x.Collection, x.Db); err != nil {
 		return err
@@ -150,7 +158,7 @@ func (x *ListDatabasesRequest) Validate() error {
 	return nil
 }
 
-func (x *StreamRequest) Validate() error {
+func (x *EventsRequest) Validate() error {
 	if err := isValidDatabase(x.Db); err != nil {
 		return err
 	}

config/server.dev.yaml

@@ -0,0 +1,28 @@
+# Copyright 2022 Tigris Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+server:
+  host: localhost
+  port: 8081
+
+search:
+  host: localhost
+  port: 8108
+  auth_key: ts_test_key
+
+log:
+  level: trace
+
+foundationdb:
+  cluster_file: "./test/config/fdb.cluster"