feat: Facet stats support for numeric fields (#409)

* feat: Facet stats support for numeric fields

* chore: Updating gh action env ubuntu to 22.04

* test: Fix marshaler test to consume pointer

* chore: Installing proto compiler from gh releases

api/server/v1/marshaler.go

@@ -501,11 +501,11 @@ func (x *SearchFacet) MarshalJSON() ([]byte, error) {
 
 func (x *FacetStats) MarshalJSON() ([]byte, error) {
 	resp := struct {
-		Avg   float64 `json:"avg,omitempty"`
-		Max   float64 `json:"max,omitempty"`
-		Min   float64 `json:"min,omitempty"`
-		Sum   float64 `json:"sum,omitempty"`
-		Count int64   `json:"count"`
+		Avg   *float64 `json:"avg,omitempty"`
+		Max   *float64 `json:"max,omitempty"`
+		Min   *float64 `json:"min,omitempty"`
+		Sum   *float64 `json:"sum,omitempty"`
+		Count int64    `json:"count"`
 	}{
 		Avg:   x.Avg,
 		Max:   x.Max,

api/server/v1/marshaler_test.go

@@ -47,6 +47,7 @@ func TestJSONEncoding(t *testing.T) {
 	})
 
 	t.Run("marshal SearchResponse", func(t *testing.T) {
+		avg := float64(40)
 		resp := &SearchResponse{
 			Hits: []*SearchHit{{
 				Data:     nil,
@@ -60,7 +61,7 @@ func TestJSONEncoding(t *testing.T) {
 					}},
 					Stats: &FacetStats{
 						Count: 50,
-						Avg:   40,
+						Avg:   &avg,
 					},
 				},
 			},

docker/Dockerfile

@@ -22,6 +22,8 @@ RUN apt-get update && \
     curl \
     gcc \
     git \
+    unzip \
+    libc6-dev \
     make \
     sudo
 

docker/Dockerfile.local

@@ -21,6 +21,8 @@ RUN apt-get update && \
     curl \
     wget \
     gcc \
+    libc6-dev \
+    unzip \
     git \
     make \
     sudo

go.mod

@@ -29,7 +29,7 @@ require (
 	github.com/spf13/viper v1.12.0
 	github.com/stretchr/testify v1.7.2
 	github.com/tigrisdata/tigris-client-go v1.0.0-alpha.21
-	github.com/typesense/typesense-go v0.5.0
+	github.com/typesense/typesense-go v0.6.0
 	github.com/uber-go/tally v3.5.0+incompatible
 	github.com/ugorji/go/codec v1.2.7
 	github.com/valyala/bytebufferpool v1.0.0

go.sum

@@ -873,8 +873,8 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri
 github.com/twitchtv/twirp v8.1.1+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A=
 github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg=
 github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
-github.com/typesense/typesense-go v0.5.0 h1:3uit/Ku1k0wtUDjN5HfFMIoR55pEeL7gcc+fL8c+xNI=
-github.com/typesense/typesense-go v0.5.0/go.mod h1:F9T3neLDqRr9ufFNhv1y0Qxe1Zs1GT85JlgijSjtKFo=
+github.com/typesense/typesense-go v0.6.0 h1:QcQ+1WGqFDyBhjP7Cd8mj7cs00zFTs6uBrB2u6we78o=
+github.com/typesense/typesense-go v0.6.0/go.mod h1:F9T3neLDqRr9ufFNhv1y0Qxe1Zs1GT85JlgijSjtKFo=
 github.com/uber-go/tally v3.5.0+incompatible h1:2vIkqVrSaspifqcJh2yQjQqqpfavvmfj/ognDrBxuSg=
 github.com/uber-go/tally v3.5.0+incompatible/go.mod h1:YDTIBxdXyOU/sCWilKB4bgyufu1cEi0jdVnRdxvjnmU=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=

scripts/install_build_deps.sh

@@ -16,6 +16,8 @@ set -ex
 
 # Settings
 FDB_VERSION=7.1.7
+PROTO_VERSION=3.15.8
+PROTO_RELEASES="https://github.com/protocolbuffers/protobuf/releases"
 
 ### Prereqs checks ###
 # Check if architecture and OS is supported
@@ -24,74 +26,98 @@ ARCH=$(uname -m)
 OS=$(uname -s)
 
 case "${OS}-${ARCH}" in
-	"Darwin-arm64")
-		BINARIES="brew curl go"
-		FDB_SHA=b456a1d03580f81394502e1b066006ec38bf6a3a17a9904e6d7a88badbea4b4a08b9dbf69fbb0057b46dd5043f23de9ec3ff7a77d767c23f872425eb73fdee18
-		;;
-	"Darwin-x86_64")
-		BINARIES="brew curl go"
-		FDB_SHA=0e9d147410eede58d121fdc9208ea7b04a7c74c8f3776f56cfc00233cfb3a358a0e2f992122718ef9c639928b081801da542e9c4b07a539c8fd73361ab43beec
-		;;
-	"Linux-aarch64")
-		BINARIES="apt-get curl go"
-		FDB_SHA=c994ebb01a660cff9ef699a0e38482a561679db04c02c256efae25ba687cf903
-		;;
-	"Linux-arm64")
-		BINARIES="apt-get curl go"
-		FDB_SHA=c994ebb01a660cff9ef699a0e38482a561679db04c02c256efae25ba687cf903
-		;;
-	"Linux-x86_64")
-		BINARIES="apt-get curl go"
-		FDB_SHA=471f6bf4a7af40abc69027aa0d4f452ee83715a43a555008303ca255f6bd6db1
-		;;
-	*)
-		echo "Unsupported architecture ${ARCH} or operating system ${OS}."
-		exit 1
+"Darwin-arm64")
+  BINARIES="brew curl go"
+  FDB_SHA=b456a1d03580f81394502e1b066006ec38bf6a3a17a9904e6d7a88badbea4b4a08b9dbf69fbb0057b46dd5043f23de9ec3ff7a77d767c23f872425eb73fdee18
+  ;;
+"Darwin-x86_64")
+  BINARIES="brew curl go"
+  FDB_SHA=0e9d147410eede58d121fdc9208ea7b04a7c74c8f3776f56cfc00233cfb3a358a0e2f992122718ef9c639928b081801da542e9c4b07a539c8fd73361ab43beec
+  ;;
+"Linux-aarch64")
+  BINARIES="apt-get curl go"
+  FDB_SHA=c994ebb01a660cff9ef699a0e38482a561679db04c02c256efae25ba687cf903
+  ;;
+"Linux-arm64")
+  BINARIES="apt-get curl go"
+  FDB_SHA=c994ebb01a660cff9ef699a0e38482a561679db04c02c256efae25ba687cf903
+  ;;
+"Linux-x86_64")
+  BINARIES="apt-get curl go"
+  FDB_SHA=471f6bf4a7af40abc69027aa0d4f452ee83715a43a555008303ca255f6bd6db1
+  ;;
+*)
+  echo "Unsupported architecture ${ARCH} or operating system ${OS}."
+  exit 1
+  ;;
 esac
 
 # Check if required binaries are available in PATH
 for bin in ${BINARIES}; do
-	binpath=$(command -v "${bin}")
-	if [ -z "${binpath}" ] || ! test -x "${binpath}"; then
-		echo "Please ensure that $bin binary is installed and in PATH."
-		exit 1
-	fi
+  binpath=$(command -v "${bin}")
+  if [ -z "${binpath}" ] || ! test -x "${binpath}"; then
+    echo "Please ensure that $bin binary is installed and in PATH."
+    exit 1
+  fi
 done
 
-# Install protobuf compiler via package manager
+# Install protobuf compiler
 case "${OS}" in
-	"Darwin")
-		brew install protobuf
-		;;
-	"Linux")
-		sudo apt-get update
-		sudo apt-get install -y protobuf-compiler
-		;;
+"Darwin")
+  brew install protobuf
+  ;;
+"Linux")
+  case "${ARCH}" in
+  "x86_64")
+    PROTO_PKG=protoc-$PROTO_VERSION-linux-x86_64.zip
+    ;;
+  "aarch64")
+    PROTO_PKG=protoc-$PROTO_VERSION-linux-aarch_64.zip
+    ;;
+  *)
+    echo "No supported proto compiler for ${ARCH} or operating system ${OS}."
+    exit 1
+    ;;
+  esac
+  ;;
+*)
+  echo "No supported proto compiler for ${ARCH} or operating system ${OS}."
+  exit 1
+  ;;
 esac
 
+if [ -n "$PROTO_PKG" ]; then
+  DOWNLOAD_URL=$PROTO_RELEASES/download/v$PROTO_VERSION/$PROTO_PKG
+  echo "Fetching protobuf release ${DOWNLOAD_URL}"
+  curl -LO $DOWNLOAD_URL
+  sudo unzip $PROTO_PKG -d "/usr/local/"
+  sudo chmod +x "/usr/local/bin/protoc"
+  sudo chmod -R 755 "/usr/local/include/"
+fi
+
 # Install protobuf
 export GO111MODULE=on
 go install google.golang.org/protobuf/cmd/protoc-gen-go@v1
 go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1
 go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway@v2
 go install github.com/google/gnostic/cmd/protoc-gen-openapi@v0 #generate openapi 3.0 spec
 go install github.com/deepmap/oapi-codegen/cmd/oapi-codegen@v1 #generate go http client
-go install github.com/mikefarah/yq/v4@latest # used to fix OpenAPI spec in scripts/fix_openapi.sh
+go install github.com/mikefarah/yq/v4@latest                   # used to fix OpenAPI spec in scripts/fix_openapi.sh
 
 # Install FoundationDB package
 case "${OS}" in
-	"Darwin")
-		FDB_PACKAGE_NAME="FoundationDB-${FDB_VERSION}_${ARCH}.pkg"
-		FDB_PACKAGE_PATH="$(mktemp -d)/${FDB_PACKAGE_NAME}"
-		curl --create-dirs -Lo "$FDB_PACKAGE_PATH" "https://tigrisdata-pub.s3.us-west-2.amazonaws.com/${FDB_PACKAGE_NAME}"
-		echo "$FDB_SHA  $FDB_PACKAGE_PATH" | shasum -a 512 -c
-		sudo installer -pkg "$FDB_PACKAGE_PATH" -target /
-		;;
-	"Linux")
-		FDB_PACKAGE_NAME="foundationdb-clients_${FDB_VERSION}-1_${ARCH}.deb"
-		FDB_PACKAGE_PATH="$(mktemp -p /tmp/ -u)/${FDB_PACKAGE_NAME}"
-		curl --create-dirs -Lo "$FDB_PACKAGE_PATH" "https://tigrisdata-pub.s3.us-west-2.amazonaws.com/ubuntu/focal/${FDB_PACKAGE_NAME}"
-		echo "$FDB_SHA $FDB_PACKAGE_PATH" | sha256sum -c
-		sudo dpkg -i "$FDB_PACKAGE_PATH" # provides /lib/libfdb_c.so shared library in the docker for CGO
-		;;
+"Darwin")
+  FDB_PACKAGE_NAME="FoundationDB-${FDB_VERSION}_${ARCH}.pkg"
+  FDB_PACKAGE_PATH="$(mktemp -d)/${FDB_PACKAGE_NAME}"
+  curl --create-dirs -Lo "$FDB_PACKAGE_PATH" "https://tigrisdata-pub.s3.us-west-2.amazonaws.com/${FDB_PACKAGE_NAME}"
+  echo "$FDB_SHA  $FDB_PACKAGE_PATH" | shasum -a 512 -c
+  sudo installer -pkg "$FDB_PACKAGE_PATH" -target /
+  ;;
+"Linux")
+  FDB_PACKAGE_NAME="foundationdb-clients_${FDB_VERSION}-1_${ARCH}.deb"
+  FDB_PACKAGE_PATH="$(mktemp -p /tmp/ -u)/${FDB_PACKAGE_NAME}"
+  curl --create-dirs -Lo "$FDB_PACKAGE_PATH" "https://tigrisdata-pub.s3.us-west-2.amazonaws.com/ubuntu/focal/${FDB_PACKAGE_NAME}"
+  echo "$FDB_SHA $FDB_PACKAGE_PATH" | sha256sum -c
+  sudo dpkg -i "$FDB_PACKAGE_PATH" # provides /lib/libfdb_c.so shared library in the docker for CGO
+  ;;
 esac

server/services/v1/query_runner.go

@@ -677,10 +677,7 @@ func (runner *SearchQueryRunner) getFacetFields(collFields []*schema.QueryableFi
 				continue
 			}
 			if !cf.Faceted {
-				return qsearch.Facets{}, api.Errorf(api.Code_INVALID_ARGUMENT, "Faceting not enabled for `%s`", ff.Name)
-			}
-			if cf.DataType != schema.StringType {
-				return qsearch.Facets{}, api.Errorf(api.Code_INVALID_ARGUMENT, "Cannot generate facets for `%s`. Faceting is only supported for text fields", ff.Name)
+				return qsearch.Facets{}, api.Errorf(api.Code_INVALID_ARGUMENT, "Cannot generate facets for `%s`. Faceting is only supported for numeric and text fields", ff.Name)
 			}
 			found = true
 			break

server/services/v1/query_runner_test.go

@@ -37,7 +37,7 @@ func TestSearchQueryRunner_getFacetFields(t *testing.T) {
 	t.Run("requested facet field is not faceted in collection", func(t *testing.T) {
 		runner.req.Facet = []byte(`{"parent.field_2":{"size":10},"field_3":{"size":10}}`)
 		facets, err := runner.getFacetFields(collFields)
-		assert.ErrorContains(t, err, "Faceting not enabled for `field_3`")
+		assert.ErrorContains(t, err, "only supported for numeric and text fields")
 		assert.NotNil(t, facets)
 		assert.Empty(t, facets.Fields)
 	})
@@ -50,14 +50,6 @@ func TestSearchQueryRunner_getFacetFields(t *testing.T) {
 		assert.Empty(t, facets.Fields)
 	})
 
-	t.Run("requested facet fields are not of String datatype", func(t *testing.T) {
-		runner.req.Facet = []byte(`{"field_1":{"size":10},"field_4":{"size":10}}`)
-		facets, err := runner.getFacetFields(collFields)
-		assert.ErrorContains(t, err, "Cannot generate facets for `field_4`")
-		assert.NotNil(t, facets)
-		assert.Empty(t, facets.Fields)
-	})
-
 	t.Run("valid facet fields requested", func(t *testing.T) {
 		runner.req.Facet = []byte(`{"field_1":{"size":10},"parent.field_2":{"size":10}}`)
 		facets, err := runner.getFacetFields(collFields)

server/services/v1/search_reader.go

@@ -233,16 +233,20 @@ func (p *pageReader) buildStats(stats tsApi.FacetCounts) *api.FacetStats {
 
 	var stat = &api.FacetStats{}
 	if stats.Stats.Avg != nil {
-		stat.Avg = float64(*stats.Stats.Avg)
+		avg := float64(*stats.Stats.Avg)
+		stat.Avg = &avg
 	}
 	if stats.Stats.Min != nil {
-		stat.Min = float64(*stats.Stats.Min)
+		min := float64(*stats.Stats.Min)
+		stat.Min = &min
 	}
 	if stats.Stats.Max != nil {
-		stat.Max = float64(*stats.Stats.Max)
+		max := float64(*stats.Stats.Max)
+		stat.Max = &max
 	}
 	if stats.Stats.Sum != nil {
-		stat.Sum = float64(*stats.Stats.Sum)
+		sum := float64(*stats.Stats.Sum)
+		stat.Sum = &sum
 	}
 	if stats.Stats.TotalValues != nil {
 		stat.Count = int64(*stats.Stats.TotalValues)

test/docker/Dockerfile

@@ -22,6 +22,8 @@ RUN apt-get update && \
     gcc \
     git \
     golang \
+    unzip \
+    libc6-dev \
     make \
     sudo