From 49285cdfd05c3d4fb67ff0b6c186318bea475101 Mon Sep 17 00:00:00 2001 From: Himank Chaudhary Date: Wed, 3 Aug 2022 12:12:35 -0700 Subject: [PATCH] fix: fixing encoding issues --- lib/json/encoder.go | 44 ++++++++++++++++++++++++++++ lib/uuid/uuid.go | 14 +++++++++ server/services/v1/search_indexer.go | 15 ++++++---- server/services/v1/search_reader.go | 6 ++-- 4 files changed, 70 insertions(+), 9 deletions(-) create mode 100644 lib/json/encoder.go diff --git a/lib/json/encoder.go b/lib/json/encoder.go new file mode 100644 index 000000000..86654ab85 --- /dev/null +++ b/lib/json/encoder.go @@ -0,0 +1,44 @@ +// Copyright 2022 Tigris Data, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package json + +import ( + "bytes" + + jsoniter "github.com/json-iterator/go" +) + +func Encode(data map[string]any) ([]byte, error) { + var buffer bytes.Buffer + encoder := jsoniter.NewEncoder(&buffer) + err := encoder.Encode(data) + if err != nil { + return nil, err + } + + return buffer.Bytes(), nil +} + +func Decode(data []byte) (map[string]any, error) { + var decoded map[string]any + + decoder := jsoniter.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() + if err := decoder.Decode(&decoded); err != nil { + return nil, err + } + + return decoded, nil +} diff --git a/lib/uuid/uuid.go b/lib/uuid/uuid.go index bd3d0db74..fd78e83b1 100644 --- a/lib/uuid/uuid.go +++ b/lib/uuid/uuid.go @@ -1,3 +1,17 @@ +// Copyright 2022 Tigris Data, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package uuid import uuid2 "github.com/google/uuid" diff --git a/server/services/v1/search_indexer.go b/server/services/v1/search_indexer.go index a8cf1ebdc..fb06c0434 100644 --- a/server/services/v1/search_indexer.go +++ b/server/services/v1/search_indexer.go @@ -25,6 +25,7 @@ import ( "github.com/apple/foundationdb/bindings/go/src/fdb/subspace" jsoniter "github.com/json-iterator/go" "github.com/tigrisdata/tigris/internal" + "github.com/tigrisdata/tigris/lib/json" "github.com/tigrisdata/tigris/schema" "github.com/tigrisdata/tigris/server/metadata" "github.com/tigrisdata/tigris/server/transaction" @@ -155,7 +156,7 @@ func CreateSearchKey(table []byte, fdbKey []byte) (string, error) { case string: value = t case []byte: - value = string(t) + value = base64.StdEncoding.EncodeToString(t) } return value, nil } else { @@ -165,10 +166,9 @@ func CreateSearchKey(table []byte, fdbKey []byte) (string, error) { } func PackSearchFields(data *internal.TableData, collection *schema.DefaultCollection, id string) ([]byte, error) { - var err error // better to decode it and then update the JSON - var decData map[string]any - if err = jsoniter.Unmarshal(data.RawData, &decData); err != nil { + decData, err := json.Decode(data.RawData) + if err != nil { return nil, err } @@ -195,7 +195,12 @@ func PackSearchFields(data *internal.TableData, collection *schema.DefaultCollec decData[schema.ReservedFields[schema.UpdatedAt]] = data.UpdatedAt.UnixNano() } - return jsoniter.Marshal(decData) + encoded, err := json.Encode(decData) + if err != nil { + return nil, err + } + + return encoded, nil } func UnpackSearchFields(doc map[string]interface{}, collection *schema.DefaultCollection) (string, *internal.TableData, map[string]interface{}, error) { diff --git a/server/services/v1/search_reader.go b/server/services/v1/search_reader.go index c57763d77..e3ecf1b2c 100644 --- a/server/services/v1/search_reader.go +++ b/server/services/v1/search_reader.go @@ -17,8 +17,8 @@ package v1 import ( "context" - jsoniter "github.com/json-iterator/go" api "github.com/tigrisdata/tigris/api/server/v1" + "github.com/tigrisdata/tigris/lib/json" "github.com/tigrisdata/tigris/query/filter" "github.com/tigrisdata/tigris/query/read" qsearch "github.com/tigrisdata/tigris/query/search" @@ -94,10 +94,8 @@ func (p *page) readRow(row *Row) bool { } var rawData []byte - // marshal the doc as bytes - rawData, p.err = jsoniter.Marshal(doc) - if p.err != nil { + if rawData, p.err = json.Encode(doc); p.err != nil { return false }