Merge pull request #431 from tigrisdata/main

Alpha release

.github/workflows/go-test.yaml

@@ -7,42 +7,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v1
-
-      - name: Get changed files
-        id: changed-files
-        uses: tj-actions/changed-files@v16
-
-      # TODO: Fix docker caching
-      - name: Pull docker images
-        run: docker-compose -f test/docker/docker-compose.yml pull
-
-      - name: Check need rebuild Docker images
-        run: |
-          echo "need_rebuild=true" >> $GITHUB_ENV
-
-      - name: Create cache directory
-        run: mkdir -p /home/runner/.cache/go-build
-
-      - name: Pull build docker images
-        if: env.need_rebuild
-        run: |
-          docker pull ubuntu:jammy-20220531
-
-      - name: Pull submodules
-        run: git submodule update --init --recursive
-
-      - name: Build docker images
-        if: env.need_rebuild
-        run: |
-          make docker_compose_build
-          docker image rm ubuntu:jammy-20220531
-          docker image prune -f
+        uses: actions/checkout@v2
+        with:
+          submodules: true
 
       - name: Run tests
-        run: make docker_test_no_build || ( docker-compose -f test/docker/docker-compose.yml logs && false )
+        run: make test || ( docker-compose -f test/docker/docker-compose.yml logs && false )
         env:
-          PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/go/bin
           TEST_PARAM: "-coverprofile=coverage.out -covermode=atomic"
 
       - name: Upload coverage to Codecov

.github/workflows/publish-debug-image.yaml

@@ -26,6 +26,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
+        with:
+          submodules: true
 
       - name: Fetch tags
         run: |
@@ -51,9 +53,6 @@ jobs:
             type=ref,event=branch
             type=ref,event=pr
 
-      - name: Pull submodules
-        run: git submodule update --init --recursive
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
 

.github/workflows/push-docker-image.yaml

@@ -28,6 +28,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
+        with:
+          submodules: true
 
       - name: Fetch tags
         run: |
@@ -56,9 +58,6 @@ jobs:
             type=semver,pattern={{major}}.{{minor}}
             type=raw,value=latest,enable=${{ github.ref == 'refs/heads/alpha' }}
 
-      - name: Pull submodules
-        run: git submodule update --init --recursive
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
 

.github/workflows/push-docker-local-image.yaml

@@ -28,6 +28,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
+        with:
+          submodules: true
 
       - name: Fetch tags
         run: |
@@ -56,9 +58,6 @@ jobs:
             type=semver,pattern={{major}}.{{minor}}
             type=raw,value=latest,enable=${{ github.ref == 'refs/heads/alpha' }}
 
-      - name: Pull submodules
-        run: git submodule update --init --recursive
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
 

.github/workflows/release.yaml

@@ -10,12 +10,9 @@ jobs:
   test:
     uses: ./.github/workflows/go-test.yaml
 
-  lint:
-    uses: ./.github/workflows/go-lint.yaml
-
   release:
     runs-on: ubuntu-latest
-    needs: [test, lint]
+    needs: [test]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v1

Makefile

@@ -38,30 +38,24 @@ server/service: $(GO_SRC) generate
 	CGO_ENABLED=$(CGO_ENABLED) go build $(BUILD_PARAM) -o server/service ./server
 
 lint: generate
-	yq --exit-status 'tag == "!!map" or tag== "!!seq"' .github/workflows/*.yaml config/*.yaml
+	yq --exit-status 'tag == "!!map" or tag== "!!seq"' .github/workflows/*.yaml config/*.yaml >/dev/null
 	shellcheck scripts/*
 	shellcheck test/docker/grafana/*
 	golangci-lint --timeout=$(LINT_TIMEOUT) run
 
-docker_compose_build:
-	$(DOCKER_COMPOSE) build
-
 # dependency on generate needed to create generated file outside of docker with
 # current user owner instead of root
-docker_test: generate
+docker_test:
 	$(DOCKER_COMPOSE) up --build tigris_test tigris_test
 	@[ $$(docker inspect tigris_test --format='{{.State.ExitCode}}') = "0" ]
 
-docker_test_no_build:
-	$(DOCKER_COMPOSE) up --exit-code-from tigris_test --no-build tigris_test tigris_test
-
 test: docker_test
 
 # Use this target to run the test from inside docker container
-local_test: generate
+local_test: generate lint
 	go test $(TEST_PARAM) ./...
 
-run: generate
+run:
 	$(DOCKER_COMPOSE) up --build --detach tigris_server2
 
 local_run: server
@@ -71,7 +65,7 @@ local_run: server
 
 # Runs tigris server and foundationdb, plus additional tools for it like:
 # - prometheus and grafana for monitoring
-run_full: generate
+run_full:
 	${DOCKER_COMPOSE} up --build --detach tigris_grafana
 	./${DOCKER_DIR}/grafana/set_admin_password.sh
 	./${DOCKER_DIR}/grafana/add_prometheus_datasource.sh
@@ -85,3 +79,6 @@ clean:
 
 upgrade_api:
 	git submodule update --remote --recursive --rebase
+
+build_and_push_base_docker:
+	docker buildx build -t tigrisdata/tigris-build-base:latest --platform linux/amd64,linux/arm64 --push -f docker/Dockerfile.base .

api/server/v1/header.go

@@ -19,16 +19,17 @@ const (
 
 	HeaderPrefix = "Tigris-"
 
-	HeaderTxID     = "Tigris-Tx-Id"
-	HeaderTxOrigin = "Tigris-Tx-Origin"
-
+	HeaderTxID        = "Tigris-Tx-Id"
+	HeaderTxOrigin    = "Tigris-Tx-Origin"
+	SetCookie         = "Set-Cookie"
+	Cookie            = "Cookie"
 	grpcGatewayPrefix = "Grpc-Gateway-"
 )
 
 func CustomMatcher(key string) (string, bool) {
 	key = textproto.CanonicalMIMEHeaderKey(key)
 	switch key {
-	case HeaderRequestTimeout, HeaderAccessControlAllowOrigin:
+	case HeaderRequestTimeout, HeaderAccessControlAllowOrigin, SetCookie, Cookie:
 		return key, true
 	default:
 		if strings.HasPrefix(key, HeaderPrefix) {

api/server/v1/marshaler.go

@@ -16,18 +16,61 @@ package api
 
 import (
 	"encoding/json"
+	"io"
+	"net/url"
 	"time"
 
+	"github.com/ajg/form"
 	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	jsoniter "github.com/json-iterator/go"
 	spb "google.golang.org/genproto/googleapis/rpc/status"
 	"google.golang.org/protobuf/proto"
 )
 
+const (
+	refreshToken = "refresh_token"
+)
+
+type CustomDecoder struct {
+	formDecoder *form.Decoder
+	jsonDecoder *json.Decoder
+	reader      io.Reader
+}
+
+func (f CustomDecoder) Decode(dst interface{}) error {
+	switch dst.(type) {
+	case *GetAccessTokenRequest:
+		{
+			byteArr, err := io.ReadAll(f.reader)
+			if err != nil {
+				return err
+			}
+			return unmarshalInternal(byteArr, dst)
+		}
+	}
+	return f.jsonDecoder.Decode(dst)
+}
+
 // CustomMarshaler is a marshaler to customize the response. Currently, it is only used to marshal custom error message
 // otherwise it just uses the inbuilt mux marshaller.
 type CustomMarshaler struct {
-	*runtime.JSONBuiltin
+	JSONBuiltin *runtime.JSONBuiltin
+}
+
+func (c *CustomMarshaler) NewDecoder(r io.Reader) runtime.Decoder {
+	return CustomDecoder{
+		formDecoder: form.NewDecoder(r),
+		jsonDecoder: json.NewDecoder(r),
+		reader:      r,
+	}
+}
+
+func (c *CustomMarshaler) NewEncoder(w io.Writer) runtime.Encoder {
+	return c.JSONBuiltin.NewEncoder(w)
+}
+
+func (c *CustomMarshaler) ContentType(v interface{}) string {
+	return c.JSONBuiltin.ContentType(v)
 }
 
 func (c *CustomMarshaler) Marshal(v interface{}) ([]byte, error) {
@@ -53,6 +96,31 @@ func (c *CustomMarshaler) Marshal(v interface{}) ([]byte, error) {
 	return c.JSONBuiltin.Marshal(v)
 }
 
+func (c *CustomMarshaler) Unmarshal(data []byte, v interface{}) error {
+	switch v.(type) {
+	case *GetAccessTokenRequest:
+		{
+			return unmarshalInternal(data, v)
+		}
+	}
+	return c.JSONBuiltin.Unmarshal(data, v)
+}
+
+func unmarshalInternal(data []byte, v interface{}) error {
+	switch v := v.(type) {
+	case *GetAccessTokenRequest:
+		{
+			values, err := url.ParseQuery(string(data))
+			if err != nil {
+				return err
+			}
+			v.RefreshToken = values.Get(refreshToken)
+			return nil
+		}
+	}
+	return Errorf(Code_INTERNAL, "not supported")
+}
+
 // UnmarshalJSON on ReadRequest avoids unmarshalling filter and instead this way we can write a custom struct to do
 // the unmarshalling and will be avoiding any extra allocation/copying.
 func (x *ReadRequest) UnmarshalJSON(data []byte) error {

cmd/consistency/consistency.go

@@ -1,3 +1,17 @@
+// Copyright 2022 Tigris Data, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (

cmd/consistency/workload/document.go

@@ -1,3 +1,17 @@
+// Copyright 2022 Tigris Data, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package workload
 
 import (

cmd/consistency/workload/queue.go

@@ -1,3 +1,17 @@
+// Copyright 2022 Tigris Data, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package workload
 
 import "sync"

cmd/consistency/workload/workload_ddl.go

@@ -1,3 +1,17 @@
+// Copyright 2022 Tigris Data, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package workload
 
 import (