From f649db849f02f2bbcf59fb6396e3ef200eb72b13 Mon Sep 17 00:00:00 2001
From: "Congqi.Xia" <congqi.xia@zilliz.com>
Date: Tue, 27 Feb 2024 22:28:46 +0800
Subject: [PATCH 1/4] enhance: Unify oss access in oss package & support gcp
This PR add `oss` package, which provided MinioClient accessing
aws/minio/gcp
Also support gcs service
Signed-off-by: Congqi.Xia <congqi.xia@zilliz.com>
---
go.mod | 2 +-
oss/gcp.go | 73 ++++++++++++++++++++++++++
oss/minio.go | 80 +++++++++++++++++++++++++++++
states/minio.go | 133 +++++++++++++++++++++++++++++++++++++++++++-----
4 files changed, 275 insertions(+), 13 deletions(-)
create mode 100644 oss/gcp.go
create mode 100644 oss/minio.go
diff --git a/go.mod b/go.mod
index 41aa2a33..08641130 100644
--- a/go.mod
+++ b/go.mod
@@ -27,6 +27,7 @@ require (
go.etcd.io/etcd/server/v3 v3.5.4
go.uber.org/atomic v1.7.0
go.uber.org/zap v1.17.0
+ golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602
google.golang.org/grpc v1.46.0
google.golang.org/protobuf v1.30.0
gopkg.in/yaml.v3 v3.0.1
@@ -140,7 +141,6 @@ require (
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/net v0.10.0 // indirect
- golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 // indirect
golang.org/x/sys v0.8.0 // indirect
golang.org/x/term v0.8.0 // indirect
golang.org/x/text v0.9.0 // indirect
diff --git a/oss/gcp.go b/oss/gcp.go
new file mode 100644
index 00000000..f0cae00a
--- /dev/null
+++ b/oss/gcp.go
@@ -0,0 +1,73 @@
+package oss
+
+import (
+ "net/http"
+ "sync/atomic"
+
+ "github.com/cockroachdb/errors"
+ "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+ "golang.org/x/oauth2"
+ "golang.org/x/oauth2/google"
+)
+
+const GcsDefaultAddress = "storage.googleapis.com"
+
+func processMinioGcpOptions(p MinioClientParam, opts *minio.Options) error {
+ if p.UseIAM {
+ transport, err := NewWrapHTTPTransport(opts.Secure)
+ if err != nil {
+ return err
+ }
+ opts.Creds = credentials.NewStaticV2("", "", "")
+ opts.Transport = transport
+ } else {
+ opts.Creds = credentials.NewStaticV2(p.AK, p.SK, "")
+ }
+ return nil
+}
+
+// WrapHTTPTransport wraps http.Transport, add an auth header to support GCP native auth
+type WrapHTTPTransport struct {
+ tokenSrc oauth2.TokenSource
+ backend transport
+ currentToken atomic.Pointer[oauth2.Token]
+}
+
+// transport abstracts http.Transport to simplify test
+type transport interface {
+ RoundTrip(req *http.Request) (*http.Response, error)
+}
+
+// NewWrapHTTPTransport constructs a new WrapHTTPTransport
+func NewWrapHTTPTransport(secure bool) (*WrapHTTPTransport, error) {
+ tokenSrc := google.ComputeTokenSource("")
+ // in fact never return err
+ backend, err := minio.DefaultTransport(secure)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to create default transport")
+ }
+ return &WrapHTTPTransport{
+ tokenSrc: tokenSrc,
+ backend: backend,
+ }, nil
+}
+
+// RoundTrip wraps original http.RoundTripper by Adding a Bearer token acquired from tokenSrc
+func (t *WrapHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+ // here Valid() means the token won't be expired in 10 sec
+ // so the http client timeout shouldn't be longer, or we need to change the default `expiryDelta` time
+ currentToken := t.currentToken.Load()
+ if currentToken.Valid() {
+ req.Header.Set("Authorization", "Bearer "+currentToken.AccessToken)
+ } else {
+ newToken, err := t.tokenSrc.Token()
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to acquire token")
+ }
+ t.currentToken.Store(newToken)
+ req.Header.Set("Authorization", "Bearer "+newToken.AccessToken)
+ }
+
+ return t.backend.RoundTrip(req)
+}
diff --git a/oss/minio.go b/oss/minio.go
new file mode 100644
index 00000000..ef73467f
--- /dev/null
+++ b/oss/minio.go
@@ -0,0 +1,80 @@
+package oss
+
+import (
+ "context"
+ "fmt"
+ "strings"
+
+ "github.com/cockroachdb/errors"
+ "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+type MinioClientParam struct {
+ Addr string
+ Port string
+ AK string
+ SK string
+ UseIAM bool
+ IAMEndpoint string
+ UseSSL bool
+ CloudProvider string
+ Region string
+
+ BucketName string
+ RootPath string
+}
+
+// MinioClient wraps minio client, bucket info within
+type MinioClient struct {
+ Client *minio.Client
+ BucketName string
+ RootPath string
+}
+
+func NewMinioClient(ctx context.Context, p MinioClientParam) (*MinioClient, error) {
+ opts := &minio.Options{
+ Secure: p.UseSSL,
+ BucketLookup: minio.BucketLookupAuto,
+ }
+ endpoint := fmt.Sprintf("%s:%s", p.Addr, p.Port)
+
+ switch p.CloudProvider {
+ case "aws":
+ processMinioAwsOptions(p, opts)
+ case "gcp":
+ // adhoc to remove port of gcs address to let minio-go know it's gcs
+ if strings.Contains(endpoint, GcsDefaultAddress) {
+ endpoint = GcsDefaultAddress
+ }
+ processMinioGcpOptions(p, opts)
+ default:
+ return nil, errors.Newf("Cloud provider %s not supported yet", p.CloudProvider)
+ }
+ client, err := minio.New(endpoint, opts)
+ if err != nil {
+ return nil, err
+ }
+
+ ok, err := client.BucketExists(ctx, p.BucketName)
+ if err != nil {
+ return nil, err
+ }
+ if !ok {
+ return nil, errors.Newf("Bucket %s not exists", p.BucketName)
+ }
+
+ return &MinioClient{
+ Client: client,
+ BucketName: p.BucketName,
+ RootPath: p.RootPath,
+ }, nil
+}
+
+func processMinioAwsOptions(p MinioClientParam, opts *minio.Options) {
+ if p.UseIAM {
+ opts.Creds = credentials.NewIAM(p.IAMEndpoint)
+ } else {
+ opts.Creds = credentials.NewStaticV4(p.AK, p.SK, "")
+ }
+}
diff --git a/states/minio.go b/states/minio.go
index 85359d32..b4a37ac7 100644
--- a/states/minio.go
+++ b/states/minio.go
@@ -2,11 +2,12 @@ package states
import (
"context"
- "fmt"
"github.com/cockroachdb/errors"
+ "github.com/manifoldco/promptui"
"github.com/milvus-io/birdwatcher/framework"
"github.com/milvus-io/birdwatcher/models"
+ "github.com/milvus-io/birdwatcher/oss"
rootcoordpbv2 "github.com/milvus-io/birdwatcher/proto/v2.2/rootcoordpb"
"github.com/milvus-io/birdwatcher/states/etcd/common"
"github.com/minio/minio-go/v7"
@@ -30,7 +31,6 @@ type MinioConnectParam struct {
}
func (s *InstanceState) TestMinioCfgCommand(ctx context.Context, p *TestMinioCfgParam) error {
- fmt.Println(p)
_, _, _, err := s.GetMinioClientFromCfg(ctx, "")
return err
}
@@ -65,16 +65,17 @@ func (s *InstanceState) GetMinioClientFromCfg(ctx context.Context, minioAddr str
return nil, "", "", err
}
+ var cloudProvider string
var addr string
var port string
var ak, sk string
var useIAM string
var useSSL string
- var secure bool
-
for _, config := range configurations {
switch config.GetKey() {
+ case "minio.cloudprovider":
+ cloudProvider = config.GetValue()
case "minio.address":
addr = config.GetValue()
case "minio.port":
@@ -94,23 +95,131 @@ func (s *InstanceState) GetMinioClientFromCfg(ctx context.Context, minioAddr str
}
}
+ mp := oss.MinioClientParam{
+ CloudProvider: cloudProvider,
+ Addr: addr,
+ Port: port,
+ AK: ak,
+ SK: sk,
+
+ BucketName: bucketName,
+ RootPath: rootPath,
+ }
+ if useIAM == "true" {
+ mp.UseIAM = true
+ }
if useSSL == "true" {
- secure = true
+ mp.UseSSL = true
}
- if minioAddr == "" {
- minioAddr = fmt.Sprintf("%s:%s", addr, port)
+ mClient, err := oss.NewMinioClient(ctx, mp)
+ if err != nil {
+ return nil, "", "", err
}
- if useIAM == "true" {
- client, _, err = getMinioWithIAM(minioAddr, bucketName, secure)
- } else {
- client, _, err = getMinioWithInfo(minioAddr, ak, sk, bucketName, secure)
+ return mClient.Client, bucketName, rootPath, nil
+}
+
+func (s *InstanceState) GetMinioClientFromPrompt(ctx context.Context) (client *minio.Client, bucketName, rootPath string, err error) {
+
+ p := promptui.Prompt{
+ Label: "BucketName",
+ }
+ bucketName, err = p.Run()
+ if err != nil {
+ return nil, "", "", err
+ }
+
+ p = promptui.Prompt{
+ Label: "Root Path",
+ }
+ rootPath, err = p.Run()
+ if err != nil {
+ return nil, "", "", err
+ }
+
+ p = promptui.Prompt{Label: "Address"}
+ address, err := p.Run()
+ if err != nil {
+ return nil, "", "", err
+ }
+
+ ssl := promptui.Select{
+ Label: "Use SSL",
+ Items: []string{"yes", "no"},
+ }
+ _, sslResult, err := ssl.Run()
+ if err != nil {
+ return nil, "", "", err
+ }
+ useSSL := false
+ switch sslResult {
+ case "yes":
+ useSSL = true
+ case "no":
+ useSSL = false
+ }
+
+ cloudProvider := promptui.Select{
+ Label: "Select Cloud provider",
+ Items: []string{"aws", "gcp"},
+ }
+ _, cloudProviderResult, err := cloudProvider.Run()
+ if err != nil {
+ return nil, "", "", err
+ }
+
+ sl := promptui.Select{
+ Label: "Select authentication method:",
+ Items: []string{"IAM", "AK/SK"},
+ }
+ _, result, err := sl.Run()
+ if err != nil {
+ return nil, "", "", err
+ }
+
+ mp := oss.MinioClientParam{
+ CloudProvider: cloudProviderResult,
+ Addr: address,
+ UseSSL: useSSL,
+ BucketName: bucketName,
+ RootPath: rootPath,
+ }
+
+ switch result {
+ case "IAM":
+ mp.UseIAM = true
+ input := promptui.Prompt{
+ Label: "IAM Endpoint",
+ }
+ iamEndpoint, err := input.Run()
+ if err != nil {
+ return nil, "", "", err
+ }
+
+ mp.IAMEndpoint = iamEndpoint
+ case "AK/SK":
+ p.HideEntered = true
+ p.Mask = rune('*')
+ p.Label = "AK"
+ ak, err := p.Run()
+ if err != nil {
+ return nil, "", "", err
+ }
+ p.Label = "SK"
+ sk, err := p.Run()
+ if err != nil {
+ return nil, "", "", err
+ }
+
+ mp.AK = ak
+ mp.SK = sk
}
+ mClient, err := oss.NewMinioClient(ctx, mp)
if err != nil {
return nil, "", "", err
}
- return client, bucketName, rootPath, nil
+ return mClient.Client, bucketName, rootPath, nil
}
From 2449a45321f6cd378088f01ccb4da708e19da8af Mon Sep 17 00:00:00 2001
From: Congqi Xia <congqi.xia@zilliz.com>
Date: Wed, 28 Feb 2024 10:26:57 +0800
Subject: [PATCH 2/4] Fix atomic pointer not found in go1.18
Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
---
go.mod | 15 +++++++++------
go.sum | 43 +++++++++++++++++++++++++++++++++++--------
2 files changed, 44 insertions(+), 14 deletions(-)
diff --git a/go.mod b/go.mod
index 08641130..4486f6b3 100644
--- a/go.mod
+++ b/go.mod
@@ -1,8 +1,9 @@
module github.com/milvus-io/birdwatcher
-go 1.18
+go 1.20
require (
+ github.com/aliyun/credentials-go v1.3.2
github.com/apache/arrow/go/v8 v8.0.0
github.com/apache/pulsar-client-go v0.6.1-0.20210728062540-29414db801a7
github.com/blang/semver/v4 v4.0.0
@@ -40,6 +41,8 @@ require (
github.com/BurntSushi/toml v1.0.0 // indirect
github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 // indirect
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
+ github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68 // indirect
+ github.com/alibabacloud-go/tea v1.1.8 // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/apache/pulsar-client-go/oauth2 v0.0.0-20201120111947-b8bd55bc02bd // indirect
github.com/apache/thrift v0.15.0 // indirect
@@ -137,13 +140,13 @@ require (
go.opentelemetry.io/proto/otlp v0.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
golang.org/x/arch v0.3.0 // indirect
- golang.org/x/crypto v0.9.0 // indirect
+ golang.org/x/crypto v0.14.0 // indirect
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
golang.org/x/mod v0.8.0 // indirect
- golang.org/x/net v0.10.0 // indirect
- golang.org/x/sys v0.8.0 // indirect
- golang.org/x/term v0.8.0 // indirect
- golang.org/x/text v0.9.0 // indirect
+ golang.org/x/net v0.17.0 // indirect
+ golang.org/x/sys v0.13.0 // indirect
+ golang.org/x/term v0.13.0 // indirect
+ golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
golang.org/x/tools v0.6.0 // indirect
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
diff --git a/go.sum b/go.sum
index 2b88405d..c213e758 100644
--- a/go.sum
+++ b/go.sum
@@ -76,6 +76,12 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68 h1:NqugFkGxx1TXSh/pBcU00Y6bljgDPaFdh5MUSeJ7e50=
+github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68/go.mod h1:6pb/Qy8c+lqua8cFpEy7g39NRRqOWc3rOwAy8m5Y2BY=
+github.com/alibabacloud-go/tea v1.1.8 h1:vFF0707fqjGiQTxrtMnIXRjOCvQXf49CuDVRtTopmwU=
+github.com/alibabacloud-go/tea v1.1.8/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4=
+github.com/aliyun/credentials-go v1.3.2 h1:L4WppI9rctC8PdlMgyTkF8bBsy9pyKQEzBD1bHMRl+g=
+github.com/aliyun/credentials-go v1.3.2/go.mod h1:tlpz4uys4Rn7Ik4/piGRrTbXy2uLKvePgQJJduE+Y5c=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@@ -391,8 +397,9 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0=
+github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
@@ -619,6 +626,7 @@ github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi
github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nrwiersma/avro-benchmarks v0.0.0-20210913175520-21aec48c8f76/go.mod h1:iKyFMidsk/sVYONJRE372sJuX/QTRPacU7imPqqsu7g=
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -745,8 +753,9 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0=
+github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -833,6 +842,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/zeebo/xxh3 v1.0.1 h1:FMSRIbkrLikb/0hZxmltpg84VkqDAT5M8ufXynuhXsI=
github.com/zeebo/xxh3 v1.0.1/go.mod h1:8VHV24/3AZLn3b6Mlp/KuC33LWH687Wq6EnziEB+rsA=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@@ -920,8 +930,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
-golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -978,6 +988,7 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1032,8 +1043,11 @@ golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1058,7 +1072,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1138,14 +1154,20 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1155,8 +1177,10 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1233,6 +1257,7 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1380,6 +1405,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/errgo.v1 v1.0.0/go.mod h1:CxwszS/Xz1C49Ucd2i6Zil5UToP1EmyrFhKaMVbg1mk=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
@@ -1390,6 +1416,7 @@ gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/R
gopkg.in/httprequest.v1 v1.2.1/go.mod h1:x2Otw96yda5+8+6ZeWwHIJTFkEHWP/qP8pJOzqEtWPM=
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU=
gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
From bf73e553026de74cc477bd228e2115c0af2830ff Mon Sep 17 00:00:00 2001
From: Congqi Xia <congqi.xia@zilliz.com>
Date: Wed, 28 Feb 2024 11:24:34 +0800
Subject: [PATCH 3/4] Use 1.20 golangci-lint
Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
---
.github/workflows/golangci-lint.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml
index 47a79da0..92bcf44a 100644
--- a/.github/workflows/golangci-lint.yml
+++ b/.github/workflows/golangci-lint.yml
@@ -29,7 +29,7 @@ jobs:
steps:
- uses: actions/setup-go@v3
with:
- go-version: 1.18
+ go-version: 1.20
- uses: actions/checkout@v3
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
From 55a054dc699d8b8d00c79c9af02d7266e5724bc1 Mon Sep 17 00:00:00 2001
From: Congqi Xia <congqi.xia@zilliz.com>
Date: Wed, 28 Feb 2024 11:38:36 +0800
Subject: [PATCH 4/4] use uber atomic pointer instead of native one
Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
---
.github/workflows/golangci-lint.yml | 2 +-
go.mod | 4 ++--
go.sum | 3 ++-
oss/gcp.go | 2 +-
4 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml
index 92bcf44a..47a79da0 100644
--- a/.github/workflows/golangci-lint.yml
+++ b/.github/workflows/golangci-lint.yml
@@ -29,7 +29,7 @@ jobs:
steps:
- uses: actions/setup-go@v3
with:
- go-version: 1.20
+ go-version: 1.18
- uses: actions/checkout@v3
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
diff --git a/go.mod b/go.mod
index 4486f6b3..6d4bb119 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
module github.com/milvus-io/birdwatcher
-go 1.20
+go 1.18
require (
github.com/aliyun/credentials-go v1.3.2
@@ -26,7 +26,7 @@ require (
go.etcd.io/etcd/api/v3 v3.5.4
go.etcd.io/etcd/client/v3 v3.5.4
go.etcd.io/etcd/server/v3 v3.5.4
- go.uber.org/atomic v1.7.0
+ go.uber.org/atomic v1.11.0
go.uber.org/zap v1.17.0
golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602
google.golang.org/grpc v1.46.0
diff --git a/go.sum b/go.sum
index c213e758..b4ad1eee 100644
--- a/go.sum
+++ b/go.sum
@@ -899,8 +899,9 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
-go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
diff --git a/oss/gcp.go b/oss/gcp.go
index f0cae00a..e1043320 100644
--- a/oss/gcp.go
+++ b/oss/gcp.go
@@ -2,11 +2,11 @@ package oss
import (
"net/http"
- "sync/atomic"
"github.com/cockroachdb/errors"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
+ "go.uber.org/atomic"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
)