From 76819ee3bf52bc8dd2b8ae3dcd1ac5a9373c121c Mon Sep 17 00:00:00 2001 From: Jun Lin Chen Date: Tue, 15 Nov 2022 16:08:29 -0500 Subject: [PATCH] ADD: more db retry --- cmd/ctr-starlight/main.go | 11 +---------- demo/.gitignore | 4 +++- demo/chart/values.yaml | 15 ++------------- docs/helm.md | 40 ++++----------------------------------- proxy/builder_test.go | 2 +- proxy/database.go | 27 +++++++++++++++++++++----- proxy/database_test.go | 7 +++++-- proxy/extractor_test.go | 2 +- proxy/server.go | 22 ++++++++++++++++----- 9 files changed, 56 insertions(+), 74 deletions(-) diff --git a/cmd/ctr-starlight/main.go b/cmd/ctr-starlight/main.go index 066ee2c..ef55023 100644 --- a/cmd/ctr-starlight/main.go +++ b/cmd/ctr-starlight/main.go @@ -70,7 +70,7 @@ https://github.com/mc256/starlight Value: "default", DefaultText: "default", EnvVars: []string{"CONTAINERD_NAMESPACE"}, - Usage: "namespace to use with commands", + Usage: "namespace to use with commands (if using kubernetes, please specify `k8s.io`)", Required: false, }, &cli.StringFlag{ @@ -90,15 +90,6 @@ https://github.com/mc256/starlight DefaultText: "info", Required: false, }, - &cli.StringFlag{ - Name: "namespace", - Aliases: []string{"n"}, - EnvVars: []string{"CONTAINERD_NAMESPACE"}, - Usage: "namespace to use with commands (if using kubernetes, please specify `k8s.io`)", - Value: "default", - DefaultText: "default", - Required: false, - }, } app.Commands = append([]*cli.Command{ cmdVersion.Command(), // 1. confirm the version of starlight-daemon diff --git a/demo/.gitignore b/demo/.gitignore index 4381c9e..0887fcc 100644 --- a/demo/.gitignore +++ b/demo/.gitignore @@ -5,4 +5,6 @@ temp.sh !convert* -*.class \ No newline at end of file +*.class + +k8s \ No newline at end of file diff --git a/demo/chart/values.yaml b/demo/chart/values.yaml index ca659b0..bbd3088 100644 --- a/demo/chart/values.yaml +++ b/demo/chart/values.yaml @@ -15,13 +15,8 @@ starlightProxy: # For example: "postgres://postgres:postgres@postgres:5432/starlight?sslmode=disable" # if left blank, the proxy will use the default connection string assuming the PostgresDB is enabled in this chart. dbConnection: "" - - persistence: - enabled: true - existingClaim: "" - storageClass: "" - accessModes: [ReadWriteOnce] - size: 2Gi + logLevel: "info" + defaultRegistry: "http://container-registry.default.svc.cluster.local:5000" # --------------------------------------------------------------------- # Metadata database - PostgreSQL @@ -48,12 +43,6 @@ adminer: enabled: true pullPolicy: IfNotPresent tag: "latest" - persistence: - enabled: true - existingClaim: "" - storageClass: "" - accessModes: [ReadWriteOnce] - size: 2Gi # --------------------------------------------------------------------- # Vanilla Container Registry diff --git a/docs/helm.md b/docs/helm.md index fe74255..fe1eddf 100644 --- a/docs/helm.md +++ b/docs/helm.md @@ -3,7 +3,7 @@ ## TL;DR ```shell -helm upgrade --install -f starlight/values.yaml starlight oci://ghcr.io/mc256/starlight/starlight-proxy-chart --version 0.1.2 +helm upgrade --install -f starlight/values.yaml starlight oci://ghcr.io/mc256/starlight/starlight-proxy-chart --version 0.2.3 ``` ## Prerequisites @@ -34,14 +34,14 @@ kubectl patch storageclass local-path -p '{"metadata": {"annotations":{"storagec To install the chart with the app name `my-starlight-proxy`: ```shell -helm install my-starlight-proxy oci://ghcr.io/mc256/starlight/starlight-proxy-chart --version 0.1.2 +helm install my-starlight-proxy oci://ghcr.io/mc256/starlight/starlight-proxy-chart --version 0.2.3 ``` You may want to set a few parameter for example the domain name for the ingress, for example set the domain name for ingress to `mydomain.local`: ```shell -helm install my-starlight-proxy oci://ghcr.io/mc256/starlight/starlight-proxy-chart --version 0.1.2 \ +helm install my-starlight-proxy oci://ghcr.io/mc256/starlight/starlight-proxy-chart --version 0.2.3 \ --set "ingress.hosts={mydomain.local}" ``` @@ -59,37 +59,5 @@ helm delete my-starlight-proxy ## Parameters - -### Common Parameters - -| Name | Description | Value| -| --- | --- | --- | -| ingress.hosts | domain name for the ingress | [starlight.lan] | -| registryAddress | customize registry address if choose not to use the container registry in this chart | null | -| registry.enable | enable container registry in this deployment | true | - -### Starlight Proxy - -| Name | Description | Value| -| --- | --- | --- | -| starlightProxy.tag | tage of the image | "latest" | -| starlightProxy.pullPolicy | pull image policy | IfNotPresent | -| starlightProxy.persistence.enabled | tage of the image | "latest" | -| starlightProxy.persistence.existingClaim | if specified, use existing PV | "" | -| starlightProxy.persistence.storageClass | storage class, if not specified, used default storage class | "" | -| starlightProxy.persistence.size | storage size | 2Gi | - -### Registry - -| Name | Description | Value| -| --- | --- | --- | -| registry.enable | enable registry | true | -| registry.repository | container image | "registry"| -| registry.pullPolicy | pull image policy | IfNotPresent | -| registry.tag | tage of the image | "latest" | -| registry.persistence.enabled | tage of the image | "latest" | -| registry.persistence.existingClaim | if specified, use existing PV | "" | -| registry.persistence.storageClass | storage class, if not specified, used default storage class | "" | -| registry.persistence.size | storage size | 2Gi | - +Please the comments in the [`values.yaml`]() file diff --git a/proxy/builder_test.go b/proxy/builder_test.go index 8488dc5..38e6fd9 100644 --- a/proxy/builder_test.go +++ b/proxy/builder_test.go @@ -39,7 +39,7 @@ func InitDatabase() (context.Context, *Configuration, *Server) { config: cfg, cache: make(map[string]*common.LayerCache), } - if db, err := NewDatabase(cfg.PostgresConnectionString); err != nil { + if db, err := NewDatabase(ctx, cfg.PostgresConnectionString); err != nil { log.G(ctx).Errorf("failed to connect to database: %v\n", err) } else { server.db = db diff --git a/proxy/database.go b/proxy/database.go index d3d6824..66ec502 100644 --- a/proxy/database.go +++ b/proxy/database.go @@ -6,9 +6,11 @@ package proxy import ( + "context" "database/sql" "encoding/json" "fmt" + "github.com/containerd/containerd/log" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/lib/pq" @@ -30,12 +32,27 @@ func (d *Database) Close() { _ = d.db.Close() } -func NewDatabase(conStr string) (*Database, error) { - d, err := sql.Open("postgres", conStr) - if err != nil { - return nil, err +func NewDatabase(ctx context.Context, conStr string) (*Database, error) { + var ( + d *sql.DB + err error + ) + i := 0 + for { + i += 1 + d, err = sql.Open("postgres", conStr) + if err == nil { + return &Database{db: d}, nil + } else if err != nil { + if i > 10 { + return nil, err + } + log.G(ctx). + WithError(err). + Errorf("failed to connect to database, retrying in 5 seconds (%d/10)", i) + time.Sleep(5 * time.Second) + } } - return &Database{db: d}, nil } func (d *Database) InitDatabase() error { diff --git a/proxy/database_test.go b/proxy/database_test.go index 38f4a68..5b3b81b 100644 --- a/proxy/database_test.go +++ b/proxy/database_test.go @@ -6,6 +6,7 @@ package proxy import ( + "context" "encoding/json" "fmt" "github.com/mc256/starlight/client/fs" @@ -20,8 +21,9 @@ var ( func TestMain(m *testing.M) { cfg, _, _, _ := LoadConfig("") + ctx := context.Background() var err error - db, err = NewDatabase(cfg.PostgresConnectionString) + db, err = NewDatabase(ctx, cfg.PostgresConnectionString) if err != nil { fmt.Printf("failed to connect to database %v\n", err) } @@ -29,7 +31,8 @@ func TestMain(m *testing.M) { } func TestDatabase_Init(t *testing.T) { - db, err := NewDatabase("postgres://postgres:example@172.18.1.61:5432/postgres?sslmode=disable") + ctx := context.Background() + db, err := NewDatabase(ctx, "postgres://postgres:example@172.18.1.61:5432/postgres?sslmode=disable") if err != nil { t.Error(err) return diff --git a/proxy/extractor_test.go b/proxy/extractor_test.go index 19ca2bc..3820209 100644 --- a/proxy/extractor_test.go +++ b/proxy/extractor_test.go @@ -41,7 +41,7 @@ func TestExtractor_SaveToC(t *testing.T) { }, config: cfg, } - if db, err := NewDatabase(cfg.PostgresConnectionString); err != nil { + if db, err := NewDatabase(ctx, cfg.PostgresConnectionString); err != nil { log.G(ctx).Errorf("failed to connect to database: %v\n", err) } else { server.db = db diff --git a/proxy/server.go b/proxy/server.go index b40273e..c1ba6fe 100644 --- a/proxy/server.go +++ b/proxy/server.go @@ -257,17 +257,29 @@ func NewServer(ctx context.Context, wg *sync.WaitGroup, cfg *Configuration) (*Se } // connect database - if db, err := NewDatabase(cfg.PostgresConnectionString); err != nil { + if db, err := NewDatabase(ctx, cfg.PostgresConnectionString); err != nil { log.G(ctx).Errorf("failed to connect to database: %v\n", err) } else { server.db = db } // init database - err := server.db.InitDatabase() - if err != nil { - log.G(ctx).Errorf("failed to init database: %v\n", err) - return nil, err + i := 0 + for { + i += 1 + err := server.db.InitDatabase() + if err != nil { + if i > 10 { + log.G(ctx).Errorf("failed to init database: %v\n", err) + return nil, err + } + log.G(ctx). + WithError(err). + Errorf("failed to connect to database, retrying in 5 seconds (%d/10)", i) + time.Sleep(5 * time.Second) + } else { + break + } } log.G(ctx).Debug("database initialized")