Merge branch 'http-ticker-fixes'

commands/run.go

@@ -1,6 +1,7 @@
 package commands
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"os/signal"
@@ -23,40 +24,61 @@ var runCmd = &cobra.Command{
 		signal.Notify(stopChan, syscall.SIGINT)
 		signal.Notify(stopChan, syscall.SIGTERM)
 
+		ctx, cancel := context.WithCancel(context.Background())
+
+		// Wait for stop signal in parallel to real work.
+		// Then cancel the work context to shut things down gracefully.
 		go func() {
-			for {
-				c := core.New(core.Config{
-					Eth2stats: core.Eth2statsConfig{
-						Version:    fmt.Sprintf("eth2stats-client/%s", RootCmd.Version),
-						ServerAddr: viper.GetString("eth2stats.addr"),
-						TLS:        viper.GetBool("eth2stats.tls"),
-						NodeName:   viper.GetString("eth2stats.node-name"),
-					},
-					BeaconNode: core.BeaconNodeConfig{
-						Type:        viper.GetString("beacon.type"),
-						Addr:        viper.GetString("beacon.addr"),
-						TLSCert:     viper.GetString("beacon.tls-cert"),
-						MetricsAddr: viper.GetString("beacon.metrics-addr"),
-					},
-					DataFolder: viper.GetString("data.folder"),
-				})
-
-				err := c.Run()
-				if err != nil {
-					log.Error(err)
-				}
-
-				// we're only getting here if there's been no error during set up
-				time.Sleep(time.Second * 12)
-				log.Info("retrying...")
+			select {
+			case <-stopChan:
+				log.Info("got stop signal. finishing work.")
+				cancel()
 			}
 		}()
 
-		select {
-		case <-stopChan:
-			log.Info("got stop signal. finishing work.")
-			log.Info("work done. goodbye!")
+	workLoop:
+		for {
+			c := core.New(core.Config{
+				Eth2stats: core.Eth2statsConfig{
+					Version:    fmt.Sprintf("eth2stats-client/%s", RootCmd.Version),
+					ServerAddr: viper.GetString("eth2stats.addr"),
+					TLS:        viper.GetBool("eth2stats.tls"),
+					NodeName:   viper.GetString("eth2stats.node-name"),
+				},
+				BeaconNode: core.BeaconNodeConfig{
+					Type:        viper.GetString("beacon.type"),
+					Addr:        viper.GetString("beacon.addr"),
+					TLSCert:     viper.GetString("beacon.tls-cert"),
+					MetricsAddr: viper.GetString("beacon.metrics-addr"),
+				},
+				DataFolder: viper.GetString("data.folder"),
+			})
+
+			err := c.Run(ctx)
+
+			// Check if the service needs to stop yet.
+			select {
+			case <-ctx.Done():
+				break workLoop
+			default:
+			}
+
+			if err == nil {
+				log.Warn("eth2stats work stopped unexpectedly without error")
+			} else {
+				log.Error(err)
+			}
+
+			// we're only getting here if there's been a setup error that is recoverable
+			log.Infof("retrying in %s...", RetryInterval)
+			select {
+			case <-ctx.Done():
+				break workLoop
+			case <-time.After(RetryInterval):
+			}
 		}
+
+		log.Info("work done. goodbye!")
 	},
 }
 

core/beacon.go

@@ -34,9 +34,9 @@ func initBeaconClient(nodeType, nodeAddr, nodeCert string) beacon.Client {
 		log.Fatal("custom TLS certificates are currently only supported for GRPC connections")
 	}
 	var netTransport = &http.Transport{
-		Dial: (&net.Dialer{
+		DialContext: (&net.Dialer{
 			Timeout: 15 * time.Second,
-		}).Dial,
+		}).DialContext,
 		TLSHandshakeTimeout: 15 * time.Second,
 	}
 	var httpClient = &http.Client{

core/core.go

@@ -1,6 +1,7 @@
 package core
 
 import (
+	"context"
 	"fmt"
 	"time"
 
@@ -59,7 +60,6 @@ func New(config Config) *Core {
 		c.metricsWatcher = metricsWatcher.New(metricsWatcher.Config{
 			MetricsURL: config.BeaconNode.MetricsAddr,
 		})
-		go c.metricsWatcher.Run()
 	}
 
 	err := c.searchToken()
@@ -123,14 +123,18 @@ func (c *Core) connectToServer() error {
 	return nil
 }
 
-func (c *Core) watchNewHeads() {
+func (c *Core) watchNewHeads(ctx context.Context) {
 	for {
 		log.Info("setting up chain heads subscription")
 		sub, err := c.beaconClient.SubscribeChainHeads()
 		if err != nil {
 			// TODO handle gracefully
 			log.Fatal(err)
 		}
+		go func() {
+			<-ctx.Done()
+			sub.Close()
+		}()
 
 		limiter := rate.NewLimiter(1, 1)
 
@@ -156,37 +160,43 @@ func (c *Core) watchNewHeads() {
 	}
 }
 
-func (c *Core) sendHeartbeat() {
-	for range time.Tick(HeartbeatInterval) {
-		log.Trace("sending heartbeat")
+func (c *Core) sendHeartbeat(ctx context.Context) {
+	ticker := time.NewTicker(HeartbeatInterval)
+	for {
+		select {
+		case <-ticker.C:
+			log.Trace("sending heartbeat")
 
-		_, err := c.statsService.Heartbeat(c.contextWithToken(), &proto.HeartbeatRequest{})
-		if err != nil {
-			log.Fatalf("sending heartbeat: %s", err)
+			_, err := c.statsService.Heartbeat(c.contextWithToken(), &proto.HeartbeatRequest{})
+			if err != nil {
+				log.Fatalf("sending heartbeat: %s", err)
 
-			continue
+				continue
+			}
+			log.Trace("done sending heartbeat")
+		case <-ctx.Done():
+			ticker.Stop()
+			return
 		}
-		log.Trace("done sending heartbeat")
 	}
 }
 
-func (c *Core) Run() error {
+func (c *Core) Run(ctx context.Context) error {
 	err := c.connectToServer()
 	if err != nil {
 		return fmt.Errorf("setting up: %s", err)
 	}
 
-	// TODO handle gracefully
-	go c.watchNewHeads()
+	if c.metricsWatcher != nil {
+		go c.metricsWatcher.Run(ctx)
+	}
+
+	go c.watchNewHeads(ctx)
 
 	t := telemetry.New(c.telemetryService, c.beaconClient, c.metricsWatcher, c.contextWithToken)
-	go t.Run()
+	go t.Run(ctx)
 
 	// block while sending heartbeat
-	c.sendHeartbeat()
+	c.sendHeartbeat(ctx)
 	return nil
 }
-
-func (c *Core) Close() {
-	log.Info("Got stop signal")
-}

core/telemetry/telemetry.go

@@ -38,8 +38,15 @@ func New(service proto.TelemetryClient, beaconClient beacon.Client, watcher *met
 	}
 }
 
-func (t *Telemetry) Run() {
+func (t *Telemetry) Run(ctx context.Context) {
 	for {
+		// Check if the service needs to stop yet.
+		select {
+		case <-ctx.Done():
+			return
+		default:
+			break
+		}
 		log.Trace("sending telemetry")
 
 		t.pollPeers()

go.mod

@@ -3,19 +3,15 @@ module github.com/alethio/eth2stats-client
 require (
 	github.com/alethio/eth2stats-proto v0.0.0-20200122120216-4625b646ae41
 	github.com/avast/retry-go v2.6.0+incompatible
-	github.com/davecgh/go-spew v1.1.1
 	github.com/dghubble/sling v1.3.0
-	github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 // indirect
 	github.com/gin-gonic/gin v1.5.0
 	github.com/golang/protobuf v1.3.2
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/kwix/logrus-module-formatter v0.0.0-20190702125859-070a70371a97
-	github.com/parnurzeal/gorequest v0.2.16
 	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4
 	github.com/prometheus/common v0.4.0
 	github.com/prysmaticlabs/ethereumapis v0.0.0-20200211032731-6720aaf75915
 	github.com/sirupsen/logrus v1.4.2
-	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/viper v1.5.0
 	golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa // indirect
@@ -24,7 +20,6 @@ require (
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
 	google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24 // indirect
 	google.golang.org/grpc v1.26.0
-	moul.io/http2curl v1.0.0 // indirect
 )
 
 go 1.13

go.sum

@@ -30,10 +30,6 @@ github.com/dghubble/sling v1.3.0 h1:pZHjCJq4zJvc6qVQ5wN1jo5oNZlNE0+8T/h0XeXBUKU=
 github.com/dghubble/sling v1.3.0/go.mod h1:XXShWaBWKzNLhu2OxikSNFrlsvowtz4kyRuXUG7oQKY=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 h1:pEtiCjIXx3RvGjlUJuCNxNOw0MNblyR9Wi+vJGBFh+8=
-github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
-github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM=
-github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@@ -67,8 +63,6 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@@ -81,8 +75,6 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -117,8 +109,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/parnurzeal/gorequest v0.2.16 h1:T/5x+/4BT+nj+3eSknXmCTnEVGSzFzPGdpqmUVVZXHQ=
-github.com/parnurzeal/gorequest v0.2.16/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
@@ -137,20 +127,13 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/prysmaticlabs/ethereumapis v0.0.0-20191220030232-5c03c8e643ba h1:snky+GjbG2FcCySXr0ptIaPg5IExtb9cIBosBYQ3nkM=
-github.com/prysmaticlabs/ethereumapis v0.0.0-20191220030232-5c03c8e643ba/go.mod h1:5OkRN6UmvgtP+kIewitcEKC7S5KOzLOGtya/Tz+HBns=
 github.com/prysmaticlabs/ethereumapis v0.0.0-20200211032731-6720aaf75915 h1:G3/BqBjiDM9QE6ZMb561XWxuYyVIyqdTuVrMHDgmn5g=
 github.com/prysmaticlabs/ethereumapis v0.0.0-20200211032731-6720aaf75915/go.mod h1:5OkRN6UmvgtP+kIewitcEKC7S5KOzLOGtya/Tz+HBns=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
@@ -234,7 +217,6 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -267,5 +249,3 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8=
-moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE=

watcher/metrics/const.go

@@ -5,3 +5,10 @@ import (
 )
 
 const PollingInterval = 30 * time.Second
+
+const PollRetryAttempts = 4
+
+const PollTimeout = 5 * time.Second
+
+const PollDialTimeout = 10 * time.Second
+const PollTLSTimeout = 10 * time.Second