Merge pull request #1285 from iotaledger/develop

Merge v0.5.9 to master

.github/workflows/integration-tests.yml

@@ -254,40 +254,6 @@ jobs:
           name: ${{ env.TEST_NAME }}
           path: tools/integration-tests/logs
 
-  syncbeacon:
-    name: syncbeacon
-    env:
-      TEST_NAME: syncbeacon
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Build GoShimmer image
-        run: docker build -t iotaledger/goshimmer .
-
-      - name: Pull additional Docker images
-        run: |
-          docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiaadm/pumba:0.7.2
-          docker pull gaiadocker/iproute2:latest
-
-      - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
-
-      - name: Create logs from tester
-        if: always()
-        run: |
-          docker logs tester &> tools/integration-tests/logs/tester.log
-
-      - name: Save logs as artifacts
-        if: always()
-        uses: actions/upload-artifact@v1
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: tools/integration-tests/logs
-
   mana:
     name: mana
     env:

CHANGELOG.md

@@ -1,3 +1,19 @@
+# v0.5.9 - 2021-05-11
+* Replace sync beacons with Tangle Time
+* Fix approval weight manager persistence
+* Fix non positive ticker
+* Fix marker issues
+* Fix solidification issues
+* Fix concurrency-related issues
+* Improve FPC metrics logging
+* Improve clock synchronization handling
+* Improve dRNG plugin
+* Improve integration tests
+* Update JS dependencies
+* Update to latest hive.go
+* Update documentation
+* **Breaking**: bumps network and database versions
+
 # v0.5.8 - 2021-05-07
 * Integrate FPC with the X-Team committee
 * Enable finality via approval weight

client/wallet/webconnector.go

@@ -28,7 +28,7 @@ func (webConnector *WebConnector) ServerStatus() (status ServerStatus, err error
 	}
 
 	status.ID = response.IdentityID
-	status.Synced = response.Synced
+	status.Synced = response.TangleTime.Synced
 	status.Version = response.Version
 	status.ManaDecay = response.ManaDecay
 

config.default.json

@@ -18,13 +18,6 @@
     ],
     "port": 14626
   },
-  "syncbeaconfollower": {
-    "followNodes": [
-      "Gm7W191NDnqyF7KJycZqK7V6ENLwqxTwoKQN4SmpkB24",
-      "9DB3j9cWYSuEEtkvanrzqkzCQMdH1FGv3TawJdVbDxkd"
-    ],
-    "syncPercentage": 0.5
-  },
   "dashboard": {
     "bindAddress": "127.0.0.1:8081",
     "dev": false,

docs/SUMMARY.md

@@ -14,8 +14,6 @@ http://goshimmer.docs.iota.org/
   - [Obtaining tokens from the faucet](./tutorials/request_funds.md)
   - [The wallet library](./tutorials/wallet.md)
   - [Writing a dApp](./tutorials/dApp.md)
-  - [How to run a sync beacon](./tutorials/syncbeacon.md)
-  - [How to configure followed sync beacon nodes](./tutorials/follow_syncbeacon.md)
   - [How to create a static identity](./tutorials/static_identity.md)
   - [How to setup a custom dRNG committee](./tutorials/custom_dRNG.md)
   - [Setting up Monitoring Dashboard](./tutorials/monitoring.md)

docs/tutorials/setup.md

@@ -248,9 +248,7 @@ $ docker logs -f --since=1m goshimmer
 
 ### Syncing
 
-When the node starts for the first time, it must synchronize its state with the rest of the network. GoShimmer currently uses a sync beacon mechanism to help nodes determine their synced status and help syncing messages.
-
-Follow these links to learn more about [how sync beacons work, why you should run sync beacons](./syncbeacon.md) on your node and how to [configure your node to listen to certain sync beacons](./follow_syncbeacon.md).
+When the node starts for the first time, it must synchronize its state with the rest of the network. GoShimmer currently uses the Tangle Time to help nodes determine their synced status.
 
 #### Dashboard
 The dashboard of your GoShimmer node should be accessible via `http://<your-ip>:8081`. If your node is still synchronizing, you might see a higher inflow of MPS.

packages/drng/ticker.go

@@ -38,12 +38,10 @@ func NewTicker(dRNGState func() *State, interval int64, defaultValue float64, aw
 
 // UpdateRandomness updates the randomness of the ticker.
 func (t *Ticker) UpdateRandomness(r Randomness) {
-	select {
-	case <-t.fromRandomnessEvent:
-		t.fromRandomnessEvent <- r
-	default:
-		t.fromRandomnessEvent <- r
+	for len(t.fromRandomnessEvent) > 0 {
+		<-t.fromRandomnessEvent
 	}
+	t.fromRandomnessEvent <- r
 }
 
 // Start starts the Ticker.
@@ -96,14 +94,15 @@ func (t *Ticker) setDelayedRoundStart(d time.Duration) {
 func (t *Ticker) send() {
 	t.setDelayedRoundStart(0)
 	randomness := t.defaultValue
+	now := clock.SyncedTime()
 
 	if t.dRNGState() != nil && t.dRNGTicker != nil {
 		// we expect a randomness, it arrived already, and its newer than the last one
 		// OR we were missing a dRNG message previously but the check if the last randomness received is "fresh"
-		if clock.Since(t.dRNGState().Randomness().Timestamp) < time.Duration(t.interval)*time.Second {
+		if now.Sub(t.dRNGState().Randomness().Timestamp) < time.Duration(t.interval)*time.Second && now.Sub(t.dRNGState().Randomness().Timestamp) > 0 {
 			t.missingDRNG = false
 			randomness = t.dRNGState().Randomness().Float64()
-			timeToNextDRNG := t.dRNGState().Randomness().Timestamp.Add(time.Duration(t.interval) * time.Second).Sub(clock.SyncedTime())
+			timeToNextDRNG := t.dRNGState().Randomness().Timestamp.Add(time.Duration(t.interval) * time.Second).Sub(now)
 			t.setDelayedRoundStart(time.Duration(t.interval)*time.Second - timeToNextDRNG)
 			t.dRNGTicker.Reset(timeToNextDRNG)
 		} else {
@@ -122,11 +121,12 @@ func (t *Ticker) send() {
 			case randomnessEvent := <-t.fromRandomnessEvent:
 				// check if the randomness is "fresh"
 				if t.dRNGTicker != nil {
+					now := clock.SyncedTime()
 					// check against awaitOffset to be no more than 6 seconds
-					if clock.Since(randomnessEvent.Timestamp) < time.Duration(t.awaitOffset)*time.Second {
+					if now.Sub(randomnessEvent.Timestamp) < time.Duration(t.awaitOffset)*time.Second && now.Sub(randomnessEvent.Timestamp) > 0 {
 						t.missingDRNG = false
 						randomness = t.dRNGState().Randomness().Float64()
-						timeToNextDRNG := randomnessEvent.Timestamp.Add(time.Duration(t.interval) * time.Second).Sub(clock.SyncedTime())
+						timeToNextDRNG := randomnessEvent.Timestamp.Add(time.Duration(t.interval) * time.Second).Sub(now)
 						t.dRNGTicker.Reset(timeToNextDRNG)
 						t.setDelayedRoundStart(time.Duration(t.interval)*time.Second - timeToNextDRNG)
 						break out

packages/metrics/events.go

@@ -18,8 +18,6 @@ type CollectionEvents struct {
 	CPUUsage *events.Event
 	// MemUsage defines the local GoShimmer memory usage.
 	MemUsage *events.Event
-	// SyncBeaconSynced defines the local sync status event based on sync beacon.
-	SyncBeaconSynced *events.Event
 	// TangleTimeSynced defines the local sync status event based on tangle time.
 	TangleTimeSynced *events.Event
 	// ValueTips defines the local value tips count event.

packages/metrics/metrics.go

@@ -19,7 +19,6 @@ func newEvents() *CollectionEvents {
 		CPUUsage:              events.NewEvent(float64Caller),
 		MemUsage:              events.NewEvent(uint64Caller),
 		TangleTimeSynced:      events.NewEvent(boolCaller),
-		SyncBeaconSynced:      events.NewEvent(boolCaller),
 		ValueTips:             events.NewEvent(uint64Caller),
 		MessageTips:           events.NewEvent(uint64Caller),
 		QueryReceived:         events.NewEvent(queryReceivedEventCaller),

packages/remotelogmetrics/events.go

@@ -8,13 +8,12 @@ import (
 
 // CollectionLogEvents defines the events for the remotelogmetrics package.
 type CollectionLogEvents struct {
-	// SyncBeaconSyncChanged defines the local sync status change event based on sync beacon.
-	SyncBeaconSyncChanged *events.Event
 	// TangleTimeSyncChanged defines the local sync status change event based on tangle time.
 	TangleTimeSyncChanged *events.Event
 }
 
-func boolCaller(handler interface{}, params ...interface{}) {
+// SyncStatusChangedEventCaller is called when a node changes its sync status.
+func SyncStatusChangedEventCaller(handler interface{}, params ...interface{}) {
 	handler.(func(SyncStatusChangedEvent))(params[0].(SyncStatusChangedEvent))
 }
 
@@ -30,8 +29,24 @@ type SyncStatusChangedEvent struct {
 	CurrentStatus bool `json:"currentStatus" bson:"currentStatus"`
 	// PreviousStatus contains previous sync status
 	PreviousStatus bool `json:"previousStatus" bson:"previousStatus"`
-	// SyncType contains the type of sync that changed - tangletime or syncbeacon
-	SyncType string `json:"syncType" bson:"syncType"`
 	// LastConfirmedMessageTime contains time of the last confirmed message
 	LastConfirmedMessageTime time.Time `json:"lastConfirmedMessageTime" bson:"lastConfirmedMessageTime"`
 }
+
+// FPCConflictRecord defines the FPC conflict record to sent be to remote logger.
+type FPCConflictRecord struct {
+	// Type defines the type of the message.
+	Type string `json:"type" bson:"type"`
+	// ConflictID defines the ID of the conflict.
+	ConflictID string `json:"conflictid" bson:"conflictid"`
+	// NodeID defines the ID of the node.
+	NodeID string `json:"nodeid" bson:"nodeid"`
+	// Rounds defines number of rounds performed to finalize the conflict.
+	Rounds int `json:"rounds" bson:"rounds"`
+	// Opinions contains the opinion of each round.
+	Opinions []int32 `json:"opinions" bson:"opinions"`
+	// Outcome defines final opinion of the conflict.
+	Outcome int32 `json:"outcome,omitempty" bson:"outcome,omitempty"`
+	// Time defines the time when the conflict has been finalized.
+	Time time.Time `json:"datetime" bson:"datetime"`
+}

packages/remotelogmetrics/metrics.go

@@ -13,8 +13,7 @@ var (
 
 func newEvents() *CollectionLogEvents {
 	return &CollectionLogEvents{
-		TangleTimeSyncChanged: events.NewEvent(boolCaller),
-		SyncBeaconSyncChanged: events.NewEvent(boolCaller),
+		TangleTimeSyncChanged: events.NewEvent(SyncStatusChangedEventCaller),
 	}
 }
 

packages/tangle/approvalweightmanager.go

@@ -181,9 +181,12 @@ func (a *ApprovalWeightManager) firstUnconfirmedMarkerIndex(sequenceID markers.S
 	index, exists := a.lastConfirmedMarkers[sequenceID]
 	if !exists {
 		a.tangle.Booker.MarkersManager.Manager.Sequence(sequenceID).Consume(func(sequence *markers.Sequence) {
-			index = sequence.LowestIndex() - 1
+			index = sequence.LowestIndex()
 		})
-		a.lastConfirmedMarkers[sequenceID] = index
+		for ; a.Events.MarkerConfirmation.Level(*markers.NewMarker(sequenceID, index)) != 0; index++ {
+			a.lastConfirmedMarkers[sequenceID] = index
+		}
+		return
 	}
 
 	index++

packages/tangle/messagefactory.go

@@ -73,7 +73,7 @@ func (f *MessageFactory) IssuePayload(p payload.Payload, t ...*Tangle) (*Message
 	}
 
 	f.issuanceMutex.Lock()
-	defer f.issuanceMutex.Unlock()
+
 	sequenceNumber, err := f.sequence.Next()
 	if err != nil {
 		err = errors.Errorf("could not create sequence number: %w", err)
@@ -118,6 +118,8 @@ func (f *MessageFactory) IssuePayload(p payload.Payload, t ...*Tangle) (*Message
 		return nil, err
 	}
 
+	f.issuanceMutex.Unlock()
+
 	// create the signature
 	signature := f.sign(strongParents, weakParents, issuingTime, issuerPublicKey, sequenceNumber, p, nonce)
 

packages/tangle/parser.go

@@ -313,7 +313,7 @@ type PowFilter struct {
 	worker     *pow.Worker
 	difficulty int
 
-	mu             sync.Mutex
+	mu             sync.RWMutex
 	acceptCallback func([]byte, *peer.Peer)
 	rejectCallback func([]byte, error, *peer.Peer)
 }
@@ -329,10 +329,10 @@ func NewPowFilter(worker *pow.Worker, difficulty int) *PowFilter {
 // Filter checks whether the given bytes pass the PoW validation and calls the corresponding callback.
 func (f *PowFilter) Filter(msgBytes []byte, p *peer.Peer) {
 	if err := f.validate(msgBytes); err != nil {
-		f.reject(msgBytes, err, p)
+		f.getRejectCallback()(msgBytes, err, p)
 		return
 	}
-	f.accept(msgBytes, p)
+	f.getAcceptCallback()(msgBytes, p)
 }
 
 // OnAccept registers the given callback as the acceptance function of the filter.
@@ -349,20 +349,18 @@ func (f *PowFilter) OnReject(callback func([]byte, error, *peer.Peer)) {
 	f.rejectCallback = callback
 }
 
-func (f *PowFilter) accept(msgBytes []byte, p *peer.Peer) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if f.acceptCallback != nil {
-		f.acceptCallback(msgBytes, p)
-	}
+func (f *PowFilter) getAcceptCallback() (result func(msgBytes []byte, peer *peer.Peer)) {
+	f.mu.RLock()
+	defer f.mu.RUnlock()
+	result = f.acceptCallback
+	return
 }
 
-func (f *PowFilter) reject(msgBytes []byte, err error, p *peer.Peer) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if f.rejectCallback != nil {
-		f.rejectCallback(msgBytes, err, p)
-	}
+func (f *PowFilter) getRejectCallback() (result func(msgBytes []byte, err error, p *peer.Peer)) {
+	f.mu.RLock()
+	defer f.mu.RUnlock()
+	result = f.rejectCallback
+	return
 }
 
 func (f *PowFilter) validate(msgBytes []byte) error {
@@ -435,16 +433,16 @@ func (r *RecentlySeenBytesFilter) OnReject(callback func(bytes []byte, err error
 }
 
 func (r *RecentlySeenBytesFilter) getAcceptCallback() (result func(bytes []byte, peer *peer.Peer)) {
-	r.onAcceptCallbackMutex.Lock()
+	r.onAcceptCallbackMutex.RLock()
 	result = r.onAcceptCallback
-	r.onAcceptCallbackMutex.Unlock()
+	r.onAcceptCallbackMutex.RUnlock()
 	return
 }
 
 func (r *RecentlySeenBytesFilter) getRejectCallback() (result func(bytes []byte, err error, peer *peer.Peer)) {
-	r.onRejectCallbackMutex.Lock()
+	r.onRejectCallbackMutex.RLock()
 	result = r.onRejectCallback
-	r.onRejectCallbackMutex.Unlock()
+	r.onRejectCallbackMutex.RUnlock()
 	return
 }