Merge pull request #1552 from iotaledger/develop

Merge v0.7.4 changes to master

CHANGELOG.md

@@ -1,3 +1,13 @@
+# v0.7.4 - 2021-07-08
+* Add ParametersDefinition structs in integration test framework
+* Add UTXO-DAG interface
+* Fix setting correct properties to newly forked branches
+* Fix using weak parents for direct approvers when looking for transactions that are approved by a message
+* Update entry node from community
+* Update docs
+* Update snapshot file with DevNet UTXO at 2021-07-08 07:09 UTC
+* **Breaking**: bumps network and database versions
+
 # v0.7.3 - 2021-07-01
 * Add latest snapshot download from S3
 * Add weak tips on local dashboard

config.default.json

@@ -14,7 +14,7 @@
   "autopeering": {
     "entryNodes": [
       "[email protected]:15626",
-      "5EDH4uY78EA6wrBkHHAVBWBMDt7EcksRq6pjzipoW15B@entryshimmer.tanglebay.com:14646"
+      "5EDH4uY78EA6wrBkHHAVBWBMDt7EcksRq6pjzipoW15B@entry-devnet.tanglebay.com:14646"
     ],
     "port": 14626
   },

docker-compose.yml

@@ -42,7 +42,7 @@ services:
       - PROMETHEUS_BINDADDRESS=0.0.0.0:9311
     command: >
       --skip-config=true
-      --autopeering.entryNodes=2PV5487xMw5rasGBXXWeqSi4hLz7r19YBt8Y1TGAsQbj@ressims.iota.cafe:15626,5EDH4uY78EA6wrBkHHAVBWBMDt7EcksRq6pjzipoW15B@entryshimmer.tanglebay.com:14646
+      --autopeering.entryNodes=2PV5487xMw5rasGBXXWeqSi4hLz7r19YBt8Y1TGAsQbj@ressims.iota.cafe:15626,5EDH4uY78EA6wrBkHHAVBWBMDt7EcksRq6pjzipoW15B@entry-devnet.tanglebay.com:14646
       --node.disablePlugins=
       --node.enablePlugins=remotelog,networkdelay,spammer,prometheus
       --logger.level=info

docs/implementation_design/configuration_parameters.md

@@ -67,7 +67,8 @@ config.Node().Int(CfgGossipPort)
 Defining configuration parameters using the new way is really similar, however the parameters are not registered directly with the package reading the configuration,
 but rather with our custom package that contains all the logic required to make it work seamlessly. 
 
-In this approach, instead of defining a parameter name, a new structure is defined with all necessary parameters, their default values and usage descriptions using Go's struct field tags.
+In this approach, instead of defining a parameter name, a new type is defined with all necessary parameters, their default values and usage descriptions using Go's struct field tags.
+A variable is then initialized with the defined type.
 
 One difference is that parameter names do not contain the namespace they belong to, the namespace is set when registering the parameters structure with the `configuration` package. One `parameters.go` file can contain definitions and register multiple parameter structures.
 
@@ -77,7 +78,7 @@ package customPlugin
 import "github.com/iotaledger/hive.go/configuration"
 
 // Parameters contains the configuration parameters used by the custom plugin.
-var Parameters = struct {
+type ParametersDefinition struct {
 	// ParamName contains some value used within the plugin
 	ParamName float64 `default:"0.31" usage:"ParamName used in some calculation"`
 
@@ -88,10 +89,12 @@ var Parameters = struct {
 		// DetailedParam2 is the example value
 		DetailedParam2        string `default:"defaultValue" usage:"DetailedParam2 used in the plugin"`
 	}
-}{}
+}
+
+var Parameters = &ParametersDefinition{}
 
 func init() {
-	configuration.BindParameters(&Parameters, "customPlugin")
+	configuration.BindParameters(Parameters, "customPlugin")
 }
 ```
 

docs/tutorials/setup.md

@@ -191,7 +191,7 @@ services:
       - PROMETHEUS_BINDADDRESS=0.0.0.0:9311
     command: >
       --skip-config=true
-      --autopeering.entryNodes=2PV5487xMw5rasGBXXWeqSi4hLz7r19YBt8Y1TGAsQbj@ressims.iota.cafe:15626,5EDH4uY78EA6wrBkHHAVBWBMDt7EcksRq6pjzipoW15B@entryshimmer.tanglebay.com:14646
+      --autopeering.entryNodes=2PV5487xMw5rasGBXXWeqSi4hLz7r19YBt8Y1TGAsQbj@ressims.iota.cafe:15626,5EDH4uY78EA6wrBkHHAVBWBMDt7EcksRq6pjzipoW15B@entry-devnet.tanglebay.com:14646
       --node.disablePlugins=
       --node.enablePlugins=remotelog,networkdelay,spammer,prometheus
       --logger.level=info
@@ -409,7 +409,7 @@ chmod -R 777 prometheus
 #### Create Grafana configs
 1. Create necessary config dirs in `/opt/goshimmer/`.
 ```
-mkdir -p grafana/provisioning/datasources grafana/provisioning/dashboards grafana/provisioning/notifiers
+mkdir -p grafana/provisioning/datasources grafana/provisioning/dashboards grafana/provisioning/notifiers grafana/provisioning/plugins
 mkdir -p grafana/dashboards
 ```
 2. Create a datasource configuration file in `grafana/provisioning/datasources`:

go.mod

@@ -12,7 +12,6 @@ require (
 	github.com/cockroachdb/errors v1.8.4
 	github.com/drand/drand v1.1.1
 	github.com/drand/kyber v1.1.2
-	github.com/emirpasic/gods v1.12.0
 	github.com/gin-gonic/gin v1.6.3
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/go-resty/resty/v2 v2.6.0

packages/ledgerstate/utxo_dag.go

@@ -24,10 +24,54 @@ import (
 
 // region UTXODAG //////////////////////////////////////////////////////////////////////////////////////////////////////
 
+// IUTXODAG is the interface for UTXODAG which is the core of the ledger state
+// that is formed by Transactions consuming Inputs and creating Outputs.  It represents all the methods
+// that helps to keep track of the balances and the different perceptions of potential conflicts.
+type IUTXODAG interface {
+	// Events returns all events of the UTXODAG
+	Events() *UTXODAGEvents
+	// Shutdown shuts down the UTXODAG and persists its state.
+	Shutdown()
+	// CheckTransaction contains fast checks that have to be performed before booking a Transaction.
+	CheckTransaction(transaction *Transaction) (err error)
+	// BookTransaction books a Transaction into the ledger state.
+	BookTransaction(transaction *Transaction) (targetBranch BranchID, err error)
+	// InclusionState returns the InclusionState of the Transaction with the given TransactionID which can either be
+	// Pending, Confirmed or Rejected.
+	InclusionState(transactionID TransactionID) (inclusionState InclusionState, err error)
+	// CachedTransaction retrieves the Transaction with the given TransactionID from the object storage.
+	CachedTransaction(transactionID TransactionID) (cachedTransaction *CachedTransaction)
+	// Transaction returns a specific transaction, consumed.
+	Transaction(transactionID TransactionID) (transaction *Transaction)
+	// Transactions returns all the transactions, consumed.
+	Transactions() (transactions map[TransactionID]*Transaction)
+	// CachedTransactionMetadata retrieves the TransactionMetadata with the given TransactionID from the object storage.
+	CachedTransactionMetadata(transactionID TransactionID) (cachedTransactionMetadata *CachedTransactionMetadata)
+	// CachedOutput retrieves the Output with the given OutputID from the object storage.
+	CachedOutput(outputID OutputID) (cachedOutput *CachedOutput)
+	// CachedOutputMetadata retrieves the OutputMetadata with the given OutputID from the object storage.
+	CachedOutputMetadata(outputID OutputID) (cachedOutput *CachedOutputMetadata)
+	// CachedConsumers retrieves the Consumers of the given OutputID from the object storage.
+	CachedConsumers(outputID OutputID) (cachedConsumers CachedConsumers)
+	// LoadSnapshot creates a set of outputs in the UTXO-DAG, that are forming the genesis for future transactions.
+	LoadSnapshot(snapshot *Snapshot)
+	// CachedAddressOutputMapping retrieves the outputs for the given address.
+	CachedAddressOutputMapping(address Address) (cachedAddressOutputMappings CachedAddressOutputMappings)
+	// SetTransactionConfirmed marks a Transaction (and all Transactions in its past cone) as confirmed. It also marks the
+	// conflicting Transactions to be rejected.
+	SetTransactionConfirmed(transactionID TransactionID) (err error)
+	// ConsumedOutputs returns the consumed (cached)Outputs of the given Transaction.
+	ConsumedOutputs(transaction *Transaction) (cachedInputs CachedOutputs)
+	// ManageStoreAddressOutputMapping mangages how to store the address-output mapping dependent on which type of output it is.
+	ManageStoreAddressOutputMapping(output Output)
+	// StoreAddressOutputMapping stores the address-output mapping.
+	StoreAddressOutputMapping(address Address, outputID OutputID)
+}
+
 // UTXODAG represents the DAG that is formed by Transactions consuming Inputs and creating Outputs. It forms the core of
 // the ledger state and keeps track of the balances and the different perceptions of potential conflicts.
 type UTXODAG struct {
-	Events *UTXODAGEvents
+	events *UTXODAGEvents
 
 	transactionStorage          *objectstorage.ObjectStorage
 	transactionMetadataStorage  *objectstorage.ObjectStorage
@@ -44,7 +88,7 @@ func NewUTXODAG(store kvstore.KVStore, cacheProvider *database.CacheTimeProvider
 	options := buildObjectStorageOptions(cacheProvider)
 	osFactory := objectstorage.NewFactory(store, database.PrefixLedgerState)
 	utxoDAG = &UTXODAG{
-		Events: &UTXODAGEvents{
+		events: &UTXODAGEvents{
 			TransactionBranchIDUpdated: events.NewEvent(transactionIDEventHandler),
 			TransactionConfirmed:       events.NewEvent(transactionIDEventHandler),
 		},
@@ -59,6 +103,11 @@ func NewUTXODAG(store kvstore.KVStore, cacheProvider *database.CacheTimeProvider
 	return
 }
 
+// Events returns all events of the UTXODAG
+func (u *UTXODAG) Events() *UTXODAGEvents {
+	return u.events
+}
+
 // Shutdown shuts down the UTXODAG and persists its state.
 func (u *UTXODAG) Shutdown() {
 	u.shutdownOnce.Do(func() {
@@ -351,7 +400,7 @@ func (u *UTXODAG) SetTransactionConfirmed(transactionID TransactionID) (err erro
 			continue
 		}
 
-		u.Events.TransactionConfirmed.Trigger(currentTransactionID)
+		u.Events().TransactionConfirmed.Trigger(currentTransactionID)
 	}
 
 	return err
@@ -515,10 +564,16 @@ func (u *UTXODAG) forkConsumer(transactionID TransactionID, conflictingInputs Ou
 		if err != nil {
 			panic(fmt.Errorf("failed to create ConflictBranch when forking Transaction with %s: %w", transactionID, err))
 		}
-		cachedConsumingConflictBranch.Release()
+		cachedConsumingConflictBranch.Consume(func(newBranch Branch) {
+			// copying the branch metadata properties from the original branch to the newly created.
+			u.branchDAG.Branch(txMetadata.BranchID()).Consume(func(oldBranch Branch) {
+				newBranch.setLiked(oldBranch.Liked())
+				newBranch.setMonotonicallyLiked(oldBranch.MonotonicallyLiked())
+			})
+		})
 
 		txMetadata.SetBranchID(conflictBranchID)
-		u.Events.TransactionBranchIDUpdated.Trigger(transactionID)
+		u.Events().TransactionBranchIDUpdated.Trigger(transactionID)
 
 		outputIds := u.createdOutputIDsOfTransaction(transactionID)
 		for _, outputID := range outputIds {
@@ -566,7 +621,7 @@ func (u *UTXODAG) propagateBranchUpdates(transactionID TransactionID) (updatedOu
 func (u *UTXODAG) updateBranchOfTransaction(transactionID TransactionID, branchID BranchID) (updatedOutputs []OutputID) {
 	if !u.CachedTransactionMetadata(transactionID).Consume(func(transactionMetadata *TransactionMetadata) {
 		if transactionMetadata.SetBranchID(branchID) {
-			u.Events.TransactionBranchIDUpdated.Trigger(transactionID)
+			u.Events().TransactionBranchIDUpdated.Trigger(transactionID)
 
 			updatedOutputs = u.createdOutputIDsOfTransaction(transactionID)
 			for _, outputID := range updatedOutputs {

packages/tangle/booker.go

@@ -57,7 +57,7 @@ func (b *Booker) Setup() {
 		}
 	}))
 
-	b.tangle.LedgerState.UTXODAG.Events.TransactionBranchIDUpdated.Attach(events.NewClosure(func(transactionID ledgerstate.TransactionID) {
+	b.tangle.LedgerState.UTXODAG.Events().TransactionBranchIDUpdated.Attach(events.NewClosure(func(transactionID ledgerstate.TransactionID) {
 		if err := b.BookConflictingTransaction(transactionID); err != nil {
 			b.Events.Error.Trigger(errors.Errorf("failed to propagate ConflictBranch of %s to tangle: %w", transactionID, err))
 		}

packages/tangle/booker_test.go

@@ -151,6 +151,11 @@ func TestScenario_1(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, branches["green"], txBranchID)
 
+	assert.True(t, tangle.LedgerState.BranchDAG.Branch(txBranchID).Consume(func(branch ledgerstate.Branch) {
+		assert.True(t, branch.Liked())
+		assert.True(t, branch.MonotonicallyLiked())
+	}))
+
 	// Message 5
 	outputs["F"] = ledgerstate.NewSigLockedSingleOutput(1, wallets["F"].address)
 	transactions["4"] = makeTransaction(ledgerstate.NewInputs(inputs["A"]), ledgerstate.NewOutputs(outputs["F"]), outputsByID, walletsByAddress)
@@ -180,6 +185,18 @@ func TestScenario_1(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, branches["red"], txBranchID)
 
+	// assess that after forking transaction 3 and thus introducing the red branch, the properties of that branch are correct
+	assert.True(t, tangle.LedgerState.BranchDAG.Branch(branches["red"]).Consume(func(branch ledgerstate.Branch) {
+		assert.True(t, branch.Liked())
+		assert.True(t, branch.MonotonicallyLiked())
+	}))
+
+	// assess that the properties of the yellow branch are correct
+	assert.True(t, tangle.LedgerState.BranchDAG.Branch(branches["yellow"]).Consume(func(branch ledgerstate.Branch) {
+		assert.False(t, branch.Liked())
+		assert.False(t, branch.MonotonicallyLiked())
+	}))
+
 	// Message 6
 	inputs["E"] = ledgerstate.NewUTXOInput(ledgerstate.NewOutputID(transactions["2"].ID(), 0))
 	outputsByID[inputs["E"].ReferencedOutputID()] = ledgerstate.NewOutputs(outputs["E"])[0]

packages/tangle/ledgerstate.go

@@ -17,7 +17,7 @@ import (
 type LedgerState struct {
 	tangle    *Tangle
 	BranchDAG *ledgerstate.BranchDAG
-	UTXODAG   *ledgerstate.UTXODAG
+	UTXODAG   ledgerstate.IUTXODAG
 
 	totalSupply uint64
 }

packages/tangle/tangle_test.go

@@ -534,7 +534,7 @@ func TestTangle_Flow(t *testing.T) {
 	}))
 
 	// data messages should not trigger this event
-	tangle.LedgerState.UTXODAG.Events.TransactionConfirmed.AttachAfter(events.NewClosure(func(transactionID ledgerstate.TransactionID) {
+	tangle.LedgerState.UTXODAG.Events().TransactionConfirmed.AttachAfter(events.NewClosure(func(transactionID ledgerstate.TransactionID) {
 		n := atomic.AddInt32(&opinionFormedTransactions, 1)
 		t.Logf("opinion formed transaction %d/%d - %s", n, totalMsgCount, transactionID)
 	}))

packages/tangle/utils.go

@@ -3,6 +3,8 @@ package tangle
 import (
 	"fmt"
 
+	"github.com/iotaledger/hive.go/typeutils"
+
 	"github.com/iotaledger/hive.go/datastructure/walker"
 	"github.com/iotaledger/hive.go/types"
 
@@ -109,46 +111,40 @@ func (u *Utils) AllTransactionsApprovedByMessages(transactionIDs ledgerstate.Tra
 // TransactionApprovedByMessage checks if the Transaction was attached by at least one Message that was directly or
 // indirectly approved by the given Message.
 func (u *Utils) TransactionApprovedByMessage(transactionID ledgerstate.TransactionID, messageID MessageID) (approved bool) {
-	for _, attachmentMessageID := range u.tangle.Storage.AttachmentMessageIDs(transactionID) {
-		if attachmentMessageID == messageID {
+	attachmentMessageIDs := u.tangle.Storage.AttachmentMessageIDs(transactionID)
+	for i := range attachmentMessageIDs {
+		if attachmentMessageIDs[i] == messageID {
 			return true
 		}
 
 		var attachmentBooked bool
-		u.tangle.Storage.MessageMetadata(attachmentMessageID).Consume(func(attachmentMetadata *MessageMetadata) {
+		u.tangle.Storage.MessageMetadata(attachmentMessageIDs[i]).Consume(func(attachmentMetadata *MessageMetadata) {
 			attachmentBooked = attachmentMetadata.IsBooked()
 		})
 		if !attachmentBooked {
 			continue
 		}
 
 		bookedParents := make(MessageIDs, 0)
-		u.tangle.Storage.Message(messageID).Consume(func(message *Message) {
-			for _, parentID := range message.StrongParents() {
-				var parentBooked bool
-				u.tangle.Storage.MessageMetadata(parentID).Consume(func(parentMetadata *MessageMetadata) {
-					parentBooked = parentMetadata.IsBooked()
-				})
-				if !parentBooked {
-					continue
-				}
-
-				// First check all of the parents to avoid unnecessary checks and possible walking.
-				if attachmentMessageID == parentID {
-					approved = true
-					return
-				}
 
-				bookedParents = append(bookedParents, parentID)
+		u.tangle.Storage.Message(messageID).Consume(func(message *Message) {
+			approved = u.checkBookedParents(message, &attachmentMessageIDs[i], func(message *Message) MessageIDs {
+				return message.WeakParents()
+			}, nil)
+			if approved {
+				return
 			}
+			approved = u.checkBookedParents(message, &attachmentMessageIDs[i], func(message *Message) MessageIDs {
+				return message.StrongParents()
+			}, &bookedParents)
 		})
 		if approved {
 			return
 		}
 
 		// Only now check all parents.
 		for _, bookedParent := range bookedParents {
-			if u.MessageApprovedBy(attachmentMessageID, bookedParent) {
+			if u.MessageApprovedBy(attachmentMessageIDs[i], bookedParent) {
 				approved = true
 				return
 			}
@@ -161,6 +157,29 @@ func (u *Utils) TransactionApprovedByMessage(transactionID ledgerstate.Transacti
 	return
 }
 
+// checkBookedParents check if message parents are booked and add then to bookedParents. If we find attachmentMessageId in the parents we stop and return true
+func (u *Utils) checkBookedParents(message *Message, attachmentMessageID *MessageID, getParents func(*Message) MessageIDs, bookedParents *MessageIDs) bool {
+	for _, parentID := range getParents(message) {
+		var parentBooked bool
+		u.tangle.Storage.MessageMetadata(parentID).Consume(func(parentMetadata *MessageMetadata) {
+			parentBooked = parentMetadata.IsBooked()
+		})
+		if !parentBooked {
+			continue
+		}
+
+		// First check all of the parents to avoid unnecessary checks and possible walking.
+		if *attachmentMessageID == parentID {
+			return true
+		}
+
+		if !typeutils.IsInterfaceNil(bookedParents) {
+			*bookedParents = append(*bookedParents, parentID)
+		}
+	}
+	return false
+}
+
 // MessageApprovedBy checks if the Message given by approvedMessageID is directly or indirectly approved by the
 // Message given by approvingMessageID.
 func (u *Utils) MessageApprovedBy(approvedMessageID MessageID, approvingMessageID MessageID) (approved bool) {

packages/txstream/tangleledger/tangleledger.go

@@ -39,7 +39,7 @@ func New() *TangleLedger {
 			go t.txConfirmedEvent.Trigger(transaction)
 		})
 	})
-	messagelayer.Tangle().LedgerState.UTXODAG.Events.TransactionConfirmed.Attach(t.txConfirmedClosure)
+	messagelayer.Tangle().LedgerState.UTXODAG.Events().TransactionConfirmed.Attach(t.txConfirmedClosure)
 
 	t.txBookedClosure = events.NewClosure(func(id tangle.MessageID) {
 		messagelayer.Tangle().Storage.Message(id).Consume(func(msg *tangle.Message) {
@@ -55,7 +55,7 @@ func New() *TangleLedger {
 
 // Detach detaches the event handlers
 func (t *TangleLedger) Detach() {
-	messagelayer.Tangle().LedgerState.UTXODAG.Events.TransactionConfirmed.Detach(t.txConfirmedClosure)
+	messagelayer.Tangle().LedgerState.UTXODAG.Events().TransactionConfirmed.Detach(t.txConfirmedClosure)
 	messagelayer.Tangle().Booker.Events.MessageBooked.Detach(t.txBookedClosure)
 }