Merge pull request #1266 from iotaledger/develop

Merge v0.5.8 to master

CHANGELOG.md

@@ -1,3 +1,24 @@
+# v0.5.8 - 2021-05-07
+* Integrate FPC with the X-Team committee
+* Enable finality via approval weight
+* Add Tangle Time
+* Add sync status monitoring
+* Add activity plugin
+* Add manual peering support
+* Add markers info to message view
+* Remove moving average for cMana
+* Replace epochs with weight provider using Tangle Time
+* Fix marker issues
+* Fix sync issues
+* Disable past-cone check when booking transactions
+* Improve integration tests
+* Refactor error handling
+* Update snapshot
+* Update JS dependencies
+* Update to latest hive.go
+* Update documentation
+* **Breaking**: bumps network and database versions
+
 # v0.5.7 - 2021-04-23
 * Add approval weight manager (soft launch)
 * Add epochs

client/autopeering.go

@@ -8,18 +8,18 @@ import (
 )
 
 const (
-	routeGetNeighbors = "autopeering/neighbors"
+	routeGetAutopeeringNeighbors = "autopeering/neighbors"
 )
 
-// GetNeighbors gets the chosen/accepted neighbors.
+// GetAutopeeringNeighbors gets the chosen/accepted neighbors.
 // If knownPeers is set, also all known peers to the node are returned additionally.
-func (api *GoShimmerAPI) GetNeighbors(knownPeers bool) (*jsonmodels.GetNeighborsResponse, error) {
+func (api *GoShimmerAPI) GetAutopeeringNeighbors(knownPeers bool) (*jsonmodels.GetNeighborsResponse, error) {
 	res := &jsonmodels.GetNeighborsResponse{}
 	if err := api.do(http.MethodGet, func() string {
 		if !knownPeers {
-			return routeGetNeighbors
+			return routeGetAutopeeringNeighbors
 		}
-		return fmt.Sprintf("%s?known=1", routeGetNeighbors)
+		return fmt.Sprintf("%s?known=1", routeGetAutopeeringNeighbors)
 	}(), nil, res); err != nil {
 		return nil, err
 	}

client/faucet.go

@@ -11,10 +11,14 @@ const (
 )
 
 // SendFaucetRequest requests funds from faucet nodes by sending a faucet request payload message.
-func (api *GoShimmerAPI) SendFaucetRequest(base58EncodedAddr string) (*jsonmodels.FaucetResponse, error) {
+func (api *GoShimmerAPI) SendFaucetRequest(base58EncodedAddr string, pledgeIDs ...string) (*jsonmodels.FaucetResponse, error) {
+	aManaPledgeID, cManaPledgeID := "", ""
+	if len(pledgeIDs) > 1 {
+		aManaPledgeID, cManaPledgeID = pledgeIDs[0], pledgeIDs[1]
+	}
 	res := &jsonmodels.FaucetResponse{}
 	if err := api.do(http.MethodPost, routeFaucet,
-		&jsonmodels.FaucetRequest{Address: base58EncodedAddr}, res); err != nil {
+		&jsonmodels.FaucetRequest{Address: base58EncodedAddr, AccessManaPledgeID: aManaPledgeID, ConsensusManaPledgeID: cManaPledgeID}, res); err != nil {
 		return nil, err
 	}
 

client/lib.go

@@ -5,10 +5,11 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
+
+	"github.com/cockroachdb/errors"
 )
 
 var (

client/manualpeering.go

@@ -0,0 +1,45 @@
+package client
+
+import (
+	"net/http"
+
+	"github.com/cockroachdb/errors"
+	"github.com/iotaledger/hive.go/autopeering/peer"
+	"github.com/iotaledger/hive.go/crypto/ed25519"
+
+	"github.com/iotaledger/goshimmer/plugins/manualpeering"
+)
+
+const (
+	routeManualConnectedPeers = "manualpeering/peers/connected"
+	routeManualPeers          = "manualpeering/peers"
+)
+
+// AddManualPeers adds the provided list of peers to the manual peering layer.
+func (api *GoShimmerAPI) AddManualPeers(peers []*peer.Peer) error {
+	if err := api.do(http.MethodPost, routeManualPeers, peers, nil); err != nil {
+		return errors.Wrap(err, "failed to add manual peers via the HTTP API")
+	}
+	return nil
+}
+
+// RemoveManualPeers remove the provided list of peers from the manual peering layer.
+func (api *GoShimmerAPI) RemoveManualPeers(keys []ed25519.PublicKey) error {
+	peersToRemove := make([]*manualpeering.PeerToRemove, len(keys))
+	for i, key := range keys {
+		peersToRemove[i] = &manualpeering.PeerToRemove{PublicKey: key.String()}
+	}
+	if err := api.do(http.MethodDelete, routeManualPeers, peersToRemove, nil); err != nil {
+		return errors.Wrap(err, "failed to remove manual peers via the HTTP API")
+	}
+	return nil
+}
+
+// GetManualConnectedPeers gets the list of connected neighbors from the manual peering layer.
+func (api *GoShimmerAPI) GetManualConnectedPeers() ([]*peer.Peer, error) {
+	var peers []*peer.Peer
+	if err := api.do(http.MethodGet, routeManualConnectedPeers, nil, &peers); err != nil {
+		return nil, errors.Wrap(err, "failed to get manual connected peers from the API")
+	}
+	return peers, nil
+}

client/tools.go

@@ -1,9 +1,10 @@
 package client
 
 import (
-	"errors"
 	"net/http"
 
+	"github.com/cockroachdb/errors"
+
 	"github.com/iotaledger/goshimmer/plugins/webapi/jsonmodels"
 )
 

client/wallet/sendfunds_options.go

@@ -1,7 +1,7 @@
 package wallet
 
 import (
-	"errors"
+	"github.com/cockroachdb/errors"
 
 	"github.com/iotaledger/goshimmer/client/wallet/packages/address"
 	"github.com/iotaledger/goshimmer/packages/ledgerstate"

client/wallet/wallet.go

@@ -1,11 +1,11 @@
 package wallet
 
 import (
-	"errors"
 	"reflect"
 	"time"
 	"unsafe"
 
+	"github.com/cockroachdb/errors"
 	"github.com/iotaledger/hive.go/bitmask"
 	"github.com/iotaledger/hive.go/identity"
 	"github.com/iotaledger/hive.go/marshalutil"

docs/SUMMARY.md

@@ -38,6 +38,7 @@ http://goshimmer.docs.iota.org/
   - [Value layer](./apis/value.md)
 - [Tooling](./tooling.md)
   - [Docker private network](./tooling/docker_private_network.md)
+  - [Integration tests](./tooling/integration_tests.md)
 
 - [Specification of Implementation](./specification.md)
   - [Mana](./specification/001-mana.md)
@@ -50,4 +51,5 @@ http://goshimmer.docs.iota.org/
   - [How to do a release](teamresources/release.md)
   - [Code Guidelines](./teamresources/guidelines.md)
   - [Local development](./teamresources/local_development.md)  
+  - [Modify the Analysis dashboard](../plugins/analysis/dashboard/frontend/README.md)
 

docs/apis/value.md

@@ -2,7 +2,7 @@
 
 The value layer builds on top of the communication layer. It encapsulates the functionality of token transfers, ledger representation, conflict state and consensus via FPC.
 
-Note that the value layer operates on `Value Objects` which is a payload type which is transferred via `Messages`. A `Value Object` encapsulates the transaction spending [UTXOs](https://en.wikipedia.org/wiki/Unspent_transaction_output) to new outputs and references two other `Value Objects`. Please refer to [concepts & layers](https://github.com/iotaledger/goshimmer/wiki/Concepts-&-Layers) for a more in depth explanation of the value layer.
+Note that the value layer operates on `Value Objects` which is a payload type which is transferred via `Messages`. A `Value Object` encapsulates the transaction spending [UTXOs](https://en.wikipedia.org/wiki/Unspent_transaction_output) to new outputs and references two other `Value Objects`. Please refer to [concepts & layers](../concepts/layers.md) for a more in depth explanation of the value layer.
 
 The API provides multiple functions to interact with the value layer and retrieve/issue the corresponding objects described above.
 

docs/application_logic/protocol.md

@@ -1,4 +1,49 @@
-# Protocol high level overview
+# Protocol high-level overview
+
+To orientate the reader, we provide a high-level overview of the protocol, following the natural life cycle of a message. The first module used—while the message is still being created—, is the **Tip Selection**(ADD LINK) module. Here, the node must choose a certain number (from two to eight) of other messages to reference, meaning that the newly created message will be cryptographically attached to these referenced messages. An honest node must always choose tips uniformly at random from a tip pool, i.e., from a set of still unreferenced messages that satisfy a certain set of conditions, as discussed on INSERT LINK. In the diagram below, the issuance process being described now is represented in the context of the complete protocol. 
+
+![title](Protocol_overview_own_message.png)
+
+Each node in the network has limited bandwidth, CPU, and memory. In order to avoid any node from being overloaded, the right to write in everybody else's Tangle is regulated by the **Rate and Congestion Control Module**. The first one dictates the maximum rate of issuance of messages by the introduction of a small amount of proof of work. However, if an attacker accelerates the message issuance—effectively flooding the network—, the difficulty of the proof of work for that node will increase exponentially. Eventually, the attacker will be incapable of issuing new messages. 
+
+Later, the Congestion control module fairly allocates the network resources accordingly to a quantity called **Access Mana**, which acts as a [Sybil protection](https://en.wikipedia.org/wiki/Sybil_attack) mechanism. The exact manner in which Access Mana is acquired is discussed in INSERT LINK. But—for the purposes of this overview—we can summarize Access Mana as a scarce resource, that makes an effective Sybil protection mechanism. Thus, each node has the right to issue messages at a rate proportional to their Access Mana. This fair rate is not constant (since the utilization of the network may fluctuate), and to correctly set its own individual rate of issuance of messages, each node uses a mechanism called the **Rate Setter**, that makes the average issuance rate of the node converge to the fair rate guaranteed by Access Mana. Nodes that do not use the rate Setter will be punished by either the Rate Control or the Congestion Control Module.
+
+Between the Rate Setter and the actual gossip of the message, several steps will take place, but—for the sake of clearness—we ignore these steps for now and return to this subject later. Then, assuming that the message was properly created, it will be propagated to the rest of the network. Since we deal with a large number of nodes, the communication graph cannot be [complete](https://en.wikipedia.org/wiki/Complete_graph). Thus, the [network topology](https://en.wikipedia.org/wiki/Network_topology) will be dictated by the **Neighbor Selection** (aka Auto-Peering) module, described in INSERT LINK. 
+
+
+![title](Protocol_overview_received_message.png)
+
+We turn our attention now to another point of view: the one of the nodes receiving new messages, represented in the diagram above. After receiving a message, the node will perform several **syntactical verifications**, that will act as a filter to the messages. Additionally, the message has to be **solidified**, meaning that the node must know all the past cone of the message (i.e., the set of all messages directly or indirectly referenced by the message in question). After this step, the node places all the messages left into an inbox (OUTBOX OR INBOX??). At a fixed global rate (meaning that all nodes use the same rate), the node uses a **scheduler** to choose a message from the inbox. This scheduler—that, technically, is part of the aforementioned congestion control mechanism—works as a gatekeeper, effectively regulating the use of the most scarce resources of the nodes. Since the scheduler works at a fixed rate, the network cannot be overwhelmed. As discussed in INSERT LINK, the scheduler is designed to ensure—even in the presence of attackers—the following properties:
+
+1. **Consistency**: all honest nodes will schedule the same messages
+2. **Fair access**: the nodes' messages will be scheduled at a fair rate according to their Access Mana
+3. **Bounded latency**: the processing time of all messages will be bounded
+
+Only after passing the scheduler the messages can be written into the local Tangle. To do that, the nodes perform most of the **semantic validation**, such as the search for irreconcilable conflicts in the message's past cone or (in the case of value transfers) unlock condition checks. At this point (if the message passes these tests), the message will be **booked** into the **local Tangle** of the node. Additionally, in the case of a value transfer, the **ledger state** and two vectors called Access Mana Vector (already mentioned in this text) and **Consensus Mana** Vector are updated accordingly. The Consensus Mana is another Sybil protection mechanism which—since it is applied to different modules than Access Mana—has the need of a different calculation (for more details on that subject, see INSERT LINK). 
+
+![title](Protocol_overview_booking.png)
+
+After having the message booked, the node is free to **gossip** it, but a crucial step of the protocol is still missing: the **Opinion Setter** and the voting protocol, that deal with the most subjective parts of the consensus mechanism (notice that, until now, the protocol has mostly dealt with objective checks). The voting protocol used here is the FPC (or **Fast Probabilistic Consensus**), which is a binary voting protocol that allows a large group of nodes to come to a consensus on the value of a single bit. The FPC begins with each node having an initial opinion, set using the node's local time perception and ordering of the messages. The nodes must set opinions about two subjects: 
+
+1. **The legitimacy of the timestamp of the message**: Whenever a node issues a message, it adds a timestamp to it, which should represent the local time of issuance (as seen by the issuer node). The other nodes will judge if this timestamp is reasonable, by checking if it is too far away from their own local clock. The specific rule to do this check is described in LINK.
+2. In the case of a value transfer, **whether it is a conflict**: We use the **FCoB Rule** (ADD LINK). Roughly, the node will have a positive opinion about a transaction A if and only if all its conflicts arrived later than a certain time interval after A's arrival. 
+
+In each round, nodes randomly choose other nodes to query about their opinions about one of the subjects above. The querying node changes its own opinion if the number of responses with a different opinion than it is greater than a certain threshold. In order to prevent liveness attacks, this threshold is determined by a random number issued by a committee of high Consensus Mana nodes via the **dRNG** application. Without the random threshold, an attacker could lie about its responses in a way that could prevent the protocol from terminating. Finally, a node will finalize on a certain opinion after holding it for a certain number of rounds. 
+
+When selecting which other nodes to query, a node must weight the list of all nodes by Consensus Mana. Thus, high Consensus Mana nodes are queried more often than low Consensus Mana nodes. This makes it difficult for an attacker to manipulate the outcome of the voting. Unless the attacker controls more than 1/3 of the Consensus Mana in the system, with high probability, we know that FPC has the following properties:
+
+1. **Termination**: every honest node will finalize on some opinion before a maximum round number.
+2. **Agreement**: all honest nodes will finalize on the same opinion.
+3. **Integrity**: if a super majority of nodes—e.g. more than 90% weighted by Consensus Mana—, have the same initial opinion, then FPC will terminate with that value.
+
+![title](Protocol_overview_consensus.png)
+
+Some concepts or modules implemented—even not strictly being part of the protocol—have an extreme importance in the understanding of the protocol as a whole, such as the **Grades of Finality** (add link). Analogously to Bitcoin's [six blocks rule](https://en.bitcoin.it/wiki/Confirmation), our protocol also has certain measures of the probability of a certain message being considered valid permanently by all nodes. The four possible grades are:
+
+1. **Grade 1**: If 10 seconds have passed since the time the message arrived and no conflict was seen.
+2. **Grade 2**: If 20 seconds have passed since the time the message arrived and no conflict was seen.
+3. **Grade 3**: If the message achieved grade 2 and it accumulated enough approval weight (see LINK for more details)
+4. **Grade 4**: If the message achieved grade 3 and 30 minutes (??????????) have passed since the timestamp of the message
+
+For more details on the grades above, see LINK.
 
-will link (in the future) to specific specs
-- Overall data flow might help here