Merge pull request #1453 from iotaledger/develop

Merge v0.7.1 changes to master
iotaledger · Jun 15, 2021 · 8fca0bb · 8fca0bb
2 parents a05c532 + 020121e
commit 8fca0bb
Show file tree

Hide file tree

Showing 80 changed files with 1,017 additions and 680 deletions.
diff --git a/.github/workflows/client-import.yml b/.github/workflows/client-import.yml
@@ -0,0 +1,27 @@
+name: Client Lib Import Check
+
+on:
+  push:
+    paths-ignore:
+      - 'docs/**'
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+
+jobs:
+
+  build:
+    name: Import Check
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Setup Go 1.16.3
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.16.3
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Run Import Check Script
+        run: ./scripts/client_import_check.sh
diff --git a/.github/workflows/docker-develop.yml b/.github/workflows/docker-develop.yml
@@ -14,13 +14,38 @@ jobs:
     env:
       DOCKER_BUILDKIT: 1
     steps:
-      -   name: Checkout repository
-          uses: actions/checkout@v2
-
-      -   name: Publish to Docker Hub
-          uses: docker/build-push-action@92e71463491f2d026a477188b8ad3a0fdd9d672c
-          with:
-            repository: iotaledger/goshimmer
-            username: '${{ secrets.IOTALEDGER_HUB_DOCKER_LOGIN }}'
-            password: '${{ secrets.IOTALEDGER_HUB_DOCKER_PASSWORD }}'
-            tags: develop
+      - name: Checkout repository
+        uses: actions/[email protected]
+
+      - name: Set up Docker Buildx
+        uses: docker/[email protected]
+
+      - name: Cache Docker layers
+        uses: actions/[email protected]
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Login to DockerHub
+        uses: docker/[email protected]
+        with:
+          username: '${{ secrets.IOTALEDGER_HUB_DOCKER_LOGIN }}'
+          password: '${{ secrets.IOTALEDGER_HUB_DOCKER_PASSWORD }}'
+
+      - name: Publish to Docker Hub
+        uses: docker/[email protected]
+        with:
+          tags: iotaledger/goshimmer:develop
+          push: true
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new
+
+      - # Temp fix
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+        name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -61,7 +61,6 @@ jobs:
       - name: Pull additional Docker images
         run: |
           docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiaadm/pumba:0.7.2
           docker pull gaiadocker/iproute2:latest
 
       - name: Run integration tests
@@ -95,7 +94,6 @@ jobs:
       - name: Pull additional Docker images
         run: |
           docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiaadm/pumba:0.7.2
           docker pull gaiadocker/iproute2:latest
 
       - name: Run integration tests
@@ -130,7 +128,6 @@ jobs:
       - name: Pull additional Docker images
         run: |
           docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiaadm/pumba:0.7.2
           docker pull gaiadocker/iproute2:latest
 
       - name: Run integration tests
@@ -149,43 +146,6 @@ jobs:
           path: tools/integration-tests/logs
 
 
-
-  message:
-    name: message
-    env:
-      TEST_NAME: message
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Build GoShimmer image
-        run: docker build -t iotaledger/goshimmer .
-
-      - name: Pull additional Docker images
-        run: |
-          docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiaadm/pumba:0.7.2
-          docker pull gaiadocker/iproute2:latest
-
-      - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
-
-      - name: Create logs from tester
-        if: always()
-        run: |
-          docker logs tester &> tools/integration-tests/logs/tester.log
-
-      - name: Save logs as artifacts
-        if: always()
-        uses: actions/upload-artifact@v1
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: tools/integration-tests/logs
-
-
-
   value:
     name: value
     env:
@@ -202,7 +162,6 @@ jobs:
       - name: Pull additional Docker images
         run: |
           docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiaadm/pumba:0.7.2
           docker pull gaiadocker/iproute2:latest
 
       - name: Run integration tests
@@ -236,7 +195,6 @@ jobs:
       - name: Pull additional Docker images
         run: |
           docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiaadm/pumba:0.7.2
           docker pull gaiadocker/iproute2:latest
 
       - name: Run integration tests
@@ -270,7 +228,6 @@ jobs:
       - name: Pull additional Docker images
         run: |
           docker pull angelocapossele/drand:1.1.3
-          docker pull gaiaadm/pumba:latest
           docker pull gaiadocker/iproute2:latest
 
       - name: Run integration tests

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,26 @@
+# v0.7.1 - 2021-06-15
+* Improve Faucet
+* Improve docs
+* Improve message requester
+* Improve common integration test
+* Improve collected Elasticsearch metrics
+* Add run callback to database plugin
+* Add Client Library Import Check
+* Add time range check in wallet lib
+* Replace builtin hash map with a sorted set
+* Enable Github Actions Caching
+* Fix wrong account mention
+* Fix gossip neighbor disconnect
+* Fix Genesis Loaded Transaction View
+* Fix memory leak in pow plugin/pkg
+* Fix overlocking and sleeping issues 
+* Fix snapshot bug
+* Fix typos
+* Update Graphana debugging dashboard
+* Update snapshot file with DevNet UTXO at 2021-06-15 09:09 UTC
+* Update JS dependencies
+* **Breaking**: bumps network and database versions
+
 # v0.7.0 - 2021-06-02
 * Add client diagnostic API
 * Improve docs and tutorials

diff --git a/README.md b/README.md
@@ -32,30 +32,30 @@
 
 This repository, called GoShimmer, is where the IOTA Foundation's Research Department tests the IOTA 2.0 modules to study and evaluate their performance.
 
-GoShimmer is first and foremost a research prototype. As such, braking changes can often happen. We invite researchers and developers to make use of this project as you see fit. Running experiments, test out new ideas, build PoC are all very welcome initiatives.
+GoShimmer is first and foremost a research prototype. As such, breaking changes can often happen. We invite researchers and developers to make use of this project as you see fit. Running experiments, test out new ideas, build PoC are all very welcome initiatives.
 
 For a documentation, including tutorials and resources, we refer to the [Documentation](http://goshimmer.docs.iota.org/) (Github [link](https://github.com/iotaledger/goshimmer/blob/develop/docs/SUMMARY.md)).
 
 ## Design
-The code in GoShimmer is modular, where each module represents either one of the *IOTA 2.0 components* or a basic node function such as the gossip, ledger state, and API - just to mention a few.  
+The code in GoShimmer is modular, where each module represents either one of the *IOTA 2.0 components*, or a basic node function such as the gossip, ledger state, and API - just to mention a few.  
 
 ![Layers](docs/protocol_specification/layers.png)
 
-GoShimmer's modularity is based on a combination of event-driven and layer-based approaches.
+GoShimmer's modularity is based on a combination of [event-driven](https://goshimmer.docs.iota.org/implementation_design/event_driven_model.html) and [layer-based](https://goshimmer.docs.iota.org/protocol_specification/components.html) approaches.
 
 ## Client-Library and HTTP API reference
 
 You can use the Go client-library to interact with GoShimmer (located under `github.com/iotaledger/goshimmer/client`).
 
-You can find more info about this on our [client-lib](https://goshimmer.docs.iota.org/apis/api.html) and [Web API](https://goshimmer.docs.iota.org/apis/webAPI.html) GitHub page.
+You can find more info about this on our [client-lib](https://goshimmer.docs.iota.org/apis/api.html) and [Web API](https://goshimmer.docs.iota.org/apis/webAPI.html) documentation.
 
 ## Getting started
 
-You can find tutorials on how to [setup a GoShimmer node](https://goshimmer.docs.iota.org/tutorials/setup.html), [writing a dApp](https://goshimmer.docs.iota.org/tutorials/dApp.html), [obtaining tokens from the faucet](https://goshimmer.docs.iota.org/tutorials/request_funds.html) and more on our [GitHub Page](https://goshimmer.docs.iota.org/goshimmer.html).
+You can find tutorials on how to [setup a GoShimmer node](https://goshimmer.docs.iota.org/tutorials/setup.html), [writing a dApp](https://goshimmer.docs.iota.org/tutorials/dApp.html), [obtaining tokens from the faucet](https://goshimmer.docs.iota.org/tutorials/request_funds.html) and more in the [GoShimmer documentation](https://goshimmer.docs.iota.org/goshimmer.html).
 
 ### Compiling from source
 
-We always recommend to run your node via [Docker](https://goshimmer.docs.iota.org/tutorials/setup.html). However, you can also compile the source and run the node from the compiled binary. GoShimmer uses [RocksDB](https://github.com/linxGnu/grocksdb) as its underlying db engine. That requires a few dependencies before building the project: 
+We always recommend running your node via [Docker](https://goshimmer.docs.iota.org/tutorials/setup.html). However, you can also compile the source and run the node from the compiled binary. GoShimmer uses [RocksDB](https://github.com/linxGnu/grocksdb) as its underlying db engine. That requires a few dependencies before building the project: 
 - librocksdb
 - libsnappy
 - libz

diff --git a/client/diagnostics.go b/client/diagnostics.go
@@ -4,8 +4,32 @@ import (
 	"encoding/csv"
 	"fmt"
 	"net/http"
+)
 
-	"github.com/iotaledger/goshimmer/plugins/webapi/tools"
+const (
+	routeDiagnostics = "tools/diagnostic"
+	// RouteDiagnosticMessages is the API route for message diagnostics
+	RouteDiagnosticMessages = routeDiagnostics + "/messages"
+	// RouteDiagnosticsFirstWeakMessageReferences is the API route for first weak message diagnostics
+	RouteDiagnosticsFirstWeakMessageReferences = RouteDiagnosticMessages + "/firstweakreferences"
+	// RouteDiagnosticsMessageRank is the API route for message diagnostics with a rank filter
+	RouteDiagnosticsMessageRank = RouteDiagnosticMessages + "/rank/:rank"
+	// RouteDiagnosticsUtxoDag is the API route for Utxo Dag diagnostics
+	RouteDiagnosticsUtxoDag = routeDiagnostics + "/utxodag"
+	// RouteDiagnosticsBranches is the API route for branches diagnostics
+	RouteDiagnosticsBranches = routeDiagnostics + "/branches"
+	// RouteDiagnosticsLazyBookedBranches is the API route for booked branches diagnostics
+	RouteDiagnosticsLazyBookedBranches = RouteDiagnosticsBranches + "/lazybooked"
+	// RouteDiagnosticsInvalidBranches is the API route for invalid branches diagnostics
+	RouteDiagnosticsInvalidBranches = RouteDiagnosticsBranches + "/invalid"
+	// RouteDiagnosticsTips is the API route for tips diagnostics
+	RouteDiagnosticsTips = routeDiagnostics + "/tips"
+	// RouteDiagnosticsStrongTips is the API route for strong tips diagnostics
+	RouteDiagnosticsStrongTips = RouteDiagnosticsTips + "/strong"
+	// RouteDiagnosticsWeakTips is the API route for weak tips diagnostics
+	RouteDiagnosticsWeakTips = RouteDiagnosticsTips + "/weak"
+	// RouteDiagnosticsDRNG is the API route for DRNG diagnostics
+	RouteDiagnosticsDRNG = routeDiagnostics + "/drng"
 )
 
 // GetDiagnosticsMessages runs full message diagnostics
@@ -17,7 +41,7 @@ import (
 //	PayloadOpinionFormed TimestampOpinionFormed MessageOpinionFormed MessageOpinionTriggered TimestampOpinion
 //	TimestampLoK
 func (api *GoShimmerAPI) GetDiagnosticsMessages() (*csv.Reader, error) {
-	return api.diagnose(tools.RouteDiagnosticMessages)
+	return api.diagnose(RouteDiagnosticMessages)
 }
 
 // GetDiagnosticsFirstWeakMessageReferences runs diagnostics over weak references only.
@@ -29,7 +53,7 @@ func (api *GoShimmerAPI) GetDiagnosticsMessages() (*csv.Reader, error) {
 //	PayloadOpinionFormed TimestampOpinionFormed MessageOpinionFormed MessageOpinionTriggered TimestampOpinion
 //	TimestampLoK
 func (api *GoShimmerAPI) GetDiagnosticsFirstWeakMessageReferences() (*csv.Reader, error) {
-	return api.diagnose(tools.RouteDiagnosticsFirstWeakMessageReferences)
+	return api.diagnose(RouteDiagnosticsFirstWeakMessageReferences)
 }
 
 // GetDiagnosticsMessagesByRank run diagnostics for messages whose markers are equal or above a certain rank
@@ -41,7 +65,7 @@ func (api *GoShimmerAPI) GetDiagnosticsFirstWeakMessageReferences() (*csv.Reader
 //	PayloadOpinionFormed TimestampOpinionFormed MessageOpinionFormed MessageOpinionTriggered TimestampOpinion
 //	TimestampLoK
 func (api *GoShimmerAPI) GetDiagnosticsMessagesByRank(rank uint64) (*csv.Reader, error) {
-	return api.diagnose(fmt.Sprintf("%s?rank=%d", tools.RouteDiagnosticMessages, rank))
+	return api.diagnose(fmt.Sprintf("%s?rank=%d", RouteDiagnosticMessages, rank))
 }
 
 // GetDiagnosticsUtxoDag runs diagnostics over utxo dag.
@@ -51,7 +75,7 @@ func (api *GoShimmerAPI) GetDiagnosticsMessagesByRank(rank uint64) (*csv.Reader,
 //	BranchID,BranchLiked,BranchMonotonicallyLiked,Conflicting,InclusionState,Finalized,LazyBooked,Liked,LoK,FCOB1Time,
 //	FCOB2Time
 func (api *GoShimmerAPI) GetDiagnosticsUtxoDag() (*csv.Reader, error) {
-	return api.diagnose(tools.RouteDiagnosticsUtxoDag)
+	return api.diagnose(RouteDiagnosticsUtxoDag)
 }
 
 // GetDiagnosticsBranches runs diagnostics over branches.
@@ -60,7 +84,7 @@ func (api *GoShimmerAPI) GetDiagnosticsUtxoDag() (*csv.Reader, error) {
 //	ID,ConflictSet,IssuanceTime,SolidTime,OpinionFormedTime,Liked,MonotonicallyLiked,InclusionState,Finalized,
 //	LazyBooked,TransactionLiked
 func (api *GoShimmerAPI) GetDiagnosticsBranches() (*csv.Reader, error) {
-	return api.diagnose(tools.RouteDiagnosticsBranches)
+	return api.diagnose(RouteDiagnosticsBranches)
 }
 
 // GetDiagnosticsLazyBookedBranches runs diagnostics over lazy booked branches.
@@ -69,7 +93,7 @@ func (api *GoShimmerAPI) GetDiagnosticsBranches() (*csv.Reader, error) {
 //	ID,ConflictSet,IssuanceTime,SolidTime,OpinionFormedTime,Liked,MonotonicallyLiked,InclusionState,Finalized,
 //	LazyBooked,TransactionLiked
 func (api *GoShimmerAPI) GetDiagnosticsLazyBookedBranches() (*csv.Reader, error) {
-	return api.diagnose(tools.RouteDiagnosticsLazyBookedBranches)
+	return api.diagnose(RouteDiagnosticsLazyBookedBranches)
 }
 
 // GetDiagnosticsInvalidBranches runs diagnostics over invalid branches.
@@ -78,7 +102,7 @@ func (api *GoShimmerAPI) GetDiagnosticsLazyBookedBranches() (*csv.Reader, error)
 //	ID,ConflictSet,IssuanceTime,SolidTime,OpinionFormedTime,Liked,MonotonicallyLiked,InclusionState,Finalized,
 //	LazyBooked,TransactionLiked
 func (api *GoShimmerAPI) GetDiagnosticsInvalidBranches() (*csv.Reader, error) {
-	return api.diagnose(tools.RouteDiagnosticsInvalidBranches)
+	return api.diagnose(RouteDiagnosticsInvalidBranches)
 }
 
 // GetDiagnosticsTips runs diagnostics over tips
@@ -89,7 +113,7 @@ func (api *GoShimmerAPI) GetDiagnosticsInvalidBranches() (*csv.Reader, error) {
 //	Eligible,Invalid,Finalized,Rank,IsPastMarker,PastMarkers,PMHI,PMLI,FutureMarkers,FMHI,FMLI,PayloadType,TransactionID,
 //	PayloadOpinionFormed,TimestampOpinionFormed,MessageOpinionFormed,MessageOpinionTriggered,TimestampOpinion,TimestampLoK
 func (api *GoShimmerAPI) GetDiagnosticsTips() (*csv.Reader, error) {
-	return api.diagnose(tools.RouteDiagnosticsTips)
+	return api.diagnose(RouteDiagnosticsTips)
 }
 
 // GetDiagnosticsStrongTips runs diagnostics over strong tips
@@ -100,7 +124,7 @@ func (api *GoShimmerAPI) GetDiagnosticsTips() (*csv.Reader, error) {
 //	Eligible,Invalid,Finalized,Rank,IsPastMarker,PastMarkers,PMHI,PMLI,FutureMarkers,FMHI,FMLI,PayloadType,TransactionID,
 //	PayloadOpinionFormed,TimestampOpinionFormed,MessageOpinionFormed,MessageOpinionTriggered,TimestampOpinion,TimestampLoK
 func (api *GoShimmerAPI) GetDiagnosticsStrongTips() (*csv.Reader, error) {
-	return api.diagnose(tools.RouteDiagnosticsStrongTips)
+	return api.diagnose(RouteDiagnosticsStrongTips)
 }
 
 // GetDiagnosticsWeakTips runs diagnostics over weak tips
@@ -111,7 +135,7 @@ func (api *GoShimmerAPI) GetDiagnosticsStrongTips() (*csv.Reader, error) {
 //	Eligible,Invalid,Finalized,Rank,IsPastMarker,PastMarkers,PMHI,PMLI,FutureMarkers,FMHI,FMLI,PayloadType,TransactionID,
 //	PayloadOpinionFormed,TimestampOpinionFormed,MessageOpinionFormed,MessageOpinionTriggered,TimestampOpinion,TimestampLoK
 func (api *GoShimmerAPI) GetDiagnosticsWeakTips() (*csv.Reader, error) {
-	return api.diagnose(tools.RouteDiagnosticsWeakTips)
+	return api.diagnose(RouteDiagnosticsWeakTips)
 }
 
 // GetDiagnosticsDRNG runs diagnostics for DRNG
@@ -120,7 +144,7 @@ func (api *GoShimmerAPI) GetDiagnosticsWeakTips() (*csv.Reader, error) {
 // 	ID,IssuerID,IssuerPublicKey,IssuanceTime,ArrivalTime,SolidTime,ScheduledTime,BookedTime,OpinionFormedTime,
 //	dRNGPayloadType,InstanceID,Round,PreviousSignature,Signature,DistributedPK
 func (api *GoShimmerAPI) GetDiagnosticsDRNG() (*csv.Reader, error) {
-	return api.diagnose(tools.RouteDiagnosticsDRNG)
+	return api.diagnose(RouteDiagnosticsDRNG)
 }
 
 // run an api call on a certain route and return a csv

diff --git a/client/faucet.go b/client/faucet.go
@@ -2,7 +2,6 @@ package client
 
 import (
 	"context"
-	"crypto"
 	"net/http"
 
 	"github.com/cockroachdb/errors"
@@ -22,7 +21,7 @@ const (
 
 var (
 	defaultPOWTarget = 25
-	powWorker        = pow.New(crypto.BLAKE2b_512, 1)
+	powWorker        = pow.New(1)
 )
 
 // SendFaucetRequest requests funds from faucet nodes by sending a faucet request payload message.