Merge pull request #1612 from iotaledger/develop

Merge v0.7.5 changes to master

.github/workflows/deploy_docs_to_gh-pages.yml

@@ -0,0 +1,53 @@
+name: deploy
+
+on:
+  pull_request:
+    branches: [develop, dev, master,  main]
+  push:
+    branches: [develop, dev, master, main]
+
+jobs:
+  checks:
+    if: github.event_name != 'push'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '12.x'
+      - name: Test Build
+        run: |
+          cd documentation
+          if [ -e yarn.lock ]; then
+          yarn install --frozen-lockfile
+          elif [ -e package-lock.json ]; then
+          npm ci
+          else
+          npm i
+          fi
+          npm run build
+  gh-release:
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '12.x'
+      - name: Build
+        run: |
+          cd documentation
+          if [ -e yarn.lock ]; then
+          yarn install --frozen-lockfile
+          elif [ -e package-lock.json ]; then
+          npm ci
+          else
+          npm i
+          fi
+          npm run build
+      - name: Release to GitHub Pages
+        uses: iotaledger/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./documentation/build
+          cname: goshimmer.docs.iota.org

.github/workflows/docker-testing.yml

@@ -5,6 +5,7 @@ on: workflow_dispatch
 jobs:
 
   docker:
+    environment: pre-devnet
     runs-on: ubuntu-latest
     env:
       DOCKER_BUILDKIT: 1

.github/workflows/docker-develop.yml → .github/workflows/pre-devnet-deploy.yml

@@ -1,15 +1,15 @@
-name: Develop Docker Image
+name: Pre-devnet deploy
 
 on:
   push:
     branches:
       - develop
     paths-ignore:
       - 'docs/**'
-
 jobs:
 
-  docker:
+  deploy:
+    environment: pre-devnet
     runs-on: ubuntu-latest
     env:
       DOCKER_BUILDKIT: 1
@@ -49,3 +49,30 @@ jobs:
         run: |
           rm -rf /tmp/.buildx-cache
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+      - name: Install SSH Key
+        env:
+          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+        run: |
+          mkdir -p ~/.ssh/
+          echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_ed25519
+          sudo chmod 600 ~/.ssh/id_ed25519
+          echo '
+               Host *
+                 StrictHostKeyChecking no
+                 UserKnownHostsFile=/dev/null
+               ' > ~/.ssh/config
+
+      - name: Ansible deploy
+        env:
+          GOSHIMMER_SEEDS: '${{ secrets.GOSHIMMER_SEEDS }}'
+          MONGODB_USER: '${{ secrets.MONGODB_USER }}'
+          MONGODB_PASSWORD: '${{ secrets.MONGODB_PASSWORD }}'
+          GRAFANA_ADMIN_PASSWORD: '${{ secrets.GRAFANA_ADMIN_PASSWORD }}'
+          ELK_ELASTIC_PASSWORD: '${{ secrets.ELK_ELASTIC_PASSWORD }}'
+          ELK_ELASTIC_USER: '${{ secrets.ELK_ELASTIC_USER }}'
+          DRANDS_SECRET: '${{ secrets.DRANDS_SECRET }}'
+          COMMIT_SHA: '${{ github.sha }}'
+        run: |
+          export NETWORK_VERSION=$((16#$(git rev-parse --short "$COMMIT_SHA")%1000000000+1000))
+          ./deploy/ansible/run.sh

.github/workflows/release.yml

@@ -5,6 +5,7 @@ name: Release
       - published
 jobs:
   goreleaser:
+    environment: devnet
     name: GoReleaser
     runs-on:
       - ubuntu-latest
@@ -27,6 +28,7 @@ jobs:
         env:
           GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
   docker:
+    environment: devnet
     name: Docker
     runs-on:
       - ubuntu-latest

CHANGELOG.md

@@ -1,3 +1,16 @@
+# v0.7.5 - 2021-08-09
+* Move the scheduler of the congestion control at the end of data flow
+* Add new metrics collection
+* Add alias initial state validation
+* Add creation of new marker sequence after maxVerticesWithoutFutureMarker threshold
+* Fix booking of transactions in multiple conflict sets
+* Fix manual peering parsing from config.json
+* Update JS dependencies
+* Update Dockerfile to explicitly expose used ports
+* Update docs to docusaurus
+* Update snapshot file with DevNet UTXO at 2021-08-09 13:33 UTC
+* **Breaking**: bumps network and database versions
+
 # v0.7.4 - 2021-07-08
 * Add ParametersDefinition structs in integration test framework
 * Add UTXO-DAG interface

Dockerfile

@@ -4,7 +4,7 @@
 # golang 1.16.3-buster amd64
 FROM golang@sha256:dfa3cef088454200d6b48e2a911138f7d5d9afff77f89243eea6342f16ddcfb0 AS build
 
-ARG BUILD_TAGS=rocksdb,builtin_static,netgo
+ARG BUILD_TAGS=rocksdb,builtin_static
 
 # Ensure ca-certficates are up to date
 RUN update-ca-certificates
@@ -29,9 +29,8 @@ RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
     GOOS=linux GOARCH=amd64 go build \
     -tags="$BUILD_TAGS" \
-    -ldflags='-w -s -extldflags "-static"' \
-    -o /go/bin/goshimmer; \
-    ./check_static.sh
+    -ldflags='-w -s' \
+    -o /go/bin/goshimmer
 
 RUN wget -O /tmp/snapshot.bin https://dbfiles-goshimmer.s3.eu-central-1.amazonaws.com/snapshots/nectar/snapshot-latest.bin
 
@@ -42,8 +41,20 @@ RUN wget -O /tmp/snapshot.bin https://dbfiles-goshimmer.s3.eu-central-1.amazonaw
 # using distroless cc image, which includes everything in the base image (glibc, libssl and openssl)
 FROM gcr.io/distroless/cc@sha256:4cad7484b00d98ecb300916b1ab71d6c71babd6860c6c5dd6313be41a8c55adb
 
+# Gossip
 EXPOSE 14666/tcp
+# Autopeering
 EXPOSE 14626/udp
+# FPC
+EXPOSE 10895/tcp
+# Pprof Profiling
+EXPOSE 6061/tcp
+# Prometheus exporter
+EXPOSE 9311/tcp
+# Webapi
+EXPOSE 8080/tcp
+# Dashboard
+EXPOSE 8081/tcp
 
 # Copy configuration
 COPY --from=build /tmp/snapshot.bin /snapshot.bin

README.md

@@ -1,6 +1,6 @@
 <h1 align="center">
   <br>
-  <a href="https://goshimmer.docs.iota.org/goshimmer.html"><img src="images/GoShimmer.png"></a>
+  <a href="https://goshimmer.docs.iota.org/"><img src="images/GoShimmer.png"></a>
 </h1>
 
 <h2 align="center">Prototype node software for an IOTA network without the Coordinator</h2>
@@ -34,28 +34,28 @@ This repository, called GoShimmer, is where the IOTA Foundation's Research Depar
 
 GoShimmer is first and foremost a research prototype. As such, breaking changes can often happen. We invite researchers and developers to make use of this project as you see fit. Running experiments, test out new ideas, build PoC are all very welcome initiatives.
 
-For a documentation, including tutorials and resources, we refer to the [Documentation](http://goshimmer.docs.iota.org/) (Github [link](https://github.com/iotaledger/goshimmer/blob/develop/docs/SUMMARY.md)).
+For a documentation, including tutorials and resources, we refer to the [Documentation](http://goshimmer.docs.iota.org/) (Github [link](https://github.com/iotaledger/goshimmer/blob/develop/documentation/README.md)).
 
 ## Design
 The code in GoShimmer is modular, where each module represents either one of the *IOTA 2.0 components*, or a basic node function such as the gossip, ledger state, and API - just to mention a few.  
 
-![Layers](docs/protocol_specification/layers.png)
+![Layers](docOps/static/img/protocol_specification/layers.png)
 
-GoShimmer's modularity is based on a combination of [event-driven](https://goshimmer.docs.iota.org/implementation_design/event_driven_model.html) and [layer-based](https://goshimmer.docs.iota.org/protocol_specification/components.html) approaches.
+GoShimmer's modularity is based on a combination of [event-driven](https://goshimmer.docs.iota.org/docs/implementation_design/event_driven_model) and [layer-based](https://goshimmer.docs.iota.org/docs/protocol_specification/components) approaches.
 
 ## Client-Library and HTTP API reference
 
 You can use the Go client-library to interact with GoShimmer (located under `github.com/iotaledger/goshimmer/client`).
 
-You can find more info about this on our [client-lib](https://goshimmer.docs.iota.org/apis/api.html) and [Web API](https://goshimmer.docs.iota.org/apis/webAPI.html) documentation.
+You can find more info about this on our [client-lib](https://goshimmer.docs.iota.org/docs/apis/api) and [Web API](https://goshimmer.docs.iota.org/docs/apis/webAPI) documentation.
 
 ## Getting started
 
-You can find tutorials on how to [setup a GoShimmer node](https://goshimmer.docs.iota.org/tutorials/setup.html), [writing a dApp](https://goshimmer.docs.iota.org/tutorials/dApp.html), [obtaining tokens from the faucet](https://goshimmer.docs.iota.org/tutorials/request_funds.html) and more in the [GoShimmer documentation](https://goshimmer.docs.iota.org/goshimmer.html).
+You can find tutorials on how to [setup a GoShimmer node](https://goshimmer.docs.iota.org/docs/tutorials/setup), [writing a dApp](https://goshimmer.docs.iota.org/docs/tutorials/dApp), [obtaining tokens from the faucet](https://goshimmer.docs.iota.org/docs/tutorials/request_funds) and more in the [GoShimmer documentation](https://goshimmer.docs.iota.org/).
 
 ### Compiling from source
 
-We always recommend running your node via [Docker](https://goshimmer.docs.iota.org/tutorials/setup.html). However, you can also compile the source and run the node from the compiled binary. GoShimmer uses [RocksDB](https://github.com/linxGnu/grocksdb) as its underlying db engine. That requires a few dependencies before building the project: 
+We always recommend running your node via [Docker](https://goshimmer.docs.iota.org/docs/tutorials/setup). However, you can also compile the source and run the node from the compiled binary. GoShimmer uses [RocksDB](https://github.com/linxGnu/grocksdb) as its underlying db engine. That requires a few dependencies before building the project: 
 - librocksdb
 - libsnappy
 - libz

client/diagnostics.go

@@ -36,7 +36,7 @@ const (
 // Returns CSV with the following fields:
 //
 //	ID IssuerID IssuerPublicKey IssuanceTime ArrivalTime SolidTime ScheduledTime BookedTime OpinionFormedTime
-//	FinalizedTime StrongParents WeakParents StrongApprovers WeakApprovers BranchID InclusionState Scheduled Booked
+//	FinalizedTime StrongParents WeakParents StrongApprovers WeakApprovers BranchID InclusionState Scheduled ScheduledBypass Booked
 //	Eligible Invalid Finalized Rank IsPastMarker PastMarkers PMHI PMLI FutureMarkers FMHI FMLI PayloadType TransactionID
 //	PayloadOpinionFormed TimestampOpinionFormed MessageOpinionFormed MessageOpinionTriggered TimestampOpinion
 //	TimestampLoK

deploy/ansible/deploy.yml

@@ -0,0 +1,21 @@
+---
+## This playbook deploys new changes to the environment.
+
+- hosts: metrics
+  vars:
+    removeData: no
+  roles:
+    - metrics
+
+- hosts: supports
+  roles:
+    - role: goshimmer-node
+      nodeType: entrynode
+    - role: goshimmer-node
+      nodeType: analysis-server
+
+- hosts: goshimmers
+  vars:
+    nodeType: goshimmer
+  roles:
+    - goshimmer-node

deploy/ansible/hosts.yml

@@ -0,0 +1,34 @@
+metrics:
+  hosts:
+    metrics-01.pre-devnet.shimmer.iota.cafe:
+
+supports:
+  hosts:
+    analysisentry-01.pre-devnet.shimmer.iota.cafe:
+      entryNodeSeed: "{{ ANALYSISSENTRY_01_ENTRYNODE_SEED }}"
+
+goshimmers:
+  hosts:
+    bootstrap-01.pre-devnet.shimmer.iota.cafe:
+      seed: "{{ BOOTSTRAP_01_SEED }}"
+      bootstrap: true
+    vanilla-01.pre-devnet.shimmer.iota.cafe:
+      seed: "{{ VANILLA_01_SEED }}"
+    drng-01.pre-devnet.shimmer.iota.cafe:
+      seed: "{{ DRNG_01_SEED }}"
+    drng-02.pre-devnet.shimmer.iota.cafe:
+      seed: "{{ DRNG_02_SEED }}"
+    drng-03.pre-devnet.shimmer.iota.cafe:
+      seed: "{{ DRNG_03_SEED }}"
+    drng-04.pre-devnet.shimmer.iota.cafe:
+      seed: "{{ DRNG_04_SEED }}"
+    drng-05.pre-devnet.shimmer.iota.cafe:
+      seed: "{{ DRNG_05_SEED }}"
+    faucet-01.pre-devnet.shimmer.iota.cafe:
+      seed: "{{ FAUCET_01_SEED }}"
+      faucet: true
+      faucetSeed: "{{ FAUCET_01_FAUCET_SEED }}"
+
+drands:
+  hosts:
+    drand-01.pre-devnet.shimmer.iota.cafe: