Merge pull request #2278 from iotaledger/develop

iotaledger · Jun 16, 2022 · 3eec0b9 · 3eec0b9
2 parents 5a98797 + 7d2cd7a
commit 3eec0b9
Show file tree

Hide file tree

Showing 443 changed files with 15,781 additions and 30,390 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -3,6 +3,7 @@
 .github/
 docs/
 .idea/
+go.work*
 
 LICENSE
 README.md

diff --git a/.github/workflows/client-import.yml b/.github/workflows/client-import.yml
@@ -1,9 +1,6 @@
 name: Client Lib Import Check
 
 on:
-  push:
-    paths-ignore:
-      - 'docs/**'
   pull_request:
     paths-ignore:
       - 'docs/**'
@@ -24,4 +21,4 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Run Import Check Script
-        run: ./scripts/client_import_check.sh
+        run: ./scripts/client_import_check.sh
diff --git a/.github/workflows/feature-network-deploy.yml b/.github/workflows/feature-network-deploy.yml
@@ -76,7 +76,6 @@ jobs:
           GRAFANA_ADMIN_PASSWORD: '${{ secrets.GRAFANA_ADMIN_PASSWORD }}'
           ELK_ELASTIC_PASSWORD: '${{ secrets.ELK_ELASTIC_PASSWORD }}'
           ELK_ELASTIC_USER: '${{ secrets.ELK_ELASTIC_USER }}'
-          DRANDS_SECRET: '${{ secrets.DRANDS_SECRET }}'
           COMMIT_SHA: '${{ github.sha }}'
           GOSHIMMER_DOCKER_IMAGE: 'iotaledgertesting/goshimmer'
           GOSHIMMER_DOCKER_TAG: 'feature'

diff --git a/.github/workflows/goreleaser-dryrun.yml b/.github/workflows/goreleaser-dryrun.yml
@@ -0,0 +1,26 @@
+name: GoReleaser dryrun
+
+on:
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+
+jobs:
+
+  goreleaser:
+    name: GoReleaser dryrun
+    runs-on:
+      - ubuntu-latest
+    container:
+      image: gohornet/goreleaser-cgo-cross-compiler:1.18.1
+      volumes: [/repo]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Create fake config.json and snapshot.bin
+        run: touch config.json snapshot.bin
+
+      - name: Run GoReleaser
+        run: goreleaser --snapshot --rm-dist
+        shell: bash
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -156,55 +156,6 @@ jobs:
           name: ${{ env.TEST_NAME }}
           path: tools/integration-tests/logs
 
-  drng:
-    name: drng
-    env:
-      TEST_NAME: drng
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Cache Go modules for tester container
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/go
-          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
-          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
-            ${{ runner.os }}-it-go-
-
-      - name: Build GoShimmer image
-        run: |
-          docker build \
-            --build-arg DOWNLOAD_SNAPSHOT=0 \
-            -t iotaledger/goshimmer .
-
-      - name: Pull additional Docker images
-        run: |
-          docker pull angelocapossele/drand:v1.1.4
-
-      - name: Run integration tests
-        run: |
-          export CURRENT_UID=$(id -u):$(id -g)
-          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
-
-      - name: Create logs from tester
-        if: always()
-        run: |
-          docker logs tester &> tools/integration-tests/logs/tester.log
-
-      - name: Save logs as artifacts
-        if: always()
-        uses: actions/upload-artifact@v1
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: tools/integration-tests/logs
-
-
   value:
     name: value
     env:
@@ -349,51 +300,3 @@ jobs:
         with:
           name: ${{ env.TEST_NAME }}
           path: tools/integration-tests/logs
-
-  diagnostics:
-    name: diagnostics
-    env:
-      TEST_NAME: diagnostics
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Cache Go modules for tester container
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/go
-          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
-          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
-            ${{ runner.os }}-it-go-
-
-      - name: Build GoShimmer image
-        run: |
-          docker build \
-            --build-arg DOWNLOAD_SNAPSHOT=0 \
-            -t iotaledger/goshimmer .
-
-      - name: Pull additional Docker images
-        run: |
-          docker pull alpine/socat:1.7.4.3-r0
-
-      - name: Run integration tests
-        run: |
-          export CURRENT_UID=$(id -u):$(id -g)
-          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
-
-      - name: Create logs from tester
-        if: always()
-        run: |
-          docker logs tester &> tools/integration-tests/logs/tester.log
-
-      - name: Save logs as artifacts
-        if: always()
-        uses: actions/upload-artifact@v1
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: tools/integration-tests/logs
diff --git a/.github/workflows/pre-devnet-deploy.yml b/.github/workflows/pre-devnet-deploy.yml
@@ -71,7 +71,6 @@ jobs:
           GRAFANA_ADMIN_PASSWORD: '${{ secrets.GRAFANA_ADMIN_PASSWORD }}'
           ELK_ELASTIC_PASSWORD: '${{ secrets.ELK_ELASTIC_PASSWORD }}'
           ELK_ELASTIC_USER: '${{ secrets.ELK_ELASTIC_USER }}'
-          DRANDS_SECRET: '${{ secrets.DRANDS_SECRET }}'
           COMMIT_SHA: '${{ github.sha }}'
           GOSHIMMER_DOCKER_IMAGE: 'iotaledger/goshimmer'
           GOSHIMMER_DOCKER_TAG: 'develop'

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -38,10 +38,6 @@ jobs:
         uses: actions/checkout@v2
         with:
           ref: "refs/tags/${{github.event.inputs.tagName}}"
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.18
 
       - name: Copy config.default.json to config.json
         run: cp config.default.json config.json
@@ -98,8 +94,6 @@ jobs:
           GRAFANA_ADMIN_PASSWORD: '${{ secrets.GRAFANA_ADMIN_PASSWORD }}'
           ELK_ELASTIC_PASSWORD: '${{ secrets.ELK_ELASTIC_PASSWORD }}'
           ELK_ELASTIC_USER: '${{ secrets.ELK_ELASTIC_USER }}'
-          DRANDS_SECRET: '${{ secrets.DRANDS_SECRET }}'
-          DRNG_XTEAM_01_SEED: '${{ secrets.DRNG_XTEAM_01_SEED }}'
           SNAPSHOTTER_BUCKET: '${{ secrets.SNAPSHOTTER_BUCKET }}'
           SNAPSHOTTER_ACCESS_KEY: '${{ secrets.SNAPSHOTTER_ACCESS_KEY }}'
           SNAPSHOTTER_SECRET_KEY: '${{ secrets.SNAPSHOTTER_SECRET_KEY }}'

diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/reviewdog.yml
@@ -10,17 +10,14 @@ jobs:
     steps:
 
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0 # required for new-from-rev option in .golangci.yml
 
-      - uses: actions/setup-go@v1
-        with:
-          go-version: "1.18"
-
       - name: Run golangci-lint
-        uses: docker://reviewdog/action-golangci-lint:latest
+        uses: reviewdog/action-golangci-lint@v2
         with:
           github_token: ${{ secrets.github_token }}
           golangci_lint_flags: "--timeout=10m"
-          reporter: "github-pr-review"
+          reporter: "github-pr-check"
+          fail_on_error: true
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
@@ -0,0 +1,71 @@
+name: Test GoShimmer
+
+on:
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+
+jobs:
+
+  unit-tests:
+    name: Unit tests
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Setup dependencies
+        run: sudo apt-get install libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
+
+      - name: Setup Go 1.18
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.18
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Cache Go build and dependencies
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}-${{ github.event.action }}${{ github.event.after }}
+          restore-keys: |
+            ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
+            ${{ runner.os }}-go-
+
+      - name: Run Tests
+        run: go test ./... -tags rocksdb -count=1 -timeout 10m
+
+
+  unit-tests-race:
+    name: Unit tests -race
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Setup dependencies
+        run: sudo apt-get install libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
+
+      - name: Setup Go 1.18
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.18
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Cache Go build and dependencies
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}-${{ github.event.action }}${{ github.event.after }}
+          restore-keys: |
+            ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
+            ${{ runner.os }}-go-
+
+      - name: Run Tests with -race
+        run: go test ./... -tags rocksdb -count=1 -race -short -timeout 20m
diff --git a/.gitignore b/.gitignore
@@ -28,6 +28,7 @@ testNodes/*
 # Project files
 .idea
 go.work
+go.work.sum
 
 # Database directory
 mainnetdb/

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,25 @@
+# v0.9.0 - 2022-06-16
+
+> This major release introduces concurrency throughout the node with asynchronous events and parallelized booking. This is achieved with serix, a generic deterministic serialization library, and through generic thread-safe data models. The ledger and conflictDAG and output processing (introduction of UTXO VM) have been completely rewritten. The Blockfactory received a total revamp as well and tip selection should show better behavior in resolving conflicts. In addition, a bunch of unused modules and code have been removed rendering the project overall more readable. The short identifier of a nodeID has been adjusted to show the first 8 characters of the full identity.
+
+The snapshot has been changed and thus the ledger state is fully reset.
+- Refactor MessageFactory, introduce ReferenceProvider and OrphanageManager (#2240)
+- Refactor models to be thread-safe with serix (#2248)
+- Remove future markers (#2241)
+- Delete unused modules (#2234)
+- Change identity.ID short representation (#2238)
+- Build(deps): bump eventsource in /plugins/dagsvisualizer/frontend (#2235)
+- Refactor models to use generic base models (#2231)
+- Build(deps): bump eventsource in /plugins/analysis/dashboard/frontend (#2233)
+- Build(deps): bump eventsource in /plugins/dashboard/frontend (#2232)
+- Multithreaded booking and new data flow with solidification of transactions (#2151)
+- Reviewdog version bump (#2223)
+- Fix serializer package dependency (#2219)
+- Force static linking of final binary in builtin script (#2216)
+- GoReleaser dryrun on PRs (#2213)
+- Update marker docs (#2146)
+
+
 # v0.8.14 - 2022-05-16
 
 > Minor revision introducing small docker changes.

diff --git a/Dockerfile b/Dockerfile
@@ -135,7 +135,7 @@ FROM prepare-runtime as debugger-enabled-0
 
 ENTRYPOINT ["/app/goshimmer", "--config=/app/config.json"]
 
-# We execute this stage only if debugging is enabled, i.e REMOTE_DEBUGGIN==1
+# We execute this stage only if debugging is enabled, i.e REMOTE_DEBUGGING==1
 FROM prepare-runtime as debugger-enabled-1
 EXPOSE 40000
-Original file line number
+Diff line change
@@ Expand Up / @@ -3,6 +3,7 @@ @@
     .github/
     docs/
     .idea/
+    go.work*
     LICENSE
     README.md
@@ Expand Down @@