Merge branch 'master' into ci/release

.github/workflows/applications.yml

@@ -338,7 +338,7 @@ jobs:
     - name: Build chainweb library
       run: cabal build --ghc-options=-j2 lib:chainweb
     - name: Build chainweb applications
-      run: cabal build -j2 --ghc-options=-j2 exe:chainweb-node test:chainweb-tests exe:cwtool chainweb:bench:bench
+      run: cabal build -j2 --ghc-options=-j2 exe:chainweb-node exe:compact test:chainweb-tests exe:cwtool chainweb:bench:bench
 
     # Checks
     - name: Check that working directory tree is clean
@@ -360,6 +360,7 @@ jobs:
       run: |
         mkdir -p artifacts/chainweb
         cp $(cabal list-bin chainweb-node) artifacts/chainweb
+        cp $(cabal list-bin compact) artifacts/chainweb
         cp $(cabal list-bin chainweb-tests) artifacts/chainweb
         cp $(cabal list-bin cwtool) artifacts/chainweb
         cp $(cabal list-bin bench) artifacts/chainweb
@@ -617,6 +618,8 @@ jobs:
     name: Build and publish docker image
     needs: [config, build]
     runs-on: ${{ matrix.os }}
+    outputs:
+      docker-suffix: ${{ steps.frozen-tag.outputs.value }}${{ needs.config.outputs.tag-suffix }}
     if: "${{ github.event_name != 'schedule' }}"
     strategy:
       fail-fast: false
@@ -631,7 +634,7 @@ jobs:
       OS: ${{ matrix.os }}
       ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}
       ARTIFACTS_ARCHIVE: chainweb.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}.${{ needs.config.outputs.git-sha-short }}${{ needs.config.outputs.tag-suffix }}.tar.gz
-
+  
     steps:
     - name: Get build artifacts
       uses: actions/download-artifact@v4
@@ -738,18 +741,21 @@ jobs:
     name: Run Integration Tests from remote repo
     needs: [ config, docker-image ]
     runs-on: ubuntu-latest
-
-    env:
-      GIT_SHA_SHORT: ${{ needs.config.outputs.git-sha-short }}
-
     steps:
+    - name: Get auth token
+      uses: Nastaliss/get-github-app-pat@683b0b370911354c411dbbc586b7c64f32b2b850
+      id: githubAppAuth
+      with:
+        app-id: ${{ secrets.GH_KADENAINFRA_APP_ID }}
+        app-installation-id: ${{ secrets.GH_KADENAINFRA_APP_INSTALLATION_ID }}
+        app-private-key: ${{ secrets.GH_KADENAINFRA_APP_PRIVATE_KEY }}
     - name: Start remote integration tests
-      uses: aurelien-baudet/workflow-dispatch@93e95b157d791ae7f42aef8f8a0d3d723eba1c31
+      uses: the-actions-org/workflow-dispatch@v4.0.0
       with:
         workflow: Integration Tests on devnet chain
-        token: ${{ secrets.GH_WORKFLOW_PAT_FOR_TESTS }}
-        inputs: '{ "chainweb_node_container_id" : "sha-${{ needs.config.outputs.git-sha-short }}" }'
+        token: ${{ steps.githubAppAuth.outputs.access-token }}
+        inputs: '{ "chainweb_node_container_id" : "sha-${{ needs.config.outputs.git-sha-short }}${{ needs.docker-image.outputs.docker-suffix }}" }'
         ref: refs/heads/master
         repo: kadena-io/integration-tests
         display-workflow-run-url: true
-        wait-for-completion: false # here you could make this pipeline wait them out
+        wait-for-completion: true # here you could make this pipeline wait them out

CHANGELOG.md

@@ -1,3 +1,21 @@
+## 2.25 (2024-08-21)
+This is a major version update. This release replaces all previous versions.
+
+Any prior version will stop working on **2022-08-21T00:00:00Z**. Node administrators must
+upgrade to this version before that date. The 2.25 feature upgrade will
+occur at block height 5060924 which is estimated to be mined at **2022-08-21T00:00:00Z**.
+
+### Changes
+- Database compaction: Significant performance improvements [`da410fc7`](https://github.com/kadena-io/chainweb-node/commit/da410fc7db2df261d8ef808380d402876ccf79f5)
+- Verifier Plugin support for Hyperlane merkle proofs. This brings Chainweb into compliance with the latest version of the Hyperlane protocol. [`bc87c68b`](https://github.com/kadena-io/chainweb-node/commit/bc87c68bf0fc4ba427ff2cbf2858933b7470543a)
+- Log current cut periodically, instead of when it changes, for more consistency and less space use. [`f5a0cf15`](https://github.com/kadena-io/chainweb-node/commit/f5a0cf157139497c902e12caa65e68bdf716a8c7)
+- Better progress messages for read-only replay, including a time estimate and smoothed rate calculation. [`bc88b4c1`](https://github.com/kadena-io/chainweb-node/commit/bc88b4c179c2cf5c8931f1817caa36bf69731887)
+- Speed up read-only replay by avoiding playing empty blocks [`9f22b079`](https://github.com/kadena-io/chainweb-node/commit/9f22b07953621de709c0320176cef985f56aaa7d)
+- Fix a performance bug in read-only replay which was not using a cache for module data [`be7497d0`](https://github.com/kadena-io/chainweb-node/commit/be7497d00458a51bbec525764927cb444da1649c)
+- Small fixes to exception safety result in Ctrl-C now working properly during read-only replay (and other scenarios) [`f258a705`](https://github.com/kadena-io/chainweb-node/commit/f258a705e77ce28c08cf934980c7d12cb4f5be30)
+- Stop exporting constructor for BlockHeader from Chainweb.Blockheader [`efc1a8ad`](https://github.com/kadena-io/chainweb-node/commit/efc1a8ad0bec715c723a7d176bce00089f75ef79)
+- Stop exporting constructor for PayloadData from Chainweb.Payload [`efd12432`](https://github.com/kadena-io/chainweb-node/commit/efd1243213d59d7841b5c024d7d8e21edb0e0808)
+
 ## 2.24.1
 This is a minor point release. Upgrading is **strongly recommended**.
 

bench/Chainweb/Pact/Backend/Bench.hs

@@ -14,9 +14,11 @@ module Chainweb.Pact.Backend.Bench
 
 
 import Control.Concurrent
+import Control.Lens (view, (.~))
 import Control.Monad
 import Control.Monad.Catch
 import qualified Criterion.Main as C
+import Data.Function ((&))
 
 import qualified Data.Vector as V
 import qualified Data.ByteString as B
@@ -46,7 +48,7 @@ import qualified Pact.Types.SQLite as PSQL
 -- chainweb imports
 
 import Chainweb.BlockHash
-import Chainweb.BlockHeader
+import Chainweb.BlockHeader.Internal
 import Chainweb.Graph
 import Chainweb.Logger
 import Chainweb.MerkleLogHash
@@ -78,11 +80,13 @@ cpRestoreAndSave cp pc blks = snd <$> _cpRestoreAndSave cp (ParentHeader <$> pc)
 
 -- | fabricate a `BlockHeader` for a block given its hash and its parent.
 childOf :: Maybe BlockHeader -> BlockHash -> BlockHeader
-childOf (Just bh) bhsh =
-  bh { _blockHash = bhsh, _blockParent = _blockHash bh, _blockHeight = _blockHeight bh + 1 }
-childOf Nothing bhsh =
-  (genesisBlockHeader testVer testChainId) { _blockHash = bhsh }
-
+childOf m bhsh = case m of
+  Just bh -> bh
+    & blockHash .~ bhsh
+    & blockParent .~ view blockHash bh
+    & blockHeight .~ view blockHeight bh + 1
+  Nothing -> genesisBlockHeader testVer testChainId
+    & blockHash .~ bhsh
 
 bench :: C.Benchmark
 bench = C.bgroup "pact-backend" $

bench/Chainweb/Pact/Backend/ForkingBench.hs

@@ -207,7 +207,7 @@ playLine pdb bhdb trunkLength startingBlock pactQueue counter = do
         evalStateT (runReaderT (mapM (const go) [startHeight :: Word64 .. pred (startHeight + l)]) pactQueue) start
       where
         startHeight :: Num a => a
-        startHeight = fromIntegral $ _blockHeight start
+        startHeight = fromIntegral $ view blockHeight start
         go = do
             r <- ask
             pblock <- gets ParentHeader
@@ -226,7 +226,7 @@ mineBlock
     -> IO (T3 ParentHeader BlockHeader PayloadWithOutputs)
 mineBlock parent nonce pdb bhdb pact = do
     r@(T3 _ newHeader payload) <- createBlock DoValidate parent nonce pact
-    addNewPayload pdb (succ (_blockHeight (_parentHeader parent))) payload
+    addNewPayload pdb (succ (view blockHeight (_parentHeader parent))) payload
     -- NOTE: this doesn't validate the block header, which is fine in this test case
     unsafeInsertBlockHeaderDb bhdb newHeader
     return r
@@ -244,7 +244,7 @@ createBlock validate parent nonce pact = do
      bip <- throwIfNoHistory =<< newBlock noMiner NewBlockFill parent pact
      let payload = blockInProgressToPayloadWithOutputs bip
 
-     let creationTime = add second $ _blockCreationTime $ _parentHeader parent
+     let creationTime = add second $ view blockCreationTime $ _parentHeader parent
      let bh = newBlockHeader
               mempty
               (_payloadWithOutputsPayloadHash payload)
@@ -301,15 +301,32 @@ withResources rdb trunkLength logLevel compact p f = C.envWithCleanup create des
         coinAccounts <- newMVar mempty
         nonceCounter <- newIORef 1
         txPerBlock <- newIORef 10
-        sqlEnv <- openSQLiteConnection "" {- temporary SQLite db -} chainwebBenchPragmas
         mp <- testMemPoolAccess txPerBlock coinAccounts
-        pactService <-
-          startPact testVer logger blockHeaderDb payloadDb mp sqlEnv
-        mainTrunkBlocks <-
-          playLine payloadDb blockHeaderDb trunkLength genesisBlock (snd pactService) nonceCounter
-        when (compact == DoCompact) $ do
-          C.withDefaultLogger Error $ \lgr -> do
-            void $ C.compact (BlockHeight trunkLength) lgr sqlEnv []
+        (sqlEnv, pactService, mainTrunkBlocks) <- do
+          srcSqlEnv <- openSQLiteConnection "" {- temporary SQLite db -} chainwebBenchPragmas
+          srcPactService <-
+            startPact testVer logger blockHeaderDb payloadDb mp srcSqlEnv
+          mainTrunkBlocks <-
+            playLine payloadDb blockHeaderDb trunkLength genesisBlock (snd srcPactService) nonceCounter
+
+          (sqlEnv, pactService) <- do
+            if compact == DoCompact
+            then do
+              targetSqlEnv <- openSQLiteConnection "" {- temporary SQLite db -} chainwebBenchPragmas
+              C.withDefaultLogger Error $ \lgr -> do
+                C.compactPactState lgr C.defaultRetainment (BlockHeight trunkLength) srcSqlEnv targetSqlEnv
+              targetPactService <-
+                startPact testVer logger blockHeaderDb payloadDb mp targetSqlEnv
+
+              -- Stop the previous pact service/close the sqlite connection
+              stopPact srcPactService
+              stopSqliteDb srcSqlEnv
+
+              pure (targetSqlEnv, targetPactService)
+            else do
+              pure (srcSqlEnv, srcPactService)
+
+          pure (sqlEnv, pactService, mainTrunkBlocks)
 
         return $ NoopNFData $ Resources {..}
 
@@ -368,7 +385,7 @@ testMemPoolAccess txsPerBlock accounts = do
   return $ mempty
     { mpaGetBlock = \bf validate bh hash header -> do
         if _bfCount bf /= 0 then pure mempty else do
-          testBlock <- getTestBlock accounts (_bct $ _blockCreationTime header) validate bh hash
+          testBlock <- getTestBlock accounts (_bct $ view blockCreationTime header) validate bh hash
           pure testBlock
     }
   where

cabal.project

@@ -87,8 +87,8 @@ source-repository-package
 source-repository-package
     type: git
     location: https://github.com/kadena-io/chainweb-storage.git
-    tag: 4b45c1ab9c070c6d16a058bcbab0c06ac0fb6d4e
-    --sha256: 0m6c7kl6x5a3k02q2i7qzfx91kxz19dzav0piqfxra52bq0x3sm6
+    tag: a5e06a8aa2da1e7981ff9fad91dfd41f7c39fc85
+    --sha256: sha256-3Zqsgrxa7SQUr8XzT2O5PLTELkI92NXZU4j2UUvlL4E=
 
 source-repository-package
     type: git
@@ -137,18 +137,6 @@ source-repository-package
     tag: 174af3523616c8fe01449da5ccbb9f16df097ac3
     --sha256: sha256-kVFIy+Aj3TNJpsM1Cs/5uGmzeWwHKYWjjCQ+L1/XOj8=
 
-source-repository-package
-    type: git
-    location: https://github.com/chessai/patience
-    tag: 2f67d546ea6608fc6ebe5f2f6976503cbf340442
-    --sha256: 0x137akvbh4kr3qagksw74xdj2xz5vjnx1fbr41bb54a0lkcb8mm
-
-source-repository-package
-    type: git
-    location: https://github.com/andrewthad/chronos
-    tag: b199bf6df1453af95832c2d2f9f0ef48c3622caa
-    --sha256: 056awkmdmkqdd5g3m8a1ibg2vp02kbppmidkfh4aildb1brq970a
-
 -- -------------------------------------------------------------------------- --
 -- Relaxed Bounds
 

cabal.project.freeze

@@ -66,7 +66,7 @@ constraints: any.Boolean ==0.2.4,
              any.chainweb-storage ==0.1.0.0,
              any.character-ps ==0.1,
              any.charset ==0.3.10,
-             any.chronos ==1.1.5.1,
+             any.chronos ==1.1.6.1,
              any.clock ==0.8.4,
              any.code-page ==0.2.1,
              any.colour ==2.3.6,
@@ -340,4 +340,4 @@ constraints: any.Boolean ==0.2.4,
              any.yet-another-logger ==0.4.2,
              any.zigzag ==0.1.0.0,
              any.zlib ==0.7.1.0
-index-state: hackage.haskell.org 2024-06-17T23:25:46Z
+index-state: hackage.haskell.org 2024-07-29T14:54:26Z

chainweb.cabal

@@ -1,7 +1,7 @@
 cabal-version: 3.8
 
 name:         chainweb
-version:      2.24.1
+version:      2.25
 synopsis:     A Proof-of-Work Parallel-Chain Architecture for Massive Throughput
 description:  A Proof-of-Work Parallel-Chain Architecture for Massive Throughput.
 homepage:     https://github.com/kadena-io/chainweb
@@ -136,6 +136,7 @@ library
         , Chainweb.BlockCreationTime
         , Chainweb.BlockHash
         , Chainweb.BlockHeader
+        , Chainweb.BlockHeader.Internal
         , Chainweb.BlockHeader.Genesis.RecapDevelopment0Payload
         , Chainweb.BlockHeader.Genesis.RecapDevelopment1to9Payload
         , Chainweb.BlockHeader.Genesis.RecapDevelopment10to19Payload
@@ -372,12 +373,15 @@ library
         , case-insensitive >= 1.2
         , cassava >= 0.5.1
         , chainweb-storage >= 0.1
-        , chronos >= 1.1
         , clock >= 0.7
         , configuration-tools >= 0.6
         , crypton-connection >= 0.2 && < 0.4
         , containers >= 0.5
         , crypton >= 0.31
+        , crypton-connection >= 0.2 && < 0.4
+        , crypton-x509 >=1.7
+        , crypton-x509-system >=1.6
+        , crypton-x509-validation >=1.6
         , cuckoo >= 0.3
         , data-default >=0.7
         , data-dword >= 0.3
@@ -410,8 +414,8 @@ library
         , mmorph >= 1.1
         , monad-control >= 1.0
         , mtl >= 2.3
-        , mwc-random >= 0.13
         , mwc-probability >= 2.0 && <2.4
+        , mwc-random >= 0.13
         , network >= 3.1.2
         , optparse-applicative >= 0.14
         , pact >= 4.2.0.1
@@ -421,6 +425,7 @@ library
         , pem >=0.2
         , primitive >= 0.7.1.0
         , random >= 1.2
+        , rocksdb-haskell-kadena >= 1.1.0
         , rosetta >= 1.0
         , safe-exceptions >= 0.1
         , scheduler >= 1.4
@@ -435,29 +440,26 @@ library
         , streaming-commons >= 0.2
         , template-haskell >= 2.14
         , text >= 2.0
-        , trifecta >= 2.1
         , these >= 1.0
         , time >= 1.12.2
         , tls >=1.9
         , tls-session-manager >= 0.0
         , token-bucket >= 0.1
         , transformers >= 0.5
+        , trifecta >= 2.1
         , unliftio ^>= 0.2
         , unordered-containers >= 0.2.16
         , uuid >= 1.3.15
+        , vector >= 0.12.2
+        , vector-algorithms >= 0.7
         , wai >= 3.2.2.1
         , wai-app-static >= 3.1.6.3
         , wai-cors >= 0.2.7
         , wai-extra >= 3.0.28
-        , wai-middleware-validation
-        , vector >= 0.12.2
-        , vector-algorithms >= 0.7
         , wai-middleware-throttle >= 0.3
+        , wai-middleware-validation
         , warp >= 3.3.6
         , warp-tls >= 3.4
-        , crypton-x509 >=1.7
-        , crypton-x509-system >=1.6
-        , crypton-x509-validation >=1.6
         , yaml >= 0.11
         , yet-another-logger >= 0.4.1
 
@@ -785,7 +787,6 @@ executable cwtool
         , resourcet >= 1.3
         , retry >= 0.9
         , wreq >= 0.5
-        , regex-tdfa >= 1.3
         , rocksdb-haskell-kadena >= 1.1.0
         , safe-exceptions >= 0.1
         , servant-client >= 0.18.2
@@ -814,6 +815,40 @@ executable cwtool
     if flag(ed25519)
         cpp-options: -DWITH_ED25519=1
 
+executable compact
+    import: warning-flags, debugging-flags
+    default-language: Haskell2010
+    ghc-options:
+        -threaded
+        -rtsopts
+        "-with-rtsopts=-N -A4M --disable-delayed-os-memory-return -qn1"
+    hs-source-dirs:
+        compact
+    main-is: Main.hs
+    build-depends:
+        -- internal
+        , chainweb
+
+        -- external
+        , base >= 4.12 && < 5
+
+executable pact-diff
+    import: warning-flags, debugging-flags
+    default-language: Haskell2010
+    ghc-options:
+        -threaded
+        -rtsopts
+        "-with-rtsopts=-N -A4M --disable-delayed-os-memory-return -qn1"
+    hs-source-dirs:
+        pact-diff
+    main-is: Main.hs
+    build-depends:
+        -- internal
+        , chainweb
+
+        -- external
+        , base >= 4.12 && < 5
+
 benchmark bench
     import: warning-flags, debugging-flags
     default-language: Haskell2010

compact/Main.hs

@@ -0,0 +1,11 @@
+{-# language
+    ImportQualifiedPost
+#-}
+
+module Main (main) where
+
+import Chainweb.Pact.Backend.Compaction qualified as Compact
+
+main :: IO ()
+main = do
+  Compact.main