KES Agent Connectivity

cabal.project

@@ -34,6 +34,10 @@ multi-repl: True
 
 import: ./asserts.cabal
 
+allow-newer: plutus-core:cardano-crypto-class
+           , bytestring
+           , serdoc-core:tasty-quickcheck
+
 package ouroboros-network
   -- Certain ThreadNet tests rely on transactions to be submitted promptly after
   -- a node (re)start. Therefore, we disable this flag (see
@@ -44,3 +48,54 @@ package ouroboros-network
 if(os(windows))
   constraints:
     bitvec -simd
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/cardano-base
+  tag: b2cec3fbcde4bacb9c961e5510d5a1d3754c4e2b
+  --sha256: sha256-TDEBINZ3SkhpRNomMdt53bR3gdzgkWR9jIlAr8yrU6o=
+  subdir:
+      cardano-crypto-class
+
+source-repository-package
+  type: git
+  location: [email protected]:input-output-hk/kes-agent
+  tag: 660de3128d953fc81aec0c52f5677c0387e0ca4a
+  --sha256: sha256-N4XRVqC+UgWej+J16RPh3EO6MSIE3wmJvmP5/nRgIuw=
+  subdir:
+      kes-agent
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/cardano-ledger
+  tag: 9d380ab7d6ae52ff66aae9a19dbb3036b1b13c94
+  --sha256: sha256-N4XRVqC+UgWej+J16RPh3EO6MSIE3wmJvmP5/nRgIuw=
+  subdir:
+      eras/allegra/impl
+      eras/alonzo/impl
+      eras/alonzo/test-suite
+      eras/babbage/impl
+      eras/babbage/test-suite
+      eras/byron/chain/executable-spec
+      eras/byron/crypto
+      eras/byron/crypto/test
+      eras/byron/ledger/executable-spec
+      eras/byron/ledger/impl
+      eras/byron/ledger/impl/test
+      eras/conway/impl
+      eras/conway/test-suite
+      eras/mary/impl
+      eras/shelley/impl
+      eras/shelley-ma/test-suite
+      eras/shelley/test-suite
+      libs/cardano-data
+      libs/cardano-ledger-api
+      libs/cardano-ledger-binary
+      libs/cardano-ledger-core
+      libs/cardano-ledger-test
+      libs/cardano-protocol-tpraos
+      libs/constrained-generators
+      libs/non-integral
+      libs/set-algebra
+      libs/small-steps
+      libs/vector-map

ouroboros-consensus-cardano/changelog.d/20250130_093803_tdammers_mlocked_kes_rebase.md

@@ -0,0 +1,18 @@
+### Breaking
+
+- Use new mlocked KES API for all internal KES sign key handling.
+- Add finalizers to all block forgings (required by `ouroboros-consensus`).
+- Change `HotKey` to manage not only KES sign keys, but also the corresponding
+  OpCerts. This is in preparation for KES agent connectivity: with the new
+  design, the KES agent will provide both KES sign keys and matching OpCerts
+  together, and we need to be able to dynamically replace them both together.
+- Add finalizer to `HotKey`. This takes care of securely forgetting any KES
+  keys the HotKey may still hold, and will be called automatically when the
+  owning block forging terminates.
+- Change `ShelleyLeaderCredentials` to not contain the KES sign key itself
+  anymore. Instead, the `CanBeLeader` data structure now contains a
+  `praosCanBeLeaderCredentialsSource` field, which specifies how to obtain the
+  actual credentials (OpCert and KES SignKey).
+- The `KesKey` data type in `unstable-cardano-tools` has been renamed to
+  `UnsoundPureKesKey`, to reflect the fact that it uses the old, unsound KES
+  API (which does not use mlocking or secure forgetting).

ouroboros-consensus-cardano/ouroboros-consensus-cardano.cabal

@@ -129,16 +129,16 @@ library
     bytestring >=0.10 && <0.13,
     cardano-binary,
     cardano-crypto,
-    cardano-crypto-class,
+    cardano-crypto-class ^>= 2.2,
     cardano-crypto-wrapper,
     cardano-ledger-allegra ^>=1.6,
     cardano-ledger-alonzo ^>=1.12,
     cardano-ledger-api ^>=1.10,
     cardano-ledger-babbage ^>=1.10,
-    cardano-ledger-binary ^>=1.5,
+    cardano-ledger-binary ^>=1.6,
     cardano-ledger-byron ^>=1.0.1,
     cardano-ledger-conway ^>=1.18,
-    cardano-ledger-core ^>=1.16,
+    cardano-ledger-core ^>=1.17,
     cardano-ledger-mary ^>=1.7,
     cardano-ledger-shelley ^>=1.15,
     cardano-prelude,
@@ -147,9 +147,10 @@ library
     cardano-strict-containers,
     cborg ^>=0.2.2,
     containers >=0.5 && <0.8,
-    cryptonite >=0.25 && <0.31,
+    crypton,
     deepseq,
     formatting >=6.3 && <7.3,
+    kes-agent,
     measures,
     microlens,
     mtl,
@@ -159,6 +160,7 @@ library
     ouroboros-network-api ^>=0.12,
     serialise ^>=0.2,
     small-steps,
+    serdoc-core,
     sop-core ^>=0.5,
     sop-extras ^>=0.2,
     strict-sop-core ^>=0.1,
@@ -301,7 +303,7 @@ library unstable-shelley-testlib
     cardano-ledger-alonzo,
     cardano-ledger-alonzo-test,
     cardano-ledger-babbage-test,
-    cardano-ledger-conway-test >=1.2.1,
+    cardano-ledger-conway-test >=1.3.0,
     cardano-ledger-core:{cardano-ledger-core, testlib},
     cardano-ledger-mary,
     cardano-ledger-shelley:{cardano-ledger-shelley, testlib},
@@ -311,6 +313,7 @@ library unstable-shelley-testlib
     cardano-strict-containers,
     containers,
     generic-random,
+    kes-agent,
     microlens,
     mtl,
     ouroboros-consensus:{ouroboros-consensus, unstable-consensus-testlib},
@@ -319,6 +322,7 @@ library unstable-shelley-testlib
     ouroboros-consensus-protocol:{ouroboros-consensus-protocol, unstable-protocol-testlib},
     ouroboros-network-api,
     quiet ^>=0.2,
+    serdoc-core,
     small-steps,
 
 test-suite shelley-test
@@ -557,6 +561,7 @@ library unstable-cardano-tools
     ouroboros-network-framework ^>=0.16,
     ouroboros-network-protocols,
     resource-registry,
+    serdoc-core,
     serialise ^>=0.2,
     singletons,
     sop-core,

ouroboros-consensus-cardano/src/byron/Ouroboros/Consensus/Byron/Crypto/DSIGN.hs

@@ -92,7 +92,7 @@ instance DSIGNAlgorithm ByronDSIGN where
       where
         seedBytes = case getBytesFromSeed 32 seed of
           Just (x,_) -> x
-          Nothing    -> throw $ SeedBytesExhausted (-1) -- TODO We can't get the seed size!
+          Nothing    -> throw $ SeedBytesExhausted (-1) (-1) -- TODO We can't get the seed size!
 
     deriveVerKeyDSIGN (SignKeyByronDSIGN sk) = VerKeyByronDSIGN $ toVerification sk
 

ouroboros-consensus-cardano/src/byron/Ouroboros/Consensus/Byron/Node.hs

@@ -140,6 +140,7 @@ byronBlockForging creds = BlockForging {
                                slot
                                tickedPBftState
     , forgeBlock       = \cfg -> return ....: forgeByronBlock cfg
+    , finalize = pure ()
     }
   where
     canBeLeader = mkPBftCanBeLeader creds

ouroboros-consensus-cardano/src/ouroboros-consensus-cardano/Ouroboros/Consensus/Cardano/Node.hs

@@ -38,6 +38,9 @@ module Ouroboros.Consensus.Cardano.Node (
   , CardanoHardForkTriggers (.., CardanoHardForkTriggers', triggerHardForkShelley, triggerHardForkAllegra, triggerHardForkMary, triggerHardForkAlonzo, triggerHardForkBabbage, triggerHardForkConway)
   , CardanoProtocolParams (..)
   , MaxMajorProtVer (..)
+  , ProtocolParamsByron
+  , ProtocolParamsShelleyBased
+  , CheckpointsMap
   , TriggerHardFork (..)
   , protocolClientInfoCardano
   , protocolInfoCardano
@@ -57,8 +60,7 @@ import qualified Cardano.Ledger.Api.Transition as L
 import qualified Cardano.Ledger.BaseTypes as SL
 import qualified Cardano.Ledger.Shelley.API as SL
 import           Cardano.Prelude (cborError)
-import qualified Cardano.Protocol.TPraos.OCert as Absolute (KESPeriod (..),
-                     ocertKESPeriod)
+import qualified Cardano.Protocol.TPraos.OCert as Absolute (KESPeriod (..))
 import qualified Codec.CBOR.Decoding as CBOR
 import           Codec.CBOR.Encoding (Encoding)
 import qualified Codec.CBOR.Encoding as CBOR
@@ -72,7 +74,7 @@ import           Data.SOP.Index (Index (..))
 import           Data.SOP.OptNP (NonEmptyOptNP, OptNP (OptSkip))
 import qualified Data.SOP.OptNP as OptNP
 import           Data.SOP.Strict
-import           Data.Word (Word16, Word64)
+import           Data.Word (Word16)
 import           Lens.Micro ((^.))
 import           Ouroboros.Consensus.Block
 import           Ouroboros.Consensus.Byron.Ledger (ByronBlock)
@@ -92,10 +94,9 @@ import           Ouroboros.Consensus.Ledger.Extended
 import           Ouroboros.Consensus.Node.NetworkProtocolVersion
 import           Ouroboros.Consensus.Node.ProtocolInfo
 import           Ouroboros.Consensus.Node.Run
-import qualified Ouroboros.Consensus.Protocol.Ledger.HotKey as HotKey
 import           Ouroboros.Consensus.Protocol.Praos (Praos, PraosParams (..))
-import           Ouroboros.Consensus.Protocol.Praos.Common
-                     (praosCanBeLeaderOpCert)
+import           Ouroboros.Consensus.Protocol.Praos.Common (PraosCanBeLeader (..), instantiatePraosCredentials)
+import           Ouroboros.Consensus.Protocol.Praos.AgentClient
 import           Ouroboros.Consensus.Protocol.TPraos (TPraos, TPraosParams (..))
 import qualified Ouroboros.Consensus.Protocol.TPraos as Shelley
 import           Ouroboros.Consensus.Shelley.HFEras ()
@@ -104,15 +105,15 @@ import qualified Ouroboros.Consensus.Shelley.Ledger as Shelley
 import           Ouroboros.Consensus.Shelley.Ledger.Block (IsShelleyBlock,
                      ShelleyBlockLedgerEra)
 import           Ouroboros.Consensus.Shelley.Ledger.NetworkProtocolVersion
+import           Ouroboros.Consensus.Protocol.Ledger.HotKey (HotKey)
 import           Ouroboros.Consensus.Shelley.Node
-import           Ouroboros.Consensus.Shelley.Node.Common (ShelleyEraWithCrypto,
-                     shelleyBlockIssuerVKey)
+import           Ouroboros.Consensus.Shelley.Node.Common (ShelleyEraWithCrypto, shelleyBlockIssuerVKey)
 import qualified Ouroboros.Consensus.Shelley.Node.Praos as Praos
 import qualified Ouroboros.Consensus.Shelley.Node.TPraos as TPraos
 import           Ouroboros.Consensus.Storage.Serialisation
 import           Ouroboros.Consensus.TypeFamilyWrappers
 import           Ouroboros.Consensus.Util.Assert
-import           Ouroboros.Consensus.Util.IOLike
+
 {-------------------------------------------------------------------------------
   SerialiseHFC
 -------------------------------------------------------------------------------}
@@ -465,7 +466,10 @@ data CardanoProtocolParams c = CardanoProtocolParams {
 -- PRECONDITION: only a single set of Shelley credentials is allowed when used
 -- for mainnet (check against @'SL.gNetworkId' == 'SL.Mainnet'@).
 protocolInfoCardano ::
-     forall c m. (IOLike m, CardanoHardForkConstraints c)
+     forall c m.
+     ( CardanoHardForkConstraints c
+     , KESAgentContext c m
+     )
   => CardanoProtocolParams c
   -> ( ProtocolInfo      (CardanoBlock c)
      , m [BlockForging m (CardanoBlock c)]
@@ -475,12 +479,12 @@ protocolInfoCardano paramsCardano
   , length credssShelleyBased > 1
   = error "Multiple Shelley-based credentials not allowed for mainnet"
   | otherwise
-  = assertWithMsg (validateGenesis genesisShelley)
+  = assertWithMsg (validateGenesis genesisShelley) $
     ( ProtocolInfo {
         pInfoConfig       = cfg
       , pInfoInitLedger   = initExtLedgerStateCardano
       }
-    , blockForging
+    , mkBlockForgings
     )
   where
     CardanoProtocolParams {
@@ -555,7 +559,7 @@ protocolInfoCardano paramsCardano
           initialNonceShelley
           genesisShelley
 
-    TPraosParams { tpraosSlotsPerKESPeriod, tpraosMaxKESEvo } = tpraosParams
+    TPraosParams { tpraosSlotsPerKESPeriod } = tpraosParams
 
     praosParams :: PraosParams
     praosParams = PraosParams
@@ -572,7 +576,7 @@ protocolInfoCardano paramsCardano
             (SL.mkActiveSlotCoeff $ SL.sgActiveSlotsCoeff genesisShelley)
       }
 
-    PraosParams { praosSlotsPerKESPeriod, praosMaxKESEvo } = praosParams
+    PraosParams { praosSlotsPerKESPeriod } = praosParams
 
     blockConfigShelley :: BlockConfig (ShelleyBlock (TPraos c) (ShelleyEra c))
     blockConfigShelley =
@@ -827,8 +831,8 @@ protocolInfoCardano paramsCardano
     -- credentials. If there are multiple Shelley credentials, we merge the
     -- Byron credentials with the first Shelley one but still have separate
     -- threads for the remaining Shelley ones.
-    blockForging :: m [BlockForging m (CardanoBlock c)]
-    blockForging = do
+    mkBlockForgings :: m ([BlockForging m (CardanoBlock c)])
+    mkBlockForgings = do
         shelleyBased <- traverse blockForgingShelleyBased credssShelleyBased
         let blockForgings :: [NonEmptyOptNP (BlockForging m) (CardanoEras c)]
             blockForgings = case (mBlockForgingByron, shelleyBased) of
@@ -854,24 +858,20 @@ protocolInfoCardano paramsCardano
          ShelleyLeaderCredentials c
       -> m (NonEmptyOptNP (BlockForging m) (CardanoEras c))
     blockForgingShelleyBased credentials = do
-        let ShelleyLeaderCredentials
-              { shelleyLeaderCredentialsInitSignKey = initSignKey
-              , shelleyLeaderCredentialsCanBeLeader = canBeLeader
-              } = credentials
-
-        hotKey <- do
-          let maxKESEvo :: Word64
-              maxKESEvo = assert (tpraosMaxKESEvo == praosMaxKESEvo) praosMaxKESEvo
-
-              startPeriod :: Absolute.KESPeriod
-              startPeriod = Absolute.ocertKESPeriod $ praosCanBeLeaderOpCert canBeLeader
-
-          HotKey.mkHotKey @m @c initSignKey startPeriod maxKESEvo
+        let canBeLeader = shelleyLeaderCredentialsCanBeLeader credentials
 
         let slotToPeriod :: SlotNo -> Absolute.KESPeriod
             slotToPeriod (SlotNo slot) = assert (tpraosSlotsPerKESPeriod == praosSlotsPerKESPeriod) $
               Absolute.KESPeriod $ fromIntegral $ slot `div` praosSlotsPerKESPeriod
 
+        let maxKESEvo = assert (tpraosMaxKESEvo tpraosParams == praosMaxKESEvo praosParams) $
+                        tpraosMaxKESEvo tpraosParams
+
+        hotKey :: HotKey c m <-
+                    instantiatePraosCredentials
+                    maxKESEvo
+                    (praosCanBeLeaderCredentialsSource canBeLeader)
+
         let tpraos :: forall era.
                  ShelleyEraWithCrypto c       (TPraos c) era
               => BlockForging m (ShelleyBlock (TPraos c) era)

ouroboros-consensus-cardano/src/shelley/Ouroboros/Consensus/Shelley/Node.hs

@@ -44,6 +44,7 @@ import           Ouroboros.Consensus.Shelley.Ledger
 import           Ouroboros.Consensus.Shelley.Ledger.Inspect ()
 import           Ouroboros.Consensus.Shelley.Ledger.NetworkProtocolVersion ()
 import           Ouroboros.Consensus.Shelley.Node.DiffusionPipelining ()
+import           Ouroboros.Consensus.Shelley.Node.Common
 import           Ouroboros.Consensus.Shelley.Node.Serialisation ()
 import           Ouroboros.Consensus.Shelley.Node.TPraos
 import           Ouroboros.Consensus.Shelley.Protocol.Abstract (pHeaderIssuer)

ouroboros-consensus-cardano/src/shelley/Ouroboros/Consensus/Shelley/Node/Common.hs

@@ -48,12 +48,7 @@ import           Ouroboros.Consensus.Storage.ImmutableDB
 -------------------------------------------------------------------------------}
 
 data ShelleyLeaderCredentials c = ShelleyLeaderCredentials
-  { -- | The unevolved signing KES key (at evolution 0).
-    --
-    -- Note that this is not inside 'ShelleyCanBeLeader' since it gets evolved
-    -- automatically, whereas 'ShelleyCanBeLeader' does not change.
-    shelleyLeaderCredentialsInitSignKey :: SL.SignKeyKES c,
-    shelleyLeaderCredentialsCanBeLeader :: PraosCanBeLeader c,
+  { shelleyLeaderCredentialsCanBeLeader :: PraosCanBeLeader c,
     -- | Identifier for this set of credentials.
     --
     -- Useful when the node is running with multiple sets of credentials.