NOT MERGE: Leios prototype remake on 11.0.1

.envrc

@@ -0,0 +1 @@
+use flake

bench/plutus-scripts-bench/plutus-scripts-bench.cabal

@@ -82,7 +82,7 @@ library
   -- IOG dependencies
   --------------------------
   build-depends:
-    , cardano-api             ^>=11.0
+    , cardano-api ^>= 11.0 || ^>= 11.1
     , plutus-ledger-api       ^>=1.63
     , plutus-tx               ^>=1.63
     , plutus-tx-plugin        ^>=1.63

bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/NodeToNode.hs

@@ -110,9 +110,11 @@ benchmarkConnectTxSubmit EnvConsts { .. } handshakeTracer submissionTracer codec
   blkN2nVer = supportedVers Map.! n2nVer
   supportedVers :: Map.Map NodeToNodeVersion (BlockNodeToNodeVersion blk)
   supportedVers = supportedNodeToNodeVersions (Proxy @blk)
+  -- Codecs gained two type params for LeiosNotify (bLN) and LeiosFetch (bLF)
+  -- via the Leios chunks; they are unused on the tx-generator initiator path.
   myCodecs :: Codecs blk NtN.RemoteAddress DeserialiseFailure IO
                 ByteString ByteString ByteString ByteString ByteString ByteString
-                ByteString
+                ByteString ByteString ByteString
   myCodecs  = defaultCodecs codecConfig blkN2nVer encodeRemoteAddress decodeRemoteAddress n2nVer
   peerMultiplex :: NtN.Versions NodeToNodeVersion
                                 NtN.NodeToNodeVersionData
@@ -170,14 +172,14 @@ benchmarkConnectTxSubmit EnvConsts { .. } handshakeTracer submissionTracer codec
     => NodeToNodeVersion
     -> remotePeer
     -> Channel IO ByteString
-    -> IO ((), Maybe ByteString)
+    -> IO ((), Maybe (Mux.Reception ByteString))
   kaClient _version them channel = do
     keepAliveRng <- newStdGen
     peerGSVMap <- liftIO . newTVarIO $ Map.singleton them defaultGSV
     runPeerWithLimits
       mempty
       (cKeepAliveCodec myCodecs)
-      (byteLimitsKeepAlive (const 0)) -- TODO: Real Bytelimits, see #1727
+      byteLimitsKeepAlive
       timeLimitsKeepAlive
       channel
       $ keepAliveClientPeer

bench/tx-generator/tx-generator.cabal

@@ -109,9 +109,9 @@ library
                       , attoparsec-aeson
                       , base16-bytestring
                       , bytestring
-                      , cardano-api ^>= 11.0
+                      , cardano-api ^>= 11.0 || ^>= 11.1
                       , cardano-binary
-                      , cardano-cli ^>= 11.0
+                      , cardano-cli ^>= 11.0 || ^>= 11.1
                       , cardano-crypto-class
                       , cardano-crypto-wrapper
                       , cardano-data
@@ -142,7 +142,7 @@ library
                       , network-mux
                       , optparse-applicative
                       , ouroboros-consensus:{ouroboros-consensus, cardano, diffusion} >= 3.0.1
-                      , ouroboros-network:{api, framework, framework-tracing, ouroboros-network, protocols} >= 1.1
+                      , ouroboros-network:{api, framework, framework-tracing, tracing, ouroboros-network, protocols} >= 1.1
                       , plutus-ledger-api
                       , plutus-tx
                       , random

cabal.project

@@ -51,7 +51,11 @@ packages:
   trace-forward
 
 -- Needed when cross compiling
-extra-packages: alex, dmq-node >= 0.4.2.0
+extra-packages: alex
+-- dmq-node dropped: the latest CHaP version (0.4.2.0) requires
+-- ouroboros-network:framework-tracing, a sublib that doesn't exist
+-- in the leios-prototype-remake network fork. Re-add once a newer
+-- dmq-node release tracks the post-split network layout.
 
 program-options
   ghc-options: -Werror
@@ -89,6 +93,42 @@ allow-newer:
 -- Do NOT add more source-repository-package stanzas here unless they are strictly
 -- temporary! Please read the section in CONTRIBUTING about updating dependencies.
 
+-- Pin ouroboros-consensus at the Leios prototype remake branch — five chunks
+-- on top of upstream main: mempool access + forging, diffusion (LeiosNotify +
+-- LeiosFetch + Reception migration), resolving + chain inclusion, voting,
+-- and the ImmDBServer schedule-driven demo.
+-- See IntersectMBO/ouroboros-consensus#2041.
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/ouroboros-consensus
+  tag: 663412f5546a5913ed8197024f82b11b61d3d3dd
+  --sha256: sha256-UG7hzcE8OoDZNJ/i6XZ5XCiwdTIlAOfUSEtmASQCPO4=
+
+-- Pin cardano-ledger at the Leios prototype remake branch — adds
+-- 'Maybe LeiosCert' on the Dijkstra block body (Dijkstra-only).
+-- See IntersectMBO/cardano-ledger#5839.
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/cardano-ledger
+  tag: c47305fcf47bd77437b837d0dfb9cb4181bfbc77
+  --sha256: 1db2xc2gs44g5035srqzgqkbv3a2yacmvnwchd1jnjrfkibnxnak
+  subdir:
+    libs/cardano-ledger-core
+    eras/dijkstra/impl
+
+-- Pin ouroboros-network at the Leios prototype remake branch — adds the
+-- 'BearerBytes' class and arrival-time 'Reception' on recv.
+-- See IntersectMBO/ouroboros-network#5370.
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/ouroboros-network
+  tag: 98229666bf4e2fcd365f4cf583c0efeb76e2e1b7
+  --sha256: 1ix6hlrvyrxwy54lq410prjvby31fz8y1hxrkc1np86zc8kan3m4
+  subdir:
+    ouroboros-network
+    cardano-diffusion
+    network-mux
+
 if impl(ghc >= 9.12)
   -- GHC 9.12 support - master branch
   source-repository-package

cardano-node-chairman/cardano-node-chairman.cabal

@@ -86,5 +86,5 @@ test-suite chairman-tests
   ghc-options:          -threaded -rtsopts "-with-rtsopts=-N -T"
 
   build-tool-depends:   cardano-node:cardano-node
-                      , cardano-cli:cardano-cli ^>= 11.0
+                      , cardano-cli:cardano-cli  ^>= 11.0 || ^>= 11.1
                       , cardano-node-chairman:cardano-node-chairman

cardano-node/cardano-node.cabal

@@ -63,6 +63,7 @@ library
   exposed-modules:      Cardano.Node.Configuration.Logging
                         Cardano.Node.Configuration.NodeAddress
                         Cardano.Node.Configuration.POM
+                        Cardano.Node.Configuration.Leios
                         Cardano.Node.Configuration.LedgerDB
                         Cardano.Node.Configuration.Socket
                         Cardano.Node.Configuration.TopologyP2P
@@ -138,7 +139,7 @@ library
                       , async
                       , base16-bytestring
                       , bytestring
-                      , cardano-api ^>= 11.0
+                      , cardano-api ^>= 11.0 || ^>= 11.1
                       , cardano-data
                       , cardano-crypto-class ^>=2.3
                       , cardano-crypto-wrapper
@@ -187,7 +188,7 @@ library
                       , nothunks
                       , optparse-applicative
                       , ouroboros-consensus:{ouroboros-consensus, lmdb, lsm, cardano, diffusion, protocol} ^>= 3.0.1
-                      , ouroboros-network:{api, ouroboros-network, orphan-instances, framework, protocols, framework-tracing, tracing} ^>= 1.1
+                      , ouroboros-network:{api, ouroboros-network, orphan-instances, framework, framework-tracing, protocols, tracing} ^>= 1.1
                       , cardano-diffusion:{api, cardano-diffusion, orphan-instances, tracing} ^>=1.0
                       , prettyprinter
                       , prettyprinter-ansi-terminal

cardano-node/src/Cardano/Node/Configuration/Leios.hs

@@ -0,0 +1,34 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+
+module Cardano.Node.Configuration.Leios(
+  LeiosDbConfig(..)
+  ) where
+
+import           Data.Aeson (FromJSON (parseJSON), ToJSON (toJSON), Value (String), object,
+                   withObject, (.:), (.=))
+
+data LeiosDbConfig = LeiosDbInMemory
+   | LeiosDbSQLite !FilePath
+   deriving (Eq, Show)
+
+instance FromJSON LeiosDbConfig where
+  parseJSON = withObject "LeiosDbConfig" $ \o -> do
+    backend :: String <- o .: "Backend"
+    case backend of
+      "InMemory" -> return LeiosDbInMemory
+      "SQLite" -> do
+        fp <- o .: "Filepath"
+        return $ LeiosDbSQLite fp
+      _ -> fail $ "Invalid LeiosDb backend " <> backend <> ", did you mean InMemory or SQLite?"
+
+instance ToJSON LeiosDbConfig where
+  toJSON LeiosDbInMemory =
+    object
+      [ "Backend" .=  String "InMemory"
+      ]
+  toJSON (LeiosDbSQLite fp) =
+    object
+      [ "Backend" .= String "SQLite",
+        "Filepath" .= fp
+      ]

cardano-node/src/Cardano/Node/Configuration/POM.hs

@@ -33,6 +33,7 @@ import           Cardano.Network.ConsensusMode (ConsensusMode (..), defaultConse
 import qualified Cardano.Network.Diffusion.Configuration as Cardano
 import           Cardano.Network.PeerSelection (NumberOfBigLedgerPeers (..))
 import           Cardano.Node.Configuration.LedgerDB
+import           Cardano.Node.Configuration.Leios (LeiosDbConfig (..))
 import           Cardano.Node.Configuration.Socket (SocketConfig (..))
 import           Cardano.Node.Handlers.Shutdown
 import           Cardano.Node.Protocol.Types (Protocol (..))
@@ -209,6 +210,9 @@ data NodeConfiguration
 
        , ncTxSubmissionLogicVersion :: TxSubmissionLogicVersion
        , ncTxSubmissionInitDelay :: TxSubmissionInitDelay
+
+       -- Leios
+       , ncLeiosDbConfig :: LeiosDbConfig
        } deriving (Eq, Show)
 
 -- | We expose the `Ouroboros.Network.Mux.ForkPolicy` as a `NodeConfiguration` field.
@@ -316,6 +320,9 @@ data PartialNodeConfiguration
 
          -- gRPC
        , pncRpcConfig :: !PartialRpcConfig
+
+       -- Leios
+       , pncLeiosDbConfig :: !(Last LeiosDbConfig)
        } deriving (Eq, Generic, Show)
 
 instance AdjustFilePaths PartialNodeConfiguration where
@@ -443,6 +450,9 @@ instance FromJSON PartialNodeConfiguration where
             maybe (pncTxSubmissionInitDelay defaultPartialNodeConfiguration) (fmap TxSubmissionInitDelay)
               <$> v .:? "TxSubmissionInitDelay"
       pncTxSubmissionInitDelay <- parseInitDelay
+
+      pncLeiosDbConfig <- Last <$> v .:? "LeiosDbConfig"
+
       pure PartialNodeConfiguration {
              pncProtocolConfig
            , pncSocketConfig = Last . Just $ SocketConfig mempty mempty mempty pncSocketPath
@@ -493,6 +503,7 @@ instance FromJSON PartialNodeConfiguration where
            , pncRpcConfig
            , pncTxSubmissionLogicVersion = txSubmissionLogicVersion
            , pncTxSubmissionInitDelay
+           , pncLeiosDbConfig
            }
     where
       parseMempoolCapacityBytesOverride v = parseNoOverride <|> parseOverride
@@ -762,6 +773,8 @@ defaultPartialNodeConfiguration =
 
     , pncTxSubmissionLogicVersion = Last $ Just TxSubmissionLogicV1
     , pncTxSubmissionInitDelay = Last $ Just defaultTxSubmissionInitDelay
+
+    , pncLeiosDbConfig = Last (Just (LeiosDbSQLite "leios.db"))
     }
 
 lastOption :: Parser a -> Parser (Last a)
@@ -918,6 +931,10 @@ makeNodeConfiguration pnc = do
 
   ncRpcConfig <- makeRpcConfig $ (pncRpcConfig pnc){nodeSocketPath=ncSocketPath socketConfig}
 
+  ncLeiosDbConfig <-
+    lastToEither "Missing LeiosDbConfig"
+    $ pncLeiosDbConfig pnc
+
   return $ NodeConfiguration
              { ncConfigFile = configFile
              , ncTopologyFile = topologyFile
@@ -969,6 +986,7 @@ makeNodeConfiguration pnc = do
              , ncRpcConfig
              , ncTxSubmissionLogicVersion
              , ncTxSubmissionInitDelay
+             , ncLeiosDbConfig
              }
 
 ncProtocol :: NodeConfiguration -> Protocol

cardano-node/src/Cardano/Node/Orphans.hs

@@ -1,3 +1,4 @@
+{-# LANGUAGE DataKinds #-}
 {-# LANGUAGE DerivingStrategies #-}
 {-# LANGUAGE FlexibleInstances #-}
 {-# LANGUAGE GeneralizedNewtypeDeriving #-}
@@ -16,9 +17,19 @@ import           Cardano.Api (FromCBOR (..), HasTextEnvelope (..), HasTypeProxy
 import qualified Cardano.Crypto.KES.Class as Crypto
 import           Cardano.Protocol.Crypto (KES, StandardCrypto)
 import           Ouroboros.Consensus.Node
+import           Ouroboros.Consensus.Byron.Ledger.Block (ByronBlock)
+import           Ouroboros.Consensus.HardFork.Combinator (HardForkBlock)
 import           Ouroboros.Consensus.Node.Genesis (GenesisConfigFlags (..))
+import           Ouroboros.Consensus.Protocol.Praos (Praos)
 import           Ouroboros.Consensus.Protocol.Praos.Common (PraosCredentialsSource (..))
+import           Ouroboros.Consensus.Protocol.TPraos (TPraos)
+import           Ouroboros.Consensus.Shelley.HFEras ()
+import           Ouroboros.Consensus.Shelley.ShelleyHFC (ShelleyBlockHFC)
+import           Ouroboros.Consensus.Storage.LedgerDB.Forker (ResolveLeiosBlock)
 import           Ouroboros.Consensus.Storage.LedgerDB.Snapshots (Flag (..))
+import           Cardano.Ledger.Api.Era
+                   (AllegraEra, AlonzoEra, BabbageEra, ConwayEra,
+                   MaryEra, ShelleyEra)
 import           Ouroboros.Network.SizeInBytes (SizeInBytes (..))
 
 import           Data.Aeson.Types
@@ -83,3 +94,14 @@ instance HasTextEnvelope (PraosCredentialsSource StandardCrypto) where
   textEnvelopeType _ =
     "PraosCredentialsSource_"
       <> fromString (Crypto.algorithmNameKES (Proxy @(KES StandardCrypto)))
+
+-- 'ResolveLeiosBlock' instances for the HFC-wrapped 'ByronBlock' and per-era
+-- 'ShelleyBlockHFC'. These use the class default (no-op resolution), since
+-- only the Cardano 'HardForkBlock' carries the Dijkstra-era EB closure splice.
+instance ResolveLeiosBlock (HardForkBlock '[ByronBlock])
+instance ResolveLeiosBlock (ShelleyBlockHFC (TPraos c) ShelleyEra)
+instance ResolveLeiosBlock (ShelleyBlockHFC (TPraos c) AllegraEra)
+instance ResolveLeiosBlock (ShelleyBlockHFC (TPraos c) MaryEra)
+instance ResolveLeiosBlock (ShelleyBlockHFC (TPraos c) AlonzoEra)
+instance ResolveLeiosBlock (ShelleyBlockHFC (Praos c) BabbageEra)
+instance ResolveLeiosBlock (ShelleyBlockHFC (Praos c) ConwayEra)

cardano-node/src/Cardano/Node/Parsers.hs

@@ -149,6 +149,7 @@ nodeRunParser = do
            , pncRpcConfig
            , pncTxSubmissionLogicVersion = mempty
            , pncTxSubmissionInitDelay = mempty
+           , pncLeiosDbConfig = mempty
            }
 
 parseSocketPath :: Text -- ^ option name

cardano-node/src/Cardano/Node/Protocol/Types.hs

@@ -16,6 +16,7 @@ import qualified Cardano.Api as Api
 import           Cardano.Node.Orphans ()
 import           Cardano.Node.Queries (HasKESInfo, HasKESMetricsData)
 import           Cardano.Node.TraceConstraints (TraceConstraints)
+import qualified Ouroboros.Consensus.Storage.LedgerDB.Forker
 import           Ouroboros.Network.Block (HeaderHash)
 
 import           Control.DeepSeq (NFData)
@@ -48,6 +49,7 @@ data SomeConsensusProtocol where
                                           , TraceConstraints blk
                                           , Api.ToCBOR (HeaderHash blk)
                                           , Api.FromCBOR (HeaderHash blk)
+                                          , Ouroboros.Consensus.Storage.LedgerDB.Forker.ResolveLeiosBlock blk
                                           )
                            => Api.BlockType blk
                            -> Api.ProtocolInfoArgs blk

cardano-node/src/Cardano/Node/Queries.hs

@@ -234,16 +234,18 @@ instance All GetKESInfo xs => GetKESInfo (HardForkBlock xs) where
 -- * General ledger
 --
 class LedgerQueries blk where
-  ledgerUtxoSize     :: LedgerState blk EmptyMK -> Int
-  ledgerDelegMapSize :: LedgerState blk EmptyMK -> Int
+  ledgerUtxoSize          :: LedgerState blk EmptyMK -> Int
+  ledgerDelegMapSize      :: LedgerState blk EmptyMK -> Int
+  ledgerCumulativeTxBytes :: LedgerState blk EmptyMK -> Word64
 
 class LedgerConwayQueries blk where
   ledgerDRepCount    :: LedgerState blk EmptyMK -> Int
   ledgerDRepMapSize  :: LedgerState blk EmptyMK -> Int
 
 instance LedgerQueries Byron.ByronBlock where
   ledgerUtxoSize = Map.size . Byron.unUTxO . Byron.cvsUtxo . Byron.byronLedgerState
-  ledgerDelegMapSize _ = 0
+  ledgerDelegMapSize      _ = 0
+  ledgerCumulativeTxBytes _ = 0
 
 instance (Ledger.EraAccounts era, Shelley.EraCertState era) => LedgerQueries (Shelley.ShelleyBlock protocol era) where
   ledgerUtxoSize =
@@ -265,6 +267,7 @@ instance (Ledger.EraAccounts era, Shelley.EraCertState era) => LedgerQueries (Sh
       .   Shelley.certPStateL
       )
     . Shelley.shelleyLedgerState
+  ledgerCumulativeTxBytes = Shelley.shelleyCumulativeTxBytes
 
 instance Conway.ConwayEraCertState era => LedgerConwayQueries (Shelley.ShelleyBlock protocol era) where
   ledgerDRepCount =
@@ -289,8 +292,9 @@ instance Conway.ConwayEraCertState era => LedgerConwayQueries (Shelley.ShelleyBl
 
 instance (LedgerQueries x, NoHardForks x)
       => LedgerQueries (HardForkBlock '[x]) where
-  ledgerUtxoSize     = ledgerUtxoSize     . unFlip . project . Flip
-  ledgerDelegMapSize = ledgerDelegMapSize . unFlip . project . Flip
+  ledgerUtxoSize          = ledgerUtxoSize          . unFlip . project . Flip
+  ledgerDelegMapSize      = ledgerDelegMapSize      . unFlip . project . Flip
+  ledgerCumulativeTxBytes = ledgerCumulativeTxBytes . unFlip . project . Flip
 
 instance (LedgerConwayQueries x, NoHardForks x)
       => LedgerConwayQueries (HardForkBlock '[x]) where
@@ -316,6 +320,15 @@ instance LedgerQueries (Cardano.CardanoBlock c) where
     Cardano.LedgerStateBabbage ledgerBabbage -> ledgerDelegMapSize ledgerBabbage
     Cardano.LedgerStateConway  ledgerConway  -> ledgerDelegMapSize ledgerConway
     Cardano.LedgerStateDijkstra  ledgerDijkstra  -> ledgerDelegMapSize ledgerDijkstra
+  ledgerCumulativeTxBytes = \case
+    Cardano.LedgerStateByron     ledgerByron    -> ledgerCumulativeTxBytes ledgerByron
+    Cardano.LedgerStateShelley   ledgerShelley  -> ledgerCumulativeTxBytes ledgerShelley
+    Cardano.LedgerStateAllegra   ledgerAllegra  -> ledgerCumulativeTxBytes ledgerAllegra
+    Cardano.LedgerStateMary      ledgerMary     -> ledgerCumulativeTxBytes ledgerMary
+    Cardano.LedgerStateAlonzo    ledgerAlonzo   -> ledgerCumulativeTxBytes ledgerAlonzo
+    Cardano.LedgerStateBabbage   ledgerBabbage  -> ledgerCumulativeTxBytes ledgerBabbage
+    Cardano.LedgerStateConway    ledgerConway   -> ledgerCumulativeTxBytes ledgerConway
+    Cardano.LedgerStateDijkstra  ledgerDijkstra -> ledgerCumulativeTxBytes ledgerDijkstra
 
 instance LedgerConwayQueries (Cardano.CardanoBlock c) where
   ledgerDRepCount = \case