diff --git a/bench/tx-generator/src/Cardano/TxGenerator/Setup/NodeConfig.hs b/bench/tx-generator/src/Cardano/TxGenerator/Setup/NodeConfig.hs index 8e78199c68b..438d643eed7 100644 --- a/bench/tx-generator/src/Cardano/TxGenerator/Setup/NodeConfig.hs +++ b/bench/tx-generator/src/Cardano/TxGenerator/Setup/NodeConfig.hs @@ -72,6 +72,7 @@ mkNodeConfig configFp_ , byronKeyFile = Just "" , shelleyKESSource = Just (KESKeyFilePath "") , shelleyVRFFile = Just "" + , shelleyBLSFile = Nothing , shelleyCertFile = Just "" , shelleyBulkCredsFile = Just "" } diff --git a/cabal.project b/cabal.project index fea01d6c4f5..2c2fc2ff999 100644 --- a/cabal.project +++ b/cabal.project @@ -13,10 +13,14 @@ repository cardano-haskell-packages -- See CONTRIBUTING for information about these, including some Nix commands -- you need to run if you change them index-state: - , hackage.haskell.org 2026-06-29T22:49:53Z - , cardano-haskell-packages 2026-06-29T12:05:19Z + , hackage.haskell.org 2026-07-15T21:58:35Z + , cardano-haskell-packages 2026-07-27T20:44:57Z constraints: + -- cardano-crypto-class is pinned back to crypton ^>=1.0 (the memory line) in the SRP; a + -- code-free ^>=1.1 refresh had swapped memory→ram and dragged the crypto closure off the + -- still-`memory` CHAP packages. Keep this global cap so nothing else pulls crypton 1.1 (ram). + , any.crypton < 1.1 -- haskell.nix patch does not work for 1.6.8 , any.crypton-x509-system < 1.6.8 -- haskell.nix tries to use 0.7.1.5 for some reason @@ -79,6 +83,8 @@ package plutus-scripts-bench allow-newer: , io-sim:time , io-classes:time + -- TODO: bump packages instead of this + , cardano-crypto-class -- ekg-forward requires ouroboros-network ^>=1.2 but the SRP is still 1.1.0.0 allow-older: @@ -88,12 +94,12 @@ allow-older: -- Do NOT add more source-repository-package stanzas here unless they are strictly -- temporary! Please read the section in CONTRIBUTING about updating dependencies. --- Points to patch on top of ouroboros-consensus/leios-prototype +-- Points to ouroboros-consensus/leios-prototype source-repository-package type: git location: https://github.com/IntersectMBO/ouroboros-consensus - tag: 1c01d7ee86f814369b631a94426b77f633cee3ec - --sha256: sha256-kwmR3z/yw47pfWkNyX912mkFN5FSC3k2wpyvBq2RnrU= + tag: 6dc84b8fead9226dada19ca26d8eb59e0000c388 + --sha256: sha256-tfeIsZvdBPitheKmmq4HDvybzDKHTKXukS4lz8vODiE= -- Points to nfrisby-pr93-lookahead-merge-commit tag source-repository-package @@ -117,6 +123,7 @@ source-repository-package libs/cardano-ledger-binary libs/cardano-ledger-core libs/cardano-protocol-tpraos + eras/byron/crypto eras/shelley/impl eras/shelley/test-suite eras/shelley-ma/test-suite @@ -143,8 +150,8 @@ source-repository-package source-repository-package type: git location: https://github.com/IntersectMBO/cardano-api - tag: 947645ea9c912da13feadc16bef44a7550c3c55b - --sha256: sha256-+fIsJuJV4YBSPdVe3wpIxq9lf6nAI5yB7qdzH0iKmrc= + tag: b66ad3f6363f48d83f0a4f2b80d93e0e21d0dd80 + --sha256: sha256-6y/PUc8y3s7bnhm1zZYYdaAYdzIcbODWkYsJf1t3AVc= subdir: cardano-api cardano-api-gen @@ -154,8 +161,8 @@ source-repository-package source-repository-package type: git location: https://github.com/IntersectMBO/cardano-cli - tag: 2377354ca8d49cd86f63b7f095cfa340b514789d - --sha256: sha256-5c3FjK6Yi8hQEzvTIZ4Ri1wJNTReuKT1rX5+B3BonM4= + tag: 83825043b928952e0db6cca21625842ba0d284fe + --sha256: sha256-QxpNvVGC1sIHGiHSPcr+ETZ9zc9isFumEuEdLl4bGuM= subdir: cardano-cli @@ -168,28 +175,20 @@ source-repository-package kes-agent kes-agent-crypto --- source-repository-package --- type: git --- location: https://github.com/f-f/dmq-node.git --- tag: b7d59834e52d46229bb3ff6e5595509eede730dd --- --sha256: sha256-8GKAaNPRZkKsk10Emokrlnc0sQOx1aHUnggqlMdY0nw= --- subdir: --- dmq-node - -source-repository-package - type: git - location: https://github.com/IntersectMBO/plutus.git - tag: b1db04cc425fab303ac3b79d7140dc2a17f29e6d - --sha256: sha256-XtYzjNVx4+IGWpgm+p/YTf56DrJtK0I98umZCTE3U80= - subdir: - plutus-core - plutus-ledger-api - plutus-tx - plutus-tx-plugin - plutus-metatheory - source-repository-package type: git location: https://github.com/input-output-hk/ekg-forward tag: 85470ae7e8fac9711682234ed39a36eec3d45a9b --sha256: sha256-yk67GraJUsFi5mSETnaeuKNSnNRLOri6FK5yzmRLzxU= + +-- Points to ch1bo/mk-committee +source-repository-package + type: git + location: https://github.com/IntersectMBO/cardano-base + tag: 9c6078d9c39c542abcb5912d0d7c222b7ba8a03d + --sha256: sha256-FfzoMXitbX/pzEJDQExQikGef6FcantzGWvqu2g19ho= + subdir: + cardano-binary + cardano-crypto-class + cardano-crypto-leios + cardano-crypto-praos diff --git a/cardano-node/ChangeLog.md b/cardano-node/ChangeLog.md index 522fc84dd5d..94878ec6d16 100644 --- a/cardano-node/ChangeLog.md +++ b/cardano-node/ChangeLog.md @@ -2,6 +2,8 @@ ## Next version +* Added a `--shelley-bls-key FILEPATH` option to `cardano-node run` for supplying a block producer's BLS (Leios) signing key alongside the existing VRF/KES/operational-certificate keys. The key is threaded into the consensus block-producer credentials and used as the Leios voting key. It is optional: producers that do not supply it no longer vote (previously a placeholder key was derived from cold-key material). Generate one with `cardano-cli dijkstra node key-gen-BLS`. + * Resolve relative `LeiosDbConfig` SQLite paths against `--database-path`, so the default `leios.db` is placed alongside `immutable/` and `volatile/`. * Added txsSyncDurationTotal counter for tracking the total time spent syncing the mempool. diff --git a/cardano-node/src/Cardano/Node/Configuration/POM.hs b/cardano-node/src/Cardano/Node/Configuration/POM.hs index ebb2dd63af9..9a89837839f 100644 --- a/cardano-node/src/Cardano/Node/Configuration/POM.hs +++ b/cardano-node/src/Cardano/Node/Configuration/POM.hs @@ -693,7 +693,7 @@ defaultPartialNodeConfiguration = , pncDiffusionMode = Last $ Just InitiatorAndResponderDiffusionMode , pncExperimentalProtocolsEnabled = Last $ Just False , pncTopologyFile = Last . Just $ TopologyFile "configuration/cardano/mainnet-topology.json" - , pncProtocolFiles = Last . Just $ ProtocolFilepaths Nothing Nothing Nothing Nothing Nothing Nothing + , pncProtocolFiles = Last . Just $ ProtocolFilepaths Nothing Nothing Nothing Nothing Nothing Nothing Nothing , pncValidateDB = Last $ Just False , pncShutdownConfig = Last . Just $ ShutdownConfig Nothing Nothing , pncStartAsNonProducingNode = Last $ Just False diff --git a/cardano-node/src/Cardano/Node/Parsers.hs b/cardano-node/src/Cardano/Node/Parsers.hs index 318dcf61346..1d6b4a44f2f 100644 --- a/cardano-node/src/Cardano/Node/Parsers.hs +++ b/cardano-node/src/Cardano/Node/Parsers.hs @@ -65,6 +65,7 @@ nodeRunParser = do byronKeyFile <- optional parseByronSigningKey shelleyKESSource <- optional parseKesSourceFilePath shelleyVRFFile <- optional parseVrfKeyFilePath + shelleyBLSFile <- optional parseBlsKeyFilePath shelleyCertFile <- optional parseOperationalCertFilePath shelleyBulkCredsFile <- optional parseBulkCredsFilePath startAsNonProducingNode <- (\depr new -> Last depr <> Last new) @@ -103,6 +104,7 @@ nodeRunParser = do , byronKeyFile , shelleyKESSource , shelleyVRFFile + , shelleyBLSFile , shelleyCertFile , shelleyBulkCredsFile } @@ -412,6 +414,15 @@ parseVrfKeyFilePath = <> completer (bashCompleter "file") ) +parseBlsKeyFilePath :: Parser FilePath +parseBlsKeyFilePath = + strOption + ( long "shelley-bls-key" + <> metavar "FILEPATH" + <> help "Path to the BLS (Leios) signing key." + <> completer (bashCompleter "file") + ) + parseStartAsNonProducingNodeDeprecated :: Parser (Maybe Bool) parseStartAsNonProducingNodeDeprecated = flag Nothing (Just True) $ mconcat diff --git a/cardano-node/src/Cardano/Node/Protocol/Shelley.hs b/cardano-node/src/Cardano/Node/Protocol/Shelley.hs index a8b930ae882..cc05b119e80 100644 --- a/cardano-node/src/Cardano/Node/Protocol/Shelley.hs +++ b/cardano-node/src/Cardano/Node/Protocol/Shelley.hs @@ -184,11 +184,18 @@ readLeaderCredentialsSingleton readLeaderCredentialsSingleton ProtocolFilepaths { shelleyCertFile = Just opCertFile, shelleyVRFFile = Just vrfFile, + shelleyBLSFile = mBlsFile, shelleyKESSource = Just kesSource } = do vrfSKey <- firstExceptT FileError (newExceptT $ readFileTextEnvelope (File vrfFile)) + -- The BLS (Leios) key is optional: only block producers participating in + -- Leios supply one alongside their VRF/KES/opcert credentials. + blsSKey <- + firstExceptT FileError $ + traverse (\blsFile -> newExceptT $ readFileTextEnvelope (File blsFile)) mBlsFile + (credentialsSource, vkey) <- case kesSource of KESKeyFilePath kesFile -> do (OperationalCertificate opCert vkey, KesSigningKey kesKey) <- @@ -202,7 +209,7 @@ readLeaderCredentialsSingleton OperationalCertificate _ vkey <- firstExceptT FileError $ newExceptT $ readFileTextEnvelope $ File opCertFile pure (PraosCredentialsAgent socketFile, vkey) - return [mkPraosLeaderCredentials credentialsSource vkey vrfSKey] + return [mkPraosLeaderCredentials credentialsSource vkey vrfSKey blsSKey] -- But not OK to supply some of the files without the others. readLeaderCredentialsSingleton ProtocolFilepaths {shelleyCertFile = Nothing} = @@ -252,7 +259,8 @@ readLeaderCredentialsBulk ProtocolFilepaths { shelleyBulkCredsFile = mfp } = KesSigningKey kesKey <- parseEnvelope scKes let credentialsSource = PraosCredentialsUnsound opCert kesKey vrfSKey <- parseEnvelope scVrf - pure $ mkPraosLeaderCredentials credentialsSource vkey vrfSKey + -- Bulk credentials files do not carry a BLS (Leios) key. + pure $ mkPraosLeaderCredentials credentialsSource vkey vrfSKey Nothing readBulkFile :: Maybe FilePath @@ -277,20 +285,25 @@ mkPraosLeaderCredentials :: PraosCredentialsSource StandardCrypto -> VerificationKey StakePoolKey -> SigningKey VrfKey + -> Maybe (SigningKey BlsKey) -> ShelleyLeaderCredentials StandardCrypto mkPraosLeaderCredentials credentialsSource (StakePoolVerificationKey vkey) - (VrfSigningKey vrfKey) = + (VrfSigningKey vrfKey) + mBlsKey = ShelleyLeaderCredentials { shelleyLeaderCredentialsCanBeLeader = PraosCanBeLeader { praosCanBeLeaderCredentialsSource = credentialsSource, praosCanBeLeaderColdVerKey = coerceKeyRole vkey, - praosCanBeLeaderSignKeyVRF = vrfKey + praosCanBeLeaderSignKeyVRF = vrfKey, + praosCanBeLeaderSignKeyBLS = unBlsSigningKey <$> mBlsKey }, shelleyLeaderCredentialsLabel = "Shelley" } + where + unBlsSigningKey (BlsSigningKey k) = k parseEnvelope :: HasTextEnvelope a diff --git a/cardano-node/src/Cardano/Node/Types.hs b/cardano-node/src/Cardano/Node/Types.hs index 9f9cad5cd57..1006323de59 100644 --- a/cardano-node/src/Cardano/Node/Types.hs +++ b/cardano-node/src/Cardano/Node/Types.hs @@ -179,6 +179,7 @@ data ProtocolFilepaths = , byronKeyFile :: !(Maybe FilePath) , shelleyKESSource :: !(Maybe KESSource) , shelleyVRFFile :: !(Maybe FilePath) + , shelleyBLSFile :: !(Maybe FilePath) , shelleyCertFile :: !(Maybe FilePath) , shelleyBulkCredsFile :: !(Maybe FilePath) } deriving (Eq, Show) @@ -192,6 +193,7 @@ hasProtocolFile ProtocolFilepaths { byronKeyFile, shelleyKESSource, shelleyVRFFile, + shelleyBLSFile, shelleyCertFile, shelleyBulkCredsFile } @@ -199,6 +201,7 @@ hasProtocolFile ProtocolFilepaths { || isJust byronKeyFile || isJust shelleyKESSource || isJust shelleyVRFFile + || isJust shelleyBLSFile || isJust shelleyCertFile || isJust shelleyBulkCredsFile diff --git a/cardano-node/test/Test/Cardano/Node/POM.hs b/cardano-node/test/Test/Cardano/Node/POM.hs index 1e7d4e80f49..5a648e765c4 100644 --- a/cardano-node/test/Test/Cardano/Node/POM.hs +++ b/cardano-node/test/Test/Cardano/Node/POM.hs @@ -197,7 +197,7 @@ testPartialCliConfig = , pncDatabaseFile = mempty , pncDiffusionMode = mempty , pncExperimentalProtocolsEnabled = Last $ Just True - , pncProtocolFiles = Last . Just $ ProtocolFilepaths Nothing Nothing Nothing Nothing Nothing Nothing + , pncProtocolFiles = Last . Just $ ProtocolFilepaths Nothing Nothing Nothing Nothing Nothing Nothing Nothing , pncValidateDB = Last $ Just True , pncProtocolConfig = mempty , pncMaxConcurrencyBulkSync = mempty @@ -249,7 +249,7 @@ eExpectedConfig = do , ncConfigFile = ConfigYamlFilePath "configuration/cardano/mainnet-config.json" , ncTopologyFile = TopologyFile "configuration/cardano/mainnet-topology.json" , ncDatabaseFile = OnePathForAllDbs "mainnet/db/" - , ncProtocolFiles = ProtocolFilepaths Nothing Nothing Nothing Nothing Nothing Nothing + , ncProtocolFiles = ProtocolFilepaths Nothing Nothing Nothing Nothing Nothing Nothing Nothing , ncValidateDB = True , ncProtocolConfig = testNodeProtocolConfiguration , ncDiffusionMode = InitiatorAndResponderDiffusionMode diff --git a/flake.lock b/flake.lock index 95f4374f6f2..cecf509664b 100644 --- a/flake.lock +++ b/flake.lock @@ -3,16 +3,16 @@ "CHaP": { "flake": false, "locked": { - "lastModified": 1782769557, - "narHash": "sha256-tCYMTwH15mNajHybvC3KGVPBPMlLlqIOYE0ZgcqRkJU=", + "lastModified": 1786144110, + "narHash": "sha256-7hIsWllt2gyPIZ8Wvb1BImf9xfQwDkz0B3sXIZpctxg=", "owner": "intersectmbo", "repo": "cardano-haskell-packages", - "rev": "3ee6b1ecce230d62f0083590f38c4ae3457da226", + "rev": "89f08ac683754f1746d5cd1ff4ea60ba13faec29", "type": "github" }, "original": { "owner": "intersectmbo", - "ref": "repo", + "ref": "index-only", "repo": "cardano-haskell-packages", "type": "github" } @@ -306,11 +306,11 @@ "hackageNix_2": { "flake": false, "locked": { - "lastModified": 1782826502, - "narHash": "sha256-G6bt7DeWkDXJWI/fHME467V/SOaUdfG6hk7bTRuWyKg=", + "lastModified": 1786117442, + "narHash": "sha256-hgIx8tjpy6tosXgl5kz38zLGJgnGmLAkix4syRYlEts=", "owner": "input-output-hk", "repo": "hackage.nix", - "rev": "6a87e2657c145eb81d5f0f53702d4f26c08f9cbf", + "rev": "2fd0dc78a03f432400ad9018f43e2ab31a00ac98", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index a4a30f04b0f..31da3854ae0 100644 --- a/flake.nix +++ b/flake.nix @@ -2,8 +2,8 @@ description = "Cardano Node"; nixConfig = { - extra-substituters = ["https://cache.iog.io"]; - extra-trusted-public-keys = ["hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ="]; + extra-substituters = [ "https://cache.iog.io" ]; + extra-trusted-public-keys = [ "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=" ]; }; inputs = { @@ -23,7 +23,7 @@ customConfig.url = "github:input-output-hk/empty-flake"; CHaP = { - url = "github:intersectmbo/cardano-haskell-packages?ref=repo"; + url = "github:intersectmbo/cardano-haskell-packages?ref=index-only"; flake = false; }; @@ -61,447 +61,509 @@ mithril.url = "github:input-output-hk/mithril?ref=refs/tags/2617.0"; }; - outputs = { - cardano-automation, - CHaP, - haskellNix, - incl, - iohkNix, - mithril, - nixpkgs, - self, - utils, - ... - } @ input: let - inherit (builtins) elem match; - inherit (nixpkgs) lib; - inherit (lib) collect getAttr genAttrs filterAttrs hasPrefix head isDerivation mapAttrs optionalAttrs optional optionals recursiveUpdate; - inherit (utils.lib) eachSystem flattenTree; - inherit (iohkNix.lib) prefixNamesWith; - removeRecurse = lib.filterAttrsRecursive (n: _: n != "recurseForDerivations"); - - macOS-security = pkgs: - # Make `/usr/bin/security` available in `PATH`, which is needed for stack - # on darwin which calls this binary to find certificates - pkgs.writeScriptBin "security" ''exec /usr/bin/security "$@"''; - - windowsCompilerNixName = "ghc9122"; - - supportedSystems = import ./nix/supported-systems.nix; - defaultSystem = head supportedSystems; - customConfig = - recursiveUpdate - (import ./nix/custom-config.nix customConfig) - input.customConfig; - - overlays = [ - # Crypto needs to come before haskell.nix. - # FIXME: _THIS_IS_BAD_ - iohkNix.overlays.crypto - haskellNix.overlay - iohkNix.overlays.haskell-nix-extra - iohkNix.overlays.haskell-nix-crypto - iohkNix.overlays.cardano-lib - iohkNix.overlays.utils - (final: prev: { - inherit customConfig; - bench-data-publish = cardano-automation.outputs.packages.${final.stdenv.hostPlatform.system}."bench-data-publish:exe:bench-data-publish"; - gitrev = final.customConfig.gitrev or self.rev or "0000000000000000000000000000000000000000"; - commonLib = - lib - // iohkNix.lib - // final.cardanoLib - // import ./nix/svclib.nix {inherit (final) pkgs;}; - }) - (final: prev: { - # For musl builds, make sure the static `liburing.a` file is not deleted in `postInstall` - # ex: https://github.com/NixOS/nixpkgs/blob/f84a9816b2d5f7caade4b2fab16a66486abb7038/pkgs/by-name/li/liburing/package.nix#L43-L45 - liburing = prev.liburing.overrideAttrs (attrs: - final.lib.optionalAttrs final.stdenv.hostPlatform.isMusl { - postInstall = builtins.replaceStrings ["rm $out/lib/liburing*.a"] [""] attrs.postInstall; - }); - }) - (import ./nix/pkgs.nix) - self.overlay - ]; + outputs = + { + cardano-automation, + CHaP, + haskellNix, + incl, + iohkNix, + mithril, + nixpkgs, + self, + utils, + ... + }@input: + let + inherit (builtins) elem match; + inherit (nixpkgs) lib; + inherit (lib) + collect + getAttr + genAttrs + filterAttrs + hasPrefix + head + isDerivation + mapAttrs + optionalAttrs + optional + optionals + recursiveUpdate + ; + inherit (utils.lib) eachSystem flattenTree; + inherit (iohkNix.lib) prefixNamesWith; + removeRecurse = lib.filterAttrsRecursive (n: _: n != "recurseForDerivations"); - collectExes = project: let - set-git-rev = import ./nix/set-git-rev.nix {inherit (project) pkgs;}; - in - # Take all executables from the project local packages - project.exes - // (with project.hsPkgs; - { - # Add some executables from other relevant packages - inherit (bech32.components.exes) bech32; - inherit (dmq-node.components.exes) dmq-node; - inherit (ouroboros-consensus.components.exes) db-analyser db-synthesizer db-truncater snapshot-converter; - inherit (kes-agent.components.exes) kes-agent kes-agent-control; - # Add cardano-node, cardano-cli and tx-generator with their git revision stamp. - # Keep available an alternative without the git revision, like the other - # passthru (profiled and asserted in nix/haskell.nix) that - # have no git revision but for the same compilation alternative. - cardano-node = let - node = project.exes.cardano-node; - in - recursiveUpdate - (set-git-rev node) - {passthru = {noGitRev = node;};}; - cardano-cli = let - cli = cardano-cli.components.exes.cardano-cli; - in - recursiveUpdate - (set-git-rev cli) - {passthru = {noGitRev = cli;};}; - cardano-submit-api = let - submit-api = project.exes.cardano-submit-api; - in - recursiveUpdate - (set-git-rev submit-api) - {passthru = {noGitRev = submit-api;};}; - } - // optionalAttrs (project.exes ? tx-generator) { - tx-generator = let - tx-gen = project.exes.tx-generator; - in - recursiveUpdate - (set-git-rev tx-gen) - {passthru = {noGitRev = tx-gen;};}; - }); - - mkCardanoNodePackages = project: - (collectExes project) - // { - inherit (project.pkgs) cardanoLib; - }; + macOS-security = + pkgs: + # Make `/usr/bin/security` available in `PATH`, which is needed for stack + # on darwin which calls this binary to find certificates + pkgs.writeScriptBin "security" ''exec /usr/bin/security "$@"''; + + windowsCompilerNixName = "ghc9122"; + + supportedSystems = import ./nix/supported-systems.nix; + defaultSystem = head supportedSystems; + customConfig = recursiveUpdate (import ./nix/custom-config.nix customConfig) input.customConfig; + + overlays = [ + # Crypto needs to come before haskell.nix. + # FIXME: _THIS_IS_BAD_ + iohkNix.overlays.crypto + haskellNix.overlay + iohkNix.overlays.haskell-nix-extra + iohkNix.overlays.haskell-nix-crypto + iohkNix.overlays.cardano-lib + iohkNix.overlays.utils + (final: prev: { + inherit customConfig; + bench-data-publish = + cardano-automation.outputs.packages.${final.stdenv.hostPlatform.system}."bench-data-publish:exe:bench-data-publish"; + gitrev = final.customConfig.gitrev or self.rev or "0000000000000000000000000000000000000000"; + commonLib = + lib // iohkNix.lib // final.cardanoLib // import ./nix/svclib.nix { inherit (final) pkgs; }; + }) + (final: prev: { + # For musl builds, make sure the static `liburing.a` file is not deleted in `postInstall` + # ex: https://github.com/NixOS/nixpkgs/blob/f84a9816b2d5f7caade4b2fab16a66486abb7038/pkgs/by-name/li/liburing/package.nix#L43-L45 + liburing = prev.liburing.overrideAttrs ( + attrs: + final.lib.optionalAttrs final.stdenv.hostPlatform.isMusl { + postInstall = builtins.replaceStrings [ "rm $out/lib/liburing*.a" ] [ "" ] attrs.postInstall; + } + ); + }) + (import ./nix/pkgs.nix) + self.overlay + ]; + + collectExes = + project: + let + set-git-rev = import ./nix/set-git-rev.nix { inherit (project) pkgs; }; + in + # Take all executables from the project local packages + project.exes + // ( + with project.hsPkgs; + { + # Add some executables from other relevant packages + inherit (bech32.components.exes) bech32; + inherit (dmq-node.components.exes) dmq-node; + inherit (ouroboros-consensus.components.exes) + db-analyser + db-synthesizer + db-truncater + snapshot-converter + ; + inherit (kes-agent.components.exes) kes-agent kes-agent-control; + # Add cardano-node, cardano-cli and tx-generator with their git revision stamp. + # Keep available an alternative without the git revision, like the other + # passthru (profiled and asserted in nix/haskell.nix) that + # have no git revision but for the same compilation alternative. + cardano-node = + let + node = project.exes.cardano-node; + in + recursiveUpdate (set-git-rev node) { + passthru = { + noGitRev = node; + }; + }; + cardano-cli = + let + cli = cardano-cli.components.exes.cardano-cli; + in + recursiveUpdate (set-git-rev cli) { + passthru = { + noGitRev = cli; + }; + }; + cardano-submit-api = + let + submit-api = project.exes.cardano-submit-api; + in + recursiveUpdate (set-git-rev submit-api) { + passthru = { + noGitRev = submit-api; + }; + }; + } + // optionalAttrs (project.exes ? tx-generator) { + tx-generator = + let + tx-gen = project.exes.tx-generator; + in + recursiveUpdate (set-git-rev tx-gen) { + passthru = { + noGitRev = tx-gen; + }; + }; + } + ); - mkFlakeAttrs = pkgs: rec { - system = pkgs.stdenv.hostPlatform.system; - inherit (pkgs.haskell-nix) haskellLib; - inherit (haskellLib) collectChecks' collectComponents'; - inherit (pkgs.commonLib) eachEnv environments mkSupervisordCluster; - inherit (pkgs.stdenv) hostPlatform; - project = pkgs.cardanoNodeProject; + mkCardanoNodePackages = + project: + (collectExes project) + // { + inherit (project.pkgs) cardanoLib; + }; - macOS-security = - utils.writeScriptBin "security" ''exec /usr/bin/security "$@"''; - - # This is used by `nix develop .` to open a devShell - devShells = let - shell = import ./shell.nix {inherit pkgs customConfig;}; - in { - inherit (shell) devops workbench-shell; - default = shell.dev; - cluster = shell; - profiled = project.profiled.shell; - }; + mkFlakeAttrs = pkgs: rec { + system = pkgs.stdenv.hostPlatform.system; + inherit (pkgs.haskell-nix) haskellLib; + inherit (haskellLib) collectChecks' collectComponents'; + inherit (pkgs.commonLib) eachEnv environments mkSupervisordCluster; + inherit (pkgs.stdenv) hostPlatform; + project = pkgs.cardanoNodeProject; - # NixOS tests a sandboxed mainnet edge node with submit-api, ensuring - # startup and port listening functionality using the nixos service. It - # also tests Linux binary artifact start up with each set of pre-bundled - # environment configuration files. - nixosTests = import ./nix/nixos/tests { - inherit pkgs ciJobs; - }; + macOS-security = utils.writeScriptBin "security" ''exec /usr/bin/security "$@"''; - checks = - flattenTree project.checks - // - # Linux only checks: - (optionalAttrs hostPlatform.isLinux ( - prefixNamesWith "nixosTests/" (mapAttrs (_: v: v.${system} or v) nixosTests) - )) - # Checks run on default system only: - // (optionalAttrs (system == defaultSystem) { - hlint = pkgs.callPackage pkgs.hlintCheck { - inherit (project.args) src; + # This is used by `nix develop .` to open a devShell + devShells = + let + shell = import ./shell.nix { inherit pkgs customConfig; }; + in + { + inherit (shell) devops workbench-shell; + default = shell.dev; + cluster = shell; + profiled = project.profiled.shell; }; - }); - exes = - (collectExes project) - // { - inherit (pkgs) checkCabalProject; - } - // flattenTree (pkgs.scripts - // { - # `tests` are the test suites which have been built. - inherit (project) tests; - # `benchmarks` (only built, not run). - inherit (project) benchmarks; - }); + # NixOS tests a sandboxed mainnet edge node with submit-api, ensuring + # startup and port listening functionality using the nixos service. It + # also tests Linux binary artifact start up with each set of pre-bundled + # environment configuration files. + nixosTests = import ./nix/nixos/tests { + inherit pkgs ciJobs; + }; - # The parameterisable workbench. - inherit (pkgs) workbench; - - packages = - exes - # Linux only packages: - // optionalAttrs (elem system ["x86_64-linux" "aarch64-linux"]) - (let - workbenchTest = { - profileName, - workbenchStartArgs ? [], - }: - (pkgs.workbench-runner - { - inherit profileName workbenchStartArgs; - backendName = "supervisor"; - useCabalRun = false; - cardano-node-rev = pkgs.gitrev; - }) - .workbench-profile-run; - in { - "dockerImage/node" = pkgs.dockerImage; - "dockerImage/submit-api" = pkgs.submitApiDockerImage; - "dockerImage/tracer" = pkgs.tracerDockerImage; - - # This is a very light profile, no caching and pinning needed. - workbench-ci-test = workbenchTest { - profileName = "ci-test-hydra-coay"; - workbenchStartArgs = ["--create-testnet-data"]; - }; - workbench-ci-test-trace = workbenchTest { - profileName = "ci-test-hydra-coay"; - workbenchStartArgs = ["--create-testnet-data" "--trace"]; - }; + checks = + flattenTree project.checks + // + # Linux only checks: + (optionalAttrs hostPlatform.isLinux ( + prefixNamesWith "nixosTests/" (mapAttrs (_: v: v.${system} or v) nixosTests) + )) + # Checks run on default system only: + // (optionalAttrs (system == defaultSystem) { + hlint = pkgs.callPackage pkgs.hlintCheck { + inherit (project.args) src; + }; + }); - inherit (pkgs) all-profiles-json profile-data-nomadperf; - - system-tests = pkgs.writeShellApplication { - name = "system-tests"; - runtimeInputs = with pkgs; [git gnused]; - text = '' - NODE_REV="${self.rev or ""}" - if [[ -z $NODE_REV ]]; then - echo "Sorry, need clean/pushed git revision to run system tests" - exit 1; - fi - MAKE_TARGET=testpr - mkdir -p tmp && cd tmp - rm -rf cardano-node-tests - git clone https://github.com/intersectmbo/cardano-node-tests.git - cd cardano-node-tests - sed -i '1 s/^.*$/#! \/usr\/bin\/env bash/' ./.github/regression.sh - export NODE_REV - export MAKE_TARGET - nix develop --accept-flake-config .#base -c ./.github/regression.sh 2>&1 - ''; - }; - }) - # Add checks to be able to build them individually - // (prefixNamesWith "checks/" checks); + exes = + (collectExes project) + // { + inherit (pkgs) checkCabalProject; + } + // flattenTree ( + pkgs.scripts + // { + # `tests` are the test suites which have been built. + inherit (project) tests; + # `benchmarks` (only built, not run). + inherit (project) benchmarks; + } + ); + + # The parameterisable workbench. + inherit (pkgs) workbench; - apps = - mapAttrs (n: p: { + packages = + exes + # Linux only packages: + // + optionalAttrs + (elem system [ + "x86_64-linux" + "aarch64-linux" + ]) + ( + let + workbenchTest = + { + profileName, + workbenchStartArgs ? [ ], + }: + (pkgs.workbench-runner { + inherit profileName workbenchStartArgs; + backendName = "supervisor"; + useCabalRun = false; + cardano-node-rev = pkgs.gitrev; + }).workbench-profile-run; + in + { + "dockerImage/node" = pkgs.dockerImage; + "dockerImage/submit-api" = pkgs.submitApiDockerImage; + "dockerImage/tracer" = pkgs.tracerDockerImage; + + # This is a very light profile, no caching and pinning needed. + workbench-ci-test = workbenchTest { + profileName = "ci-test-hydra-coay"; + workbenchStartArgs = [ "--create-testnet-data" ]; + }; + workbench-ci-test-trace = workbenchTest { + profileName = "ci-test-hydra-coay"; + workbenchStartArgs = [ + "--create-testnet-data" + "--trace" + ]; + }; + + inherit (pkgs) all-profiles-json profile-data-nomadperf; + + system-tests = pkgs.writeShellApplication { + name = "system-tests"; + runtimeInputs = with pkgs; [ + git + gnused + ]; + text = '' + NODE_REV="${self.rev or ""}" + if [[ -z $NODE_REV ]]; then + echo "Sorry, need clean/pushed git revision to run system tests" + exit 1; + fi + MAKE_TARGET=testpr + mkdir -p tmp && cd tmp + rm -rf cardano-node-tests + git clone https://github.com/intersectmbo/cardano-node-tests.git + cd cardano-node-tests + sed -i '1 s/^.*$/#! \/usr\/bin\/env bash/' ./.github/regression.sh + export NODE_REV + export MAKE_TARGET + nix develop --accept-flake-config .#base -c ./.github/regression.sh 2>&1 + ''; + }; + } + ) + # Add checks to be able to build them individually + // (prefixNamesWith "checks/" checks); + + apps = mapAttrs (n: p: { type = "app"; - program = - p.exePath - or ( - if (p.executable or false) - then "${p}" - else "${p}/bin/${p.name or n}" - ); - }) - exes; - - ciJobs = let - releaseBins = [ - "bech32" - "cardano-cli" - "cardano-node" - "cardano-submit-api" - "cardano-testnet" - "cardano-tracer" - "db-analyser" - "db-synthesizer" - "db-truncater" - # "dmq-node" FIXME: not updated within leios-prototype - "kes-agent" - "kes-agent-control" - "snapshot-converter" - "tx-generator" - ]; - - # Binaries only supported on Linux; excluded from Windows and Darwin releases. - linuxOnlyBins = ["kes-agent" "kes-agent-control"]; - - ciJobsVariants = - mapAttrs ( - _: p: - (mkFlakeAttrs (pkgs.extend (prev: final: {cardanoNodeProject = p;}))).ciJobs - ) - project.projectVariants; + program = p.exePath or (if (p.executable or false) then "${p}" else "${p}/bin/${p.name or n}"); + }) exes; ciJobs = - { - cardano-deployment = pkgs.cardanoLib.mkConfigHtml {inherit (pkgs.cardanoLib.environments) mainnet preview preprod;}; - } - // optionalAttrs (elem system ["x86_64-linux" "aarch64-linux"]) { - native = - packages - // { - shells = devShells; - internal = { - roots.project = project.roots; - plan-nix.project = project.plan-nix; - }; - profiled = genAttrs ["cardano-node" "tx-generator" "locli"] ( - n: - packages.${n}.passthru.profiled - ); - asserted = genAttrs ["cardano-node"] ( - n: - packages.${n}.passthru.asserted - ); - variants = mapAttrs (_: v: removeAttrs v.native ["variants"]) ciJobsVariants; + let + releaseBins = [ + "bech32" + "cardano-cli" + "cardano-node" + "cardano-submit-api" + "cardano-testnet" + "cardano-tracer" + "db-analyser" + "db-synthesizer" + "db-truncater" + # "dmq-node" FIXME: not updated within leios-prototype + "kes-agent" + "kes-agent-control" + "snapshot-converter" + "tx-generator" + ]; + + # Binaries only supported on Linux; excluded from Windows and Darwin releases. + linuxOnlyBins = [ + "kes-agent" + "kes-agent-control" + ]; + + ciJobsVariants = mapAttrs ( + _: p: (mkFlakeAttrs (pkgs.extend (prev: final: { cardanoNodeProject = p; }))).ciJobs + ) project.projectVariants; + + ciJobs = { + cardano-deployment = pkgs.cardanoLib.mkConfigHtml { + inherit (pkgs.cardanoLib.environments) mainnet preview preprod; }; - musl = let - muslProject = - if system == "x86_64-linux" - then project.projectCross.musl64 - else project.projectCross.aarch64-multiplatform-musl; - projectExes = collectExes muslProject; - in - projectExes - // { - cardano-node-linux = import ./nix/binary-release.nix { - inherit pkgs; - inherit (exes.cardano-node.identifier) version; - platform = "linux"; - exes = - collect isDerivation ( - filterAttrs (n: _: elem n releaseBins) projectExes - ) - ++ optional (system == "x86_64-linux") mithril.packages.${system}.mithril-signer; + } + // + optionalAttrs + (elem system [ + "x86_64-linux" + "aarch64-linux" + ]) + { + native = packages // { + shells = devShells; + internal = { + roots.project = project.roots; + plan-nix.project = project.plan-nix; + }; + profiled = genAttrs [ "cardano-node" "tx-generator" "locli" ] (n: packages.${n}.passthru.profiled); + asserted = genAttrs [ "cardano-node" ] (n: packages.${n}.passthru.asserted); + variants = mapAttrs (_: v: removeAttrs v.native [ "variants" ]) ciJobsVariants; + }; + musl = + let + muslProject = + if system == "x86_64-linux" then + project.projectCross.musl64 + else + project.projectCross.aarch64-multiplatform-musl; + projectExes = collectExes muslProject; + in + projectExes + // { + cardano-node-linux = import ./nix/binary-release.nix { + inherit pkgs; + inherit (exes.cardano-node.identifier) version; + platform = "linux"; + exes = + collect isDerivation (filterAttrs (n: _: elem n releaseBins) projectExes) + ++ optional (system == "x86_64-linux") mithril.packages.${system}.mithril-signer; + }; + internal.roots.project = muslProject.roots; + variants = mapAttrs (_: v: removeAttrs v.musl [ "variants" ]) ciJobsVariants; + }; + } + # Compiling windows on aarch64-linux requires aarch64 wine64 for TH code. + # Currently github:NixOS/nixpkgs/nixpkgs-unstable#legacyPackages.aarch64-linux.wine64 does not build. + # Once building, windowsProject candidate for win-arm64 is project.projectCross.ucrtAarch64. + // optionalAttrs (elem system [ "x86_64-linux" ]) { + windows = + let + windowsProject = + (project.appendModule { compiler-nix-name = windowsCompilerNixName; }).projectCross.ucrt64; + projectExes = collectExes windowsProject; + in + projectExes + // (removeRecurse { + inherit (windowsProject) checks tests benchmarks; + cardano-node-win = import ./nix/binary-release.nix { + inherit pkgs; + inherit (exes.cardano-node.identifier) version; + platform = "win"; + exes = collect isDerivation ( + filterAttrs (n: _: elem n releaseBins && !(elem n linuxOnlyBins)) projectExes + ); + }; + internal.roots.project = windowsProject.roots; + variants = mapAttrs (_: v: removeAttrs v.windows [ "variants" ]) ciJobsVariants; + }); + } + // + optionalAttrs + (elem system [ + "x86_64-darwin" + "aarch64-darwin" + ]) + { + native = + filterAttrs ( + n: _: + # Only build docker images once on linux: + !(hasPrefix "dockerImage" n) + ) packages + // { + cardano-node-macos = import ./nix/binary-release.nix { + inherit pkgs; + inherit (exes.cardano-node.identifier) version; + platform = "macos"; + exes = collect isDerivation ( + filterAttrs (n: _: elem n releaseBins && !(elem n linuxOnlyBins)) (collectExes project) + ); + }; + shells = removeAttrs devShells [ "profiled" ]; + internal = { + roots.project = project.roots; + plan-nix.project = project.plan-nix; + }; + variants = mapAttrs (_: v: removeAttrs v.native [ "variants" ]) ciJobsVariants; + }; }; - internal.roots.project = muslProject.roots; - variants = mapAttrs (_: v: removeAttrs v.musl ["variants"]) ciJobsVariants; - }; - } - # Compiling windows on aarch64-linux requires aarch64 wine64 for TH code. - # Currently github:NixOS/nixpkgs/nixpkgs-unstable#legacyPackages.aarch64-linux.wine64 does not build. - # Once building, windowsProject candidate for win-arm64 is project.projectCross.ucrtAarch64. - // optionalAttrs (elem system ["x86_64-linux"]) { - windows = let - windowsProject = (project.appendModule {compiler-nix-name = windowsCompilerNixName;}).projectCross.ucrt64; - projectExes = collectExes windowsProject; - in - projectExes - // (removeRecurse { - inherit (windowsProject) checks tests benchmarks; - cardano-node-win = import ./nix/binary-release.nix { - inherit pkgs; - inherit (exes.cardano-node.identifier) version; - platform = "win"; - exes = collect isDerivation ( - filterAttrs (n: _: elem n releaseBins && !(elem n linuxOnlyBins)) projectExes - ); - }; - internal.roots.project = windowsProject.roots; - variants = mapAttrs (_: v: removeAttrs v.windows ["variants"]) ciJobsVariants; - }); + + nonRequiredPaths = [ + # FIXME: cardano-tracer-test for windows should probably be disabled in haskell.nix config: + "windows\\.(.*\\.)?checks\\.cardano-tracer\\.cardano-tracer-test" + # hlint required status is controlled via the github action: + "native\\.(.*\\.)?checks/hlint" + # system-tests are build and run separately: + "native\\.(.*\\.)?system-tests" + ] + ++ optionals (system == "x86_64-darwin") [ + # FIXME: make variants nonrequired for macos until CI has more capacity for macos builds + "native\\.variants\\..*" + "native\\.checks/cardano-testnet/cardano-testnet-test" + ]; + in + pkgs.callPackages iohkNix.utils.ciJobsAggregates { + inherit ciJobs; + nonRequiredPaths = map (r: p: match r p != null) nonRequiredPaths; } - // optionalAttrs (elem system ["x86_64-darwin" "aarch64-darwin"]) { - native = - filterAttrs - (n: _: - # Only build docker images once on linux: - !(hasPrefix "dockerImage" n)) - packages - // { - cardano-node-macos = import ./nix/binary-release.nix { - inherit pkgs; - inherit (exes.cardano-node.identifier) version; - platform = "macos"; - exes = collect isDerivation ( - filterAttrs (n: _: elem n releaseBins && !(elem n linuxOnlyBins)) (collectExes project) - ); - }; - shells = removeAttrs devShells ["profiled"]; - internal = { - roots.project = project.roots; - plan-nix.project = project.plan-nix; - }; - variants = mapAttrs (_: v: removeAttrs v.native ["variants"]) ciJobsVariants; - }; - }; + // ciJobs; + }; - nonRequiredPaths = - [ - # FIXME: cardano-tracer-test for windows should probably be disabled in haskell.nix config: - "windows\\.(.*\\.)?checks\\.cardano-tracer\\.cardano-tracer-test" - # hlint required status is controlled via the github action: - "native\\.(.*\\.)?checks/hlint" - # system-tests are build and run separately: - "native\\.(.*\\.)?system-tests" - ] - ++ optionals (system == "x86_64-darwin") [ - # FIXME: make variants nonrequired for macos until CI has more capacity for macos builds - "native\\.variants\\..*" - "native\\.checks/cardano-testnet/cardano-testnet-test" - ]; - in - pkgs.callPackages iohkNix.utils.ciJobsAggregates + flake = eachSystem supportedSystems ( + system: + let + inherit (haskellNix) config; + pkgs = import nixpkgs { + inherit config system overlays; + }; + inherit (mkFlakeAttrs pkgs) + environments + packages + checks + apps + project + ciJobs + devShells + workbench + ; + in { - inherit ciJobs; - nonRequiredPaths = map (r: p: match r p != null) nonRequiredPaths; - } - // ciJobs; - }; - - flake = eachSystem supportedSystems ( - system: let - inherit (haskellNix) config; - pkgs = import nixpkgs { - inherit config system overlays; - }; - inherit (mkFlakeAttrs pkgs) environments packages checks apps project ciJobs devShells workbench; - in { - inherit environments checks project ciJobs devShells workbench; + inherit + environments + checks + project + ciJobs + devShells + workbench + ; - legacyPackages = - pkgs - // { + legacyPackages = pkgs // { # Allows access to hydraJobs without specifying : hydraJobs = ciJobs; }; - packages = - packages - // { + packages = packages // { # Built by `nix build .` default = packages.cardano-node; }; - # Run by `nix run .` - apps = - apps - // { + # Run by `nix run .` + apps = apps // { default = apps.cardano-node; }; - } - ); - in - removeAttrs flake ["ciJobs"] + } + ); + in + removeAttrs flake [ "ciJobs" ] // { hydraJobs = flake.ciJobs - // (let - pkgs = self.legacyPackages.${defaultSystem}; - in { - inherit - (pkgs.callPackages iohkNix.utils.ciJobsAggregates { - ciJobs = - mapAttrs (_: getAttr "required") flake.ciJobs - // { + // ( + let + pkgs = self.legacyPackages.${defaultSystem}; + in + { + inherit + (pkgs.callPackages iohkNix.utils.ciJobsAggregates { + ciJobs = mapAttrs (_: getAttr "required") flake.ciJobs // { # Ensure hydra notify: gitrev = pkgs.writeText "gitrev" pkgs.gitrev; }; - }) - required - ; - }); + }) + required + ; + } + ); # Allows precise paths (avoid fallbacks) with nix build/eval: outputs = self; @@ -512,13 +574,12 @@ inherit (final) haskell-nix; inherit CHaP incl windowsCompilerNixName; macOS-security = macOS-security (final.pkgs); - }) - .appendModule [ - customConfig.haskellNix - ]; + }).appendModule + [ + customConfig.haskellNix + ]; cardanoNodePackages = mkCardanoNodePackages final.cardanoNodeProject; - inherit - (final.cardanoNodePackages) + inherit (final.cardanoNodePackages) bech32 cardano-cli cardano-node @@ -534,30 +595,42 @@ ; }; nixosModules = { - cardano-node = { - pkgs, - lib, - ... - }: { - imports = [./nix/nixos/cardano-node-service.nix]; - services.cardano-node.cardanoNodePackages = lib.mkDefault (mkCardanoNodePackages flake.project.${pkgs.stdenv.hostPlatform.system}); - }; - cardano-submit-api = { - pkgs, - lib, - ... - }: { - imports = [./nix/nixos/cardano-submit-api-service.nix]; - services.cardano-submit-api.cardanoNodePackages = lib.mkDefault (mkCardanoNodePackages flake.project.${pkgs.stdenv.hostPlatform.system}); - }; - cardano-tracer = { - pkgs, - lib, - ... - }: { - imports = [./nix/nixos/cardano-tracer-service.nix]; - services.cardano-tracer.cardanoNodePackages = lib.mkDefault (mkCardanoNodePackages flake.project.${pkgs.stdenv.hostPlatform.system}); - }; + cardano-node = + { + pkgs, + lib, + ... + }: + { + imports = [ ./nix/nixos/cardano-node-service.nix ]; + services.cardano-node.cardanoNodePackages = lib.mkDefault ( + mkCardanoNodePackages flake.project.${pkgs.stdenv.hostPlatform.system} + ); + }; + cardano-submit-api = + { + pkgs, + lib, + ... + }: + { + imports = [ ./nix/nixos/cardano-submit-api-service.nix ]; + services.cardano-submit-api.cardanoNodePackages = lib.mkDefault ( + mkCardanoNodePackages flake.project.${pkgs.stdenv.hostPlatform.system} + ); + }; + cardano-tracer = + { + pkgs, + lib, + ... + }: + { + imports = [ ./nix/nixos/cardano-tracer-service.nix ]; + services.cardano-tracer.cardanoNodePackages = lib.mkDefault ( + mkCardanoNodePackages flake.project.${pkgs.stdenv.hostPlatform.system} + ); + }; }; }; }