From 1db69e84abde9e2589775d6ccd7dc0220fa3a6c4 Mon Sep 17 00:00:00 2001 From: Kosyrev Serge Date: Fri, 1 Nov 2019 17:35:44 +0300 Subject: [PATCH 1/4] crypto: add a nix shell --- crypto/shell.nix | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 crypto/shell.nix diff --git a/crypto/shell.nix b/crypto/shell.nix new file mode 100644 index 00000000000..9000e4f750e --- /dev/null +++ b/crypto/shell.nix @@ -0,0 +1,6 @@ +let + self = import ../. {}; +in (self.nix-tools.shellFor { + name = "cardano-sl-crypto"; + packages = ps: [ ps.cardano-sl-crypto ps.cabal-install ]; +}) From e9956551ee90efeae9209a60c345b947a4263582 Mon Sep 17 00:00:00 2001 From: Kosyrev Serge Date: Fri, 22 Nov 2019 14:30:58 +0300 Subject: [PATCH 2/4] network-transport-tcp: bump for the https://github.com/input-output-hk/cardano-node/issues/296 fix --- nix/.stack.nix/network-transport-tcp.nix | 4 ++-- pkgs/default.nix | 4 ++-- stack.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nix/.stack.nix/network-transport-tcp.nix b/nix/.stack.nix/network-transport-tcp.nix index a4f5a3f208c..a7772b3cf20 100644 --- a/nix/.stack.nix/network-transport-tcp.nix +++ b/nix/.stack.nix/network-transport-tcp.nix @@ -62,7 +62,7 @@ } // { src = (pkgs.lib).mkDefault (pkgs.fetchgit { url = "https://github.com/avieth/network-transport-tcp"; - rev = "2634e5e32178bb0456d800d133f8664321daa2ef"; - sha256 = "03shp9prflhgqzw7pymw1pq2q7s1wf46pyjm2csx8646snrhf35i"; + rev = "46f2942423d9ad3c81040565a9e9d5b9e08ddcc4"; + sha256 = "1nkj414whns56s7p9znn8qn43zp49q7pc5p0py3hdqcva50fd45d"; }); } \ No newline at end of file diff --git a/pkgs/default.nix b/pkgs/default.nix index d09927648f0..8d95ddd786f 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -56567,8 +56567,8 @@ version = "0.6.0"; src = fetchgit { url = "https://github.com/avieth/network-transport-tcp"; -sha256 = "03shp9prflhgqzw7pymw1pq2q7s1wf46pyjm2csx8646snrhf35i"; -rev = "2634e5e32178bb0456d800d133f8664321daa2ef"; +sha256 = "1nkj414whns56s7p9znn8qn43zp49q7pc5p0py3hdqcva50fd45d"; +rev = "46f2942423d9ad3c81040565a9e9d5b9e08ddcc4"; fetchSubmodules = true; }; diff --git a/stack.yaml b/stack.yaml index d0c26ad2ffa..d766309b2e3 100644 --- a/stack.yaml +++ b/stack.yaml @@ -95,7 +95,7 @@ packages: # As above, seems relatively straightforward to PR. - location: git: https://github.com/avieth/network-transport-tcp - commit: 2634e5e32178bb0456d800d133f8664321daa2ef # csl-0.6.0 + commit: 46f2942423d9ad3c81040565a9e9d5b9e08ddcc4 # csl-0.6.0 extra-dep: true # This one has more changes: From 93bfeb84813d560be0d8f0810ee27b727410c1e6 Mon Sep 17 00:00:00 2001 From: Alexander Vieth Date: Thu, 21 Nov 2019 22:41:29 -0500 Subject: [PATCH 3/4] remove network-transport-inmemory dependency --- networking/cardano-sl-networking.cabal | 2 - networking/src/Network/Discovery/Abstract.hs | 31 ----------- .../Network/Discovery/Transport/InMemory.hs | 55 ------------------- nix/.stack.nix/default.nix | 1 - nix/.stack.nix/network-transport-inmemory.nix | 53 ------------------ stack.yaml | 8 --- 6 files changed, 150 deletions(-) delete mode 100644 networking/src/Network/Discovery/Abstract.hs delete mode 100644 networking/src/Network/Discovery/Transport/InMemory.hs delete mode 100644 nix/.stack.nix/network-transport-inmemory.nix diff --git a/networking/cardano-sl-networking.cabal b/networking/cardano-sl-networking.cabal index c0ff45008de..ac0376b97f8 100644 --- a/networking/cardano-sl-networking.cabal +++ b/networking/cardano-sl-networking.cabal @@ -17,8 +17,6 @@ flag benchmarks Library exposed-modules: Network.QDisc.Fair - Network.Discovery.Abstract - Network.Broadcast.OutboundQueue Network.Broadcast.OutboundQueue.Types Network.Broadcast.OutboundQueue.ConcurrentMultiQueue diff --git a/networking/src/Network/Discovery/Abstract.hs b/networking/src/Network/Discovery/Abstract.hs deleted file mode 100644 index 1d7a22e694d..00000000000 --- a/networking/src/Network/Discovery/Abstract.hs +++ /dev/null @@ -1,31 +0,0 @@ -{-# LANGUAGE DeriveDataTypeable #-} -{-# LANGUAGE DeriveGeneric #-} - -module Network.Discovery.Abstract ( - - NetworkDiscovery(..) - , DiscoveryError(..) - - ) where - -import Control.Exception (Exception) -import Data.Set (Set) -import Data.Typeable (Typeable) -import GHC.Generics (Generic) -import Network.Transport (EndPointAddress) - --- Note: we'll need a way to be notified of new peers. - -data NetworkDiscovery err = NetworkDiscovery { - knownPeers :: IO (Set EndPointAddress) - , discoverPeers :: IO (Either (DiscoveryError err) (Set EndPointAddress)) - -- ^ The set is the newly discovered peers (disjoint from 'knownPeers' at - -- the time it's used). Subsequent uses of 'knownPeers' should include these - -- new endpoints. - , closeDiscovery :: IO () - } - -data DiscoveryError error = DiscoveryError error String - deriving (Show, Typeable, Generic) - -instance (Typeable error, Show error) => Exception (DiscoveryError error) diff --git a/networking/src/Network/Discovery/Transport/InMemory.hs b/networking/src/Network/Discovery/Transport/InMemory.hs deleted file mode 100644 index 6fdc999bb7b..00000000000 --- a/networking/src/Network/Discovery/Transport/InMemory.hs +++ /dev/null @@ -1,55 +0,0 @@ --- | This module contains in-memory implementation of Kademlia P2P-discovery. - -module Network.Discovery.Transport.InMemory - ( InMemoryDiscoveryErrorCode - , inMemoryDiscovery - ) where - -import qualified Control.Concurrent.STM as STM -import qualified Control.Concurrent.STM.TVar as TVar -import Control.Monad.IO.Class (MonadIO, liftIO) -import qualified Data.Map as M -import Data.Set (Set) -import qualified Data.Set as S -import Data.Typeable (Typeable) -import GHC.Generics (Generic) -import Network.Transport.InMemory -import Network.Transport.InMemory.Internal (TransportState (..), - ValidTransportState (..)) - -import Network.Discovery.Abstract -import Network.Transport - --- | An InMemory network transport can be used as a NetworkDiscovery. It takes --- addresses from the transport's internal state. -inMemoryDiscovery - :: ( MonadIO m ) - => TransportInternals - -> m (NetworkDiscovery InMemoryDiscoveryErrorCode m) -inMemoryDiscovery internals = do - peersTVar <- liftIO . TVar.newTVarIO $ S.empty - let knownPeers = liftIO . TVar.readTVarIO $ peersTVar - let discoverPeers = liftIO . STM.atomically $ do - mAllPeers <- getPeersFromInternals internals - case mAllPeers of - Nothing -> pure (Left (DiscoveryError InMemoryDiscoveryTransportClosed "Transport is closed")) - Just allPeers -> do - knownPeers <- TVar.readTVar peersTVar - let newPeers = allPeers `S.difference` knownPeers - TVar.writeTVar peersTVar allPeers - pure (Right newPeers) - -- No need to do anything on close, since we don't exactly use any - -- resources. - let close = pure () - pure $ NetworkDiscovery knownPeers discoverPeers close - -getPeersFromInternals :: TransportInternals -> STM.STM (Maybe (Set EndPointAddress)) -getPeersFromInternals (TransportInternals tvar) = do - state <- TVar.readTVar tvar - case state of - TransportValid (ValidTransportState map _) -> - pure . Just $ S.fromList (M.keys map) - TransportClosed -> pure Nothing - -data InMemoryDiscoveryErrorCode = InMemoryDiscoveryTransportClosed - deriving (Show, Typeable, Generic) diff --git a/nix/.stack.nix/default.nix b/nix/.stack.nix/default.nix index 0a2cf8956e5..7b5168b21a3 100644 --- a/nix/.stack.nix/default.nix +++ b/nix/.stack.nix/default.nix @@ -71,7 +71,6 @@ kademlia = ./kademlia.nix; network-transport = ./network-transport.nix; network-transport-tcp = ./network-transport-tcp.nix; - network-transport-inmemory = ./network-transport-inmemory.nix; acid-state = ./acid-state.nix; socket-io = ./socket-io.nix; engine-io = ./engine-io.nix; diff --git a/nix/.stack.nix/network-transport-inmemory.nix b/nix/.stack.nix/network-transport-inmemory.nix deleted file mode 100644 index 25fa6d08879..00000000000 --- a/nix/.stack.nix/network-transport-inmemory.nix +++ /dev/null @@ -1,53 +0,0 @@ -{ system, compiler, flags, pkgs, hsPkgs, pkgconfPkgs, ... }: - { - flags = {}; - package = { - specVersion = "1.8"; - identifier = { name = "network-transport-inmemory"; version = "0.5.1"; }; - license = "BSD-3-Clause"; - copyright = "Well-Typed LLP, Tweag I/O Limited"; - maintainer = "Facundo Domínguez "; - author = "Duncan Coutts, Nicolas Wu, Edsko de Vries, Alexander Vershilov"; - homepage = "http://haskell-distributed.github.com"; - url = ""; - synopsis = "In-memory instantiation of Network.Transport"; - description = "This is a transport implementation that could be used for local\ncommunication in the same address space (i.e. one process).\n\nIt could be used either for testing purposes or for local\ncommunication that require the network-transport semantics.\n\nNB: network-tranpsport-inmemory does not support cross-transport\ncommunication. All endpoints that want to comminicate should be\ncreated using the same transport."; - buildType = "Simple"; - }; - components = { - "library" = { - depends = [ - (hsPkgs.base) - (hsPkgs.network-transport) - (hsPkgs.data-accessor) - (hsPkgs.bytestring) - (hsPkgs.containers) - (hsPkgs.stm) - ]; - }; - tests = { - "TestMulticastInMemory" = { - depends = [ - (hsPkgs.base) - (hsPkgs.network-transport-inmemory) - (hsPkgs.network-transport) - (hsPkgs.network-transport-tests) - ]; - }; - "TestInMemory" = { - depends = [ - (hsPkgs.base) - (hsPkgs.network-transport-inmemory) - (hsPkgs.network-transport-tests) - (hsPkgs.network-transport) - ]; - }; - }; - }; - } // { - src = (pkgs.lib).mkDefault (pkgs.fetchgit { - url = "https://github.com/avieth/network-transport-inmemory"; - rev = "5d8ff2b07b9df35cf61329a3d975e2c8cf95c12a"; - sha256 = "0ak64rks3lk3kk5wyndrrk2swmd84h9diribzix305xwz1jhjj9w"; - }); - } \ No newline at end of file diff --git a/stack.yaml b/stack.yaml index d766309b2e3..0f17fe9b9d1 100644 --- a/stack.yaml +++ b/stack.yaml @@ -98,14 +98,6 @@ packages: commit: 46f2942423d9ad3c81040565a9e9d5b9e08ddcc4 # csl-0.6.0 extra-dep: true -# This one has more changes: -# https://github.com/haskell-distributed/network-transport-inmemory/compare/master...avieth:5d8ff2b07b9df35cf61329a3d975e2c8cf95c12a?expand=1 -# Do we have an issue that documents why we made these changes, so we can make a PR upstream? -- location: - git: https://github.com/avieth/network-transport-inmemory - commit: 5d8ff2b07b9df35cf61329a3d975e2c8cf95c12a - extra-dep: true - # Waiting on the space leak bug fix PR to be merged: # https://github.com/acid-state/acid-state/pull/104 - location: From d0cc2e6336e3c57771c775cafe831bc83db303d3 Mon Sep 17 00:00:00 2001 From: Samuel Leathers Date: Thu, 21 Nov 2019 11:55:31 -0500 Subject: [PATCH 4/4] pin haskell.nix and iohk-nix using niv --- lib.nix | 33 +++++------ nix/sources.json | 62 ++++++++++++++++++++ nix/sources.nix | 130 +++++++++++++++++++++++++++++++++++++++++ nix/update-iohk-nix.sh | 10 +--- shell.nix | 23 ++++++++ 5 files changed, 230 insertions(+), 28 deletions(-) create mode 100644 nix/sources.json create mode 100644 nix/sources.nix create mode 100644 shell.nix diff --git a/lib.nix b/lib.nix index 8b6e910a6de..77299adf3ea 100644 --- a/lib.nix +++ b/lib.nix @@ -1,21 +1,16 @@ let - # Allow overriding pinned nixpkgs for debugging purposes via cardano_pkgs - # Imports the iohk-nix library. - # The version can be overridden for debugging purposes by setting - # NIX_PATH=iohk_nix=/path/to/iohk-nix - iohkNix = import ( - let try = builtins.tryEval ; - in if try.success - then builtins.trace "using host " try.value - else - let - spec = builtins.fromJSON (builtins.readFile ./nix/iohk-nix-src.json); - in builtins.fetchTarball { - url = "${spec.url}/archive/${spec.rev}.tar.gz"; - inherit (spec) sha256; - }) {}; - # Gets the value of an environment variable, with a default if it's - # unset or empty. + sources = import ./nix/sources.nix; + pkgs' = import sources.nixpkgs {}; + haskellNixJson = let + src = sources."haskell.nix"; + in __toJSON { + inherit (sources."haskell.nix") rev sha256; + url = "https://github.com/${src.owner}/${src.repo}"; + }; + iohkNix = import sources.iohk-nix { haskellNixJsonOverride = pkgs'.writeText "haskell-nix.json" haskellNixJson; }; + pkgs = iohkNix.pkgs; + lib = pkgs.lib; + niv = (import sources.niv {}).niv; maybeEnv = env: default: let result = builtins.getEnv env; @@ -23,8 +18,6 @@ let then result else default; - pkgs = iohkNix.pkgs; - lib = pkgs.lib; environments = iohkNix.cardanoLib.environments // { demo = { confKey = "dev"; @@ -36,7 +29,7 @@ let (name: env: f (env // { inherit name; })) environments; in lib // iohkNix.cardanoLib // iohkNix // { - inherit environments forEnvironments; + inherit environments forEnvironments niv iohkNix; utf8LocaleSetting = '' export LC_ALL=en_GB.UTF-8 export LANG=en_GB.UTF-8 diff --git a/nix/sources.json b/nix/sources.json new file mode 100644 index 00000000000..0484af116db --- /dev/null +++ b/nix/sources.json @@ -0,0 +1,62 @@ +{ + "gitignore": { + "branch": "master", + "description": "Nix function for filtering local git sources", + "homepage": "", + "owner": "hercules-ci", + "repo": "gitignore", + "rev": "f9e996052b5af4032fe6150bba4a6fe4f7b9d698", + "sha256": "0jrh5ghisaqdd0vldbywags20m2cxpkbbk5jjjmwaw0gr8nhsafv", + "type": "tarball", + "url": "https://github.com/hercules-ci/gitignore/archive/f9e996052b5af4032fe6150bba4a6fe4f7b9d698.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "haskell.nix": { + "branch": "master", + "description": "Alternative Haskell Infrastructure for Nixpkgs", + "homepage": "https://input-output-hk.github.io/haskell.nix", + "owner": "input-output-hk", + "repo": "haskell.nix", + "rev": "3ae2303b14bdebd00437b0e995af2d0eb21abafe", + "sha256": "0dk6541kwfyc5sp5h1vfk6zl4ghln4i2zqkaijbjwv8bnixnm6wl", + "type": "tarball", + "url": "https://github.com/input-output-hk/haskell.nix/archive/3ae2303b14bdebd00437b0e995af2d0eb21abafe.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "iohk-nix": { + "branch": "master", + "description": "nix scripts shared across projects", + "homepage": null, + "owner": "input-output-hk", + "repo": "iohk-nix", + "rev": "f854c957a65295639d2062867552b65c7dc227a6", + "sha256": "1jl07ng2nx6lgj9rrqzl242r263505llzzxb592fn81h0dq6jx1r", + "type": "tarball", + "url": "https://github.com/input-output-hk/iohk-nix/archive/f854c957a65295639d2062867552b65c7dc227a6.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "niv": { + "branch": "iohk", + "description": "Easy dependency management for Nix projects", + "homepage": "https://github.com/nmattia/niv", + "owner": "input-output-hk", + "repo": "niv", + "rev": "c25268460b72cea5b7294354ab019997530a6f3d", + "sha256": "1fgq1nkycm54f431h6wflh6rr7xsq3q2fq9drbgra2fnrsyn65ll", + "type": "tarball", + "url": "https://github.com/input-output-hk/niv/archive/c25268460b72cea5b7294354ab019997530a6f3d.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs": { + "branch": "iohk-19.03", + "description": "Nix Packages collection", + "homepage": null, + "owner": "input-output-hk", + "repo": "nixpkgs", + "rev": "5bd4c4ea6804b54683ef2b802d970ae473e1b658", + "sha256": "12gah6y1lkkgvvdgswyzsqyvxvsvljn61vpy606sy37yfgkc1a63", + "type": "tarball", + "url": "https://github.com/input-output-hk/nixpkgs/archive/5bd4c4ea6804b54683ef2b802d970ae473e1b658.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + } +} diff --git a/nix/sources.nix b/nix/sources.nix new file mode 100644 index 00000000000..b7be13a8a7e --- /dev/null +++ b/nix/sources.nix @@ -0,0 +1,130 @@ +# This file has been generated by Niv. + +# A record, from name to path, of the third-party packages +with rec +{ + pkgs = + if hasNixpkgsPath + then + if hasThisAsNixpkgsPath + then import (builtins_fetchTarball { inherit (sources_nixpkgs) url sha256; }) {} + else import {} + else + import (builtins_fetchTarball { inherit (sources_nixpkgs) url sha256; }) {}; + + sources_nixpkgs = + if builtins.hasAttr "nixpkgs" sources + then sources.nixpkgs + else abort + '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; + + sources_gitignore = + if builtins.hasAttr "gitignore" sources + then sources.gitignore + else abort + '' + Please add "gitignore" to your sources.json: + niv add hercules-ci/gitignore + ''; + + inherit (import (builtins_fetchTarball { inherit (sources_gitignore) url sha256; }) { + inherit (pkgs) lib; + }) gitignoreSource; + + # fetchTarball version that is compatible between all the versions of Nix + builtins_fetchTarball = + { url, sha256 ? null }@attrs: + let + inherit (builtins) lessThan nixVersion fetchTarball; + in + if sha256 == null || lessThan nixVersion "1.12" then + fetchTarball { inherit url; } + else + fetchTarball attrs; + + # fetchurl version that is compatible between all the versions of Nix + builtins_fetchurl = + { url, sha256 ? null }@attrs: + let + inherit (builtins) lessThan nixVersion fetchurl; + in + if sha256 == null || lessThan nixVersion "1.12" then + fetchurl { inherit url; } + else + fetchurl attrs; + + # A wrapper around pkgs.fetchzip that has inspectable arguments, + # annoyingly this means we have to specify them + fetchzip = { url, sha256 ? null }@attrs: if sha256 == null + then builtins.fetchTarball { inherit url; } + else pkgs.fetchzip attrs; + + # A wrapper around pkgs.fetchurl that has inspectable arguments, + # annoyingly this means we have to specify them + fetchurl = { url, sha256 }@attrs: pkgs.fetchurl attrs; + + hasNixpkgsPath = (builtins.tryEval ).success; + hasThisAsNixpkgsPath = + (builtins.tryEval ).success && == ./.; + + sources = builtins.fromJSON (builtins.readFile ./sources.json); + + mapAttrs = builtins.mapAttrs or + (f: set: with builtins; + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))); + + # borrowed from nixpkgs + functionArgs = f: f.__functionArgs or (builtins.functionArgs f); + callFunctionWith = autoArgs: f: args: + let auto = builtins.intersectAttrs (functionArgs f) autoArgs; + in f (auto // args); + + getFetcher = spec: + let fetcherName = + if builtins.hasAttr "type" spec + then builtins.getAttr "type" spec + else "builtin-tarball"; + in builtins.getAttr fetcherName { + "tarball" = fetchzip; + "builtin-tarball" = builtins_fetchTarball; + "file" = fetchurl; + "builtin-url" = builtins_fetchurl; + }; +}; +# NOTE: spec must _not_ have an "outPath" attribute +mapAttrs (name: spec: + if builtins.hasAttr "outPath" spec + then abort + "The values in sources.json should not have an 'outPath' attribute" + else + let + host = if (name == "nixpkgs") then "custom_nixpkgs" else name; + tryFromPath = builtins.tryEval (builtins.findFile builtins.nixPath host); + defaultSpec = (if builtins.hasAttr "url" spec && builtins.hasAttr "sha256" spec + then spec // + { outPath = callFunctionWith spec (getFetcher spec) { }; } + else spec) // (if tryFromPath.success + then let path = tryFromPath.value; + in { + outPath = builtins.trace "using search host <${host}>" ( + if pkgs.lib.hasPrefix "/nix/store" (builtins.toString path) + then path else gitignoreSource path); + } + else {}); + in if builtins.hasAttr "rev" spec && builtins.hasAttr "url" spec then + defaultSpec // + { revOverride = rev: if (rev == null) then defaultSpec else + let + spec' = removeAttrs (spec // { + rev = rev; + url = builtins.replaceStrings [defaultSpec.rev] [rev] defaultSpec.url; + }) [ "sha256" ]; + in + spec' // + { outPath = callFunctionWith spec' (getFetcher spec') { }; }; + } + else defaultSpec + ) sources diff --git a/nix/update-iohk-nix.sh b/nix/update-iohk-nix.sh index 6ed58247028..9cbbe911233 100755 --- a/nix/update-iohk-nix.sh +++ b/nix/update-iohk-nix.sh @@ -1,9 +1,3 @@ #!/usr/bin/env nix-shell -#!nix-shell -i bash -p nix-prefetch-git - -set -euo pipefail - -NIX_DIR=$(dirname "$0") - -nix-prefetch-git https://github.com/input-output-hk/iohk-nix refs/heads/haskell-nix-cardano-sl \ - > "$NIX_DIR/iohk-nix-src.json" +#!nix-shell -A devops ../shell.nix -i bash +niv update iohk-nix diff --git a/shell.nix b/shell.nix new file mode 100644 index 00000000000..90414f26062 --- /dev/null +++ b/shell.nix @@ -0,0 +1,23 @@ +{ localLib ? import ./lib.nix +}: +let + pkgs = localLib.iohkNix.pkgs; + default = import ./default.nix {}; + devops = pkgs.stdenv.mkDerivation { + name = "devops-shell"; + buildInputs = [ + localLib.niv + ]; + shellHook = '' + echo "DevOps Tools" \ + | ${pkgs.figlet}/bin/figlet -f banner -c \ + | ${pkgs.lolcat}/bin/lolcat + + echo "NOTE: you may need to export GITHUB_TOKEN if you hit rate limits with niv" + echo "Commands: + * niv update - update package + + " + ''; + }; +in { inherit devops; }