Merge branch 'release/2.0.0-master-pr-resolve-conflicts' into master-…

…release-merge

* release/2.0.0-master-pr-resolve-conflicts: (1333 commits)
  [DEVOPS-1112] Bump staging applicationVersion 17 → 18
  Add entry for CBR-495 & CBR-496 to 2.0.0 CHANGELOG
  [CBR-495] Make UndisputableLookupFailed lenient, since we may end up with no Txs-Accounts relations in SQlite
  Add missing entries for RCD-44 to RCD-47
  [CBR-496] Fix --rebuild-db not rebuilding sqlite
  [DEVOPS-1112] Add more ChangeLog entries
  Clean up transaction metadata after deleting an account or wallet
  bump last known block version for staging
  [DEVOPS-1112] configuration.yaml: staging applicationVersion 16 -> 17
  Revert "applicationVersion | staging:  17 <- 16"
  Revert "configuration | staging:  factor a little"
  [DEVOPS-1172] Regenerate pkgs/default.nix and stylish haskell
  [DEVOPS-1172] faucet: Use OptimizeForHighThroughput grouping policy
  RCD-43: Clear failure on win update
  applicationVersion | testnet:  4 <- 3
  applicationVersion | staging:  17 <- 16
  configuration | staging:  factor a little
  cleanup tmp on every buildkite windows build
  [RCD-52] launcher:  -Wno-type-defaults locally
  [RCD-52]  launcher | windows:  fix CPP
  ...

.buildkite/hooks/pre-command

@@ -0,0 +1,4 @@
+set -x
+
+NIX_PATH="nixpkgs=$(nix-build ./nix/fetch-nixpkgs.nix -o nixpkgs)"
+export NIX_PATH

.buildkite/nightly.yml

@@ -0,0 +1,14 @@
+steps:
+  - label: 'Acceptance Tests - mainnet - full'
+    command:
+      - nix-build -A acceptanceTests.mainnet.full -o acceptance-tests-mainnet-full.sh
+      - echo "+++ Syncing mainnet blockchain"
+      - ./acceptance-tests-mainnet-full.sh
+    timeout_in_minutes: 120
+
+  - label: 'Acceptance Tests - testnet - full'
+    command:
+      - nix-build -A acceptanceTests.testnet.full -o acceptance-tests-testnet-full.sh
+      - echo "+++ Syncing testnet blockchain"
+      - ./acceptance-tests-testnet-full.sh
+    timeout_in_minutes: 60

.buildkite/pipeline.yml

@@ -1,35 +1,25 @@
 steps:
-  - label: 'hlint'
-    command: 'scripts/ci/nix-shell.sh -p hlint moreutils --run scripts/haskell/lint.sh'
-    agents:
-      system: x86_64-linux
-
   - label: 'Windows'
     command: 'PowerShell.exe -executionpolicy remotesigned -File .\.buildkite\windows.ps1'
     agents:
       system: x86_64-win2016
 
   - label: 'stack2nix'
-    command: 'scripts/ci/nix-shell.sh -p cabal2nix stack cabal-install ghc moreutils expect -Q -j 4 --run scripts/check-stack2nix.sh'
+    command: 'nix-shell -Q -j 4 --run scripts/check-stack2nix.sh'
     agents:
       system: x86_64-linux
 
-  - label: 'cardano-x86_64-linux'
-    command: 'scripts/ci/nix-shell.sh --run scripts/ci/ci.sh'
+  - label: 'release.nix'
+    command: 'nix-shell --run scripts/ci/check-hydra.sh'
     agents:
       system: x86_64-linux
 
-  - label: 'cardano-x86_64-darwin'
-    command: 'scripts/ci/nix-shell.sh --run scripts/ci/ci.sh'
-    agents:
-      system: x86_64-darwin
-
-  - label: 'release.nix'
-    command: 'scripts/ci/nix-shell.sh --run scripts/ci/check-hydra.sh'
+  - label: 'explorer frontend: checking generated dependencies'
+    command: "cd explorer/frontend && ./nix/generate-frontend-deps.hs --test"
     agents:
       system: x86_64-linux
 
-  - label: 'explorer frontend: checking generated dependencies'
-    command: 'scripts/ci/nix-shell.sh --run "cd explorer/frontend && ./nix/generate-frontend-deps.hs --test"'
+  - label: 'triggerShutdown test'
+    command: "rm -rf test-state ; nix-shell node-ipc/shell.nix --run 'node node-ipc/server.js'"
     agents:
       system: x86_64-linux

.buildkite/windows.ps1

@@ -1,79 +1,141 @@
-Set-PSDebug -Trace 1
+############################################################################
+# Windows build script for Buildkite
+#
+# Requires the devops buildkite-windows AMI
+############################################################################
+
+#Set-PSDebug -Trace 1
+
+Write-Output "--- Prepare work directory"
 
 # Avoid long paths on Windows
-$env:STACK_ROOT="D:\s"
+$env:STACK_ROOT="Z:\s"
 $env:STACK_WORK=".w"
-$env:WORK_DIR="D:\w"
+$env:WORK_DIR="Z:\w"
 # Override the temp directory to avoid sed escaping issues
 # See https://github.com/haskell/cabal/issues/5386
-$env:TMP="D:\\tmp"
+$env:TMP="Z:\\tmp"
 
 # Store the original checkout directory
 $env:CHECKOUT_PATH=(Get-Item -Path ".\").FullName
 
 # Temp directory needs to exist
+rd -r -fo $env:TMP
 New-Item -ItemType Directory -Force -Path $env:TMP
 
-#  Setup stuff (commented out)
-#mkdir d:\ghc
-#
-#Invoke-WebRequest "https://s3.eu-central-1.amazonaws.com/ci-static/ghc-8.2.2-x86_64-unknown-mingw32.tar.xz" -OutFile "D:\ghc\ghc.tar.xz" -UserAgent "Curl"
-#
-#7z x D:\ghc\ghc.tar.xz -oD:\ghc
+New-Item -ItemType Directory -Force -Path z:\ghc
+
+New-Item -ItemType Directory -Force -Path z:\Downloads
+New-Item -ItemType Directory -Force -Path $env:STACK_ROOT
+
+$StackConfig = @"
+templates:
+  params: null
+system-ghc: true
+local-programs-path: "z:\\s\\programs"
+local-bin-path: "z:\\s\\bin"
+"@
+
+$StackConfig | Out-File -FilePath $env:STACK_ROOT\config.yaml -Encoding ASCII
+
 #
-#7z x D:\ghc\ghc.tar -oD:\ghc
+if (!([System.IO.File]::Exists("Z:\ghc\ghc-8.2.2.tar.xz"))) {
+  echo "Downloading and extracting GHC"
+  curl.exe https://s3.eu-central-1.amazonaws.com/ci-static/ghc-8.2.2-x86_64-unknown-mingw32.tar.xz -o Z:\ghc\ghc-8.2.2.tar.xz -L
+  7z x Z:\ghc\ghc-8.2.2.tar.xz -oZ:\ghc
+  7z x Z:\ghc\ghc-8.2.2.tar -oZ:\ghc
+}
 
 # OpenSSL
 #
 #$env:USERPROFILE
-#(New-Object Net.WebClient).DownloadFile('https://slproweb.com/download/Win64OpenSSL-1_0_2p.exe', "D:\Downloads\Win64OpenSSL.exe")
-#cmd /c start /wait "D:\Downloads\Win64OpenSSL.exe" /silent /verysilent /sp- /suppressmsgboxes /DIR=D:\OpenSSL-Win64-v102
+if (!([System.IO.File]::Exists("Z:\Downloads\Win64OpenSSL-1_0_2q.exe"))) {
+  echo "Downloading and installing OpenSSL"
+  rd -r -fo z:\OpenSSL-Win64-v102
+  curl.exe https://s3.eu-central-1.amazonaws.com/ci-static/Win64OpenSSL-1_0_2q.exe -o Z:\Downloads\Win64OpenSSL-1_0_2q.exe -L
+  cmd /c start /wait "Z:\Downloads\Win64OpenSSL-1_0_2q.exe" /silent /verysilent /sp- /suppressmsgboxes /DIR=Z:\OpenSSL-Win64-v102
+}
 ## Install stack
-#Start-FileDownload http://www.stackage.org/stack/windows-x86_64 -FileName D:\Downloads\stack.zip
-#curl.exe http://www.stackage.org/stack/windows-x86_64 -o d:\Downloads\stack.zip -L
-#7z -oD:\stack x D:\Downloads\stack.zip
+if (!([System.IO.File]::Exists("Z:\Downloads\stack-1.9.1-windows-x86_64.zip"))) {
+  echo "Downloading and extracting stack"
+  rd -r -fo z:\stack
+  curl.exe https://s3.eu-central-1.amazonaws.com/ci-static/stack-1.9.1-windows-x86_64.zip -o z:\Downloads\stack-1.9.1-windows-x86_64.zip -L
+  7z -oZ:\stack x Z:\Downloads\stack-1.9.1-windows-x86_64.zip
+}
 
-$env:PATH="$env:PATH;D:\ghc\ghc-8.2.2\bin;D:\stack;$Env:Programfiles\7-Zip;D:\w"
+$env:PATH="$env:PATH;Z:\ghc\ghc-8.2.2\bin;Z:\stack;Z:\w"
 
 # Install liblzma/xz
-#curl -L https://tukaani.org/xz/xz-5.2.3-windows.zip -o xz-5.2.3-windows.zip
-#7z -oD:\xz_extracted x xz-5.2.3-windows.zip
-
+if (!([System.IO.File]::Exists("Z:\Downloads\xz-5.2.3-windows.zip"))) {
+  echo "Downloading and extracting xz"
+  rd -r -fo z:\xz_extracted
+  curl.exe -L https://s3.eu-central-1.amazonaws.com/ci-static/xz-5.2.3-windows.zip -o Z:\Downloads\xz-5.2.3-windows.zip
+  7z -oZ:\xz_extracted x Z:\Downloads\xz-5.2.3-windows.zip
+}
+
+# Clear out work directory, copy source tree there, then cd into it.
 rd -r -fo $env:WORK_DIR
 mkdir $env:WORK_DIR
 copy-item $env:CHECKOUT_PATH\* $env:WORK_DIR -force -recurse
 cd $env:WORK_DIR
 
+# Install pre-built patched version of rocksdb. The git repo provides
+# headers and the zip provides rocksdb.dll. This is needed to build
+# rocksdb-haskell.
 git.exe clone https://github.com/facebook/rocksdb.git --branch v4.13.5
-# curl.exe -L 'https://s3.eu-central-1.amazonaws.com/ci-static/serokell-rocksdb-haskell-325427fc709183c8fdf777ad5ea09f8d92bf8585.zip' -o D:\Downloads\rocksdb.zip
-7z x D:\Downloads\rocksdb.zip
+if (!([System.IO.File]::Exists("Z:\Downloads\rocksdb-325427fc709183c8fdf777ad5ea09f8d92bf8585.zip"))) {
+  echo "Downloading rocksdb"
+  curl.exe -L 'https://s3.eu-central-1.amazonaws.com/ci-static/serokell-rocksdb-haskell-325427fc709183c8fdf777ad5ea09f8d92bf8585.zip' -o Z:\Downloads\rocksdb-325427fc709183c8fdf777ad5ea09f8d92bf8585.zip
+}
+7z x Z:\Downloads\rocksdb-325427fc709183c8fdf777ad5ea09f8d92bf8585.zip
 
 # CSL-1509: After moving the 'cardano-sl' project itself into a separate folder ('lib/'), the 'cardano-text.exe' executable fails on AppVeyor CI.
 # After some investigation, it was discovered that this was because 'rocksdb.dll' has to be located in this folder as well, or else the test executable doesn't work.
 copy rocksdb.dll node
 copy rocksdb.dll lib
 copy rocksdb.dll wallet-new
 
-# Start doing stuff
+# Terminate on all errors
+$ErrorActionPreference = "Stop"
+
+############################################################################
+# Prepare stack and build tools
+#
+
+Write-Output "--- Stack setup and install tools"
 
+# Make stack use a pached system-ghc. This will allow us to work around the command
+# line argument limit on windows by having GHC copy all libraries into a common folder
+# prior to linking; and thus reduces the number library search patch flags passed on the
+# command line.
+#
+# WARNING: if you bump the LTS, make sure you have the right (patched) GHC version in path!
 stack.exe config --system-ghc set system-ghc --global true
-# stack.exe config --system-ghc set programs c:\s\programs --global true
+
 stack.exe path
 stack.exe exec -- ghc-pkg recache
 stack.exe --verbosity warn setup --no-reinstall
+
 # Install happy separately: https://github.com/commercialhaskell/stack/issues/3151#issuecomment-310642487. Also install cpphs because it's a build-tool and Stack can't figure out by itself that it should be installed
-stack.exe --verbosity warn install happy cpphs -j 2 --no-terminal --local-bin-path $env:SYSTEMROOT\system32 --extra-include-dirs="D:\OpenSSL-Win64-v102\include" --extra-lib-dirs="D:\OpenSSL-Win64-v102" --extra-include-dirs="D:\xz_extracted\include" --extra-lib-dirs="D:\xz_extracted\bin_x86-64" --extra-include-dirs="$env:WORK_DIR\rocksdb\include" --extra-lib-dirs="$env:WORK_DIR"
-# TODO: CSL-1133. To be reenabled.
-# stack.exe test --coverage
-# stack.exe hpc report cardano-sl cardano-sl-txp cardano-sl-core cardano-sl-db cardano-sl-update cardano-sl-infra cardano-sl-lrc cardano-sl-ssc
+stack.exe --verbosity warn install happy cpphs -j 2 --no-terminal --local-bin-path $env:SYSTEMROOT\system32 --extra-include-dirs="Z:\OpenSSL-Win64-v102\include" --extra-lib-dirs="Z:\OpenSSL-Win64-v102" --extra-include-dirs="Z:\xz_extracted\include" --extra-lib-dirs="Z:\xz_extracted\bin_x86-64" --extra-include-dirs="$env:WORK_DIR\rocksdb\include" --extra-lib-dirs="$env:WORK_DIR"
+
+############################################################################
+# Build and test.
+#
+# We intentionally don't build auxx here, because this build is for installer.
+#
 
-#We intentionally don't build auxx here, because this build is for installer.
-stack.exe --dump-logs install cardano-sl cardano-sl-tools cardano-sl-wallet-new -j 3 --no-terminal --local-bin-path $env:WORK_DIR --no-haddock-deps --flag cardano-sl-core:-asserts --flag cardano-sl-tools:for-installer --extra-include-dirs="D:\OpenSSL-Win64-v102\include" --extra-lib-dirs="D:\OpenSSL-Win64-v102" --extra-include-dirs="D:\xz_extracted\include" --extra-lib-dirs="D:\xz_extracted\bin_x86-64" --extra-include-dirs="$env:WORK_DIR\rocksdb\include" --extra-lib-dirs="$env:WORK_DIR"
+Write-Output "+++ Stack build and test"
 
-# from here onwards, errors terminate the script
-$ErrorActionPreference = "Stop"
+stack.exe --dump-logs install cardano-sl cardano-sl-tools cardano-sl-wallet-new -j 3 --no-terminal --local-bin-path $env:WORK_DIR --no-haddock-deps --flag cardano-sl-core:-asserts --flag cardano-sl-tools:for-installer --extra-include-dirs="Z:\OpenSSL-Win64-v102\include" --extra-lib-dirs="Z:\OpenSSL-Win64-v102" --extra-include-dirs="Z:\xz_extracted\include" --extra-lib-dirs="Z:\xz_extracted\bin_x86-64" --extra-include-dirs="$env:WORK_DIR\rocksdb\include" --extra-lib-dirs="$env:WORK_DIR"
+
+############################################################################
+# Assemble artifact zip file for use by the Daedalus installers build
+#
+
+Write-Output "--- Create zip file"
 
-#Cardano pieces, modulo the frontend
+# Cardano pieces, modulo the frontend
 mkdir daedalus
 ## log config is called `log-config-prod.yaml` just in case, it's the old name
 copy log-configs\daedalus.yaml daedalus\log-config-prod.yaml
@@ -83,8 +145,9 @@ copy cardano-launcher.exe daedalus\
 copy cardano-node.exe daedalus\
 copy cardano-x509-certificates.exe daedalus\
 cd daedalus
-$env:BUILDKITE_BUILD_NUMBER | Out-File build-id
-$env:BUILDKITE_COMMIT | Out-File commit-id
+$env:BUILDKITE_BUILD_NUMBER | Out-File -Encoding ASCII build-id
+$env:BUILDKITE_COMMIT | Out-File -Encoding ASCII commit-id
+$env:BUILDKITE_BUILD_URL | Out-File -Encoding ASCII ci-url
 cd ..
 
 $daedaluszip = "$env:BUILDKITE_COMMIT.zip"

.gitignore

@@ -1,10 +1,18 @@
 # Cabal & Stack
 .stack-work/
+.stack-work-profile/
 */dist
+dist-newstyle/
+.ghc.environment.*
+cabal.project.local
 
 # From daedalus-bridge
 node_modules/*
 
+# tags
+*.tags
+tags
+
 # Config files
 .ghci
 
@@ -17,7 +25,8 @@ b
 # Various DBs
 node-db*
 wallet-db/
-wallet-new-db/
+wallet-db-acid/
+wallet-db-sqlite.sqlite3
 db-*/
 wdb-*/
 
@@ -64,11 +73,11 @@ scripts/haskell/dependencies.hi
 scripts/haskell/dependencies.o
 scripts/haskell/dependencies
 
-# 'pkgs/stack2nix' is a symlink into the nix store, it can safely be ignored
-pkgs/stack2nix
+# Cache for pkgs/generate.sh
+/pkgs/.cabal/
+/pkgs/.stack/
 # in case generate.sh clones nixpkgs in here
 nixpkgs
-pkgs/result
 
 # explorer
 explorer-node.log.*
@@ -207,6 +216,14 @@ venv.bak/
 .mypy_cache/
 
 # End of https://www.gitignore.io/api/python
+*.aux
+/docs/on-the-wire/binary.toc
+/docs/on-the-wire/binary.synctex.gz
+/docs/on-the-wire/binary.pdf
+/docs/on-the-wire/binary.log
+/docs/on-the-wire/binary.fls
+/docs/on-the-wire/binary.fdb_latexmk
+/docs/on-the-wire/binary.dvi
 
 # Don't include config files with sensitive data or tls certs
 /faucet/test-config.json

.stylish-haskell.yaml

@@ -63,7 +63,7 @@ steps:
       #   >     (concat, foldl, foldr, head, init, last, length)
       #
       # Default: after_alias
-      list_align: after_alias
+      list_align: with_module_name
 
       # Right-pad the module names to align imports in a group:
       #
@@ -194,11 +194,7 @@ steps:
 
 # A common setting is the number of columns (parts of) code will be wrapped
 # to. Different steps take this into account. Default: 80.
-#
-# Note: we tend to write code which fits into 80 characters, in some cases
-# 100 is acceptable. For imports we always permit 100 characters because it
-# decreases verbosity of diffs and makes merging easier.
-columns: 100
+columns: 80
 
 # By default, line endings are converted according to the OS. You can override
 # preferred format here.
@@ -226,6 +222,7 @@ language_extensions:
   - DefaultSignatures
   - DeriveDataTypeable
   - DeriveGeneric
+  - ExistentialQuantification
   - FlexibleContexts
   - FlexibleInstances
   - FunctionalDependencies
@@ -245,3 +242,4 @@ language_extensions:
   - TypeApplications
   - TypeFamilies
   - ViewPatterns
+  - ExplicitNamespaces