Skip to content

Commit

Permalink
refactor CampaignConf and Campaign (#384)
Browse files Browse the repository at this point in the history
  • Loading branch information
incertia authored Apr 15, 2020
1 parent 53cd315 commit bc25142
Show file tree
Hide file tree
Showing 10 changed files with 170 additions and 151 deletions.
141 changes: 26 additions & 115 deletions lib/Echidna/Campaign.hs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
{-# LANGUAGE PatternSynonyms #-}
{-# LANGUAGE RankNTypes #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE TemplateHaskell #-}
{-# LANGUAGE TupleSections #-}
{-# LANGUAGE ViewPatterns #-}

Expand All @@ -20,130 +19,43 @@ import Control.Monad.Reader.Class (MonadReader)
import Control.Monad.State.Strict (MonadState(..), StateT(..), evalStateT, execStateT)
import Control.Monad.Trans (lift)
import Control.Monad.Trans.Random.Strict (liftCatch)
import Data.Aeson (ToJSON(..), object)
import Data.Binary.Get (runGetOrFail)
import Data.Bool (bool)
import Data.Map (Map, mapWithKey, mapKeys, unionWith, toList, (\\), keys, lookup, insert)
import Data.Maybe (fromMaybe, isJust, mapMaybe, maybeToList)
import Data.Map (Map, unionWith, (\\), keys, lookup, insert, mapWithKey)
import Data.Maybe (fromMaybe, isJust, mapMaybe)
import Data.Ord (comparing)
import Data.Has (Has(..))
import Data.Text (Text)
import Data.Traversable (traverse)
import EVM
import EVM.ABI (getAbi, AbiType(AbiAddressType), AbiValue(AbiAddress))
import EVM.Types (Addr)
import EVM.Keccak (keccak)
import Numeric (showHex)
import System.Random (mkStdGen)

import qualified Data.HashMap.Strict as H
import qualified Data.HashSet as S
import qualified Data.Foldable as DF
import qualified Data.Set as DS

import Echidna.ABI
import Echidna.Exec
import Echidna.Solidity
import Echidna.Test
import Echidna.Transaction
import Echidna.Types.Campaign

instance MonadThrow m => MonadThrow (RandT g m) where
throwM = lift . throwM
instance MonadCatch m => MonadCatch (RandT g m) where
catch = liftCatch catch

type MutationConsts = (Integer, Integer, Integer)

-- | Configuration for running an Echidna 'Campaign'.
data CampaignConf = CampaignConf { testLimit :: Int
-- ^ Maximum number of function calls to execute while fuzzing
, stopOnFail :: Bool
-- ^ Whether to stop the campaign immediately if any property fails
, estimateGas :: Bool
-- ^ Whether to collect gas usage statistics
, seqLen :: Int
-- ^ Number of calls between state resets (e.g. \"every 10 calls,
-- reset the state to avoid unrecoverable states/save memory\"
, shrinkLimit :: Int
-- ^ Maximum number of candidate sequences to evaluate while shrinking
, knownCoverage :: Maybe CoverageMap
-- ^ If applicable, initially known coverage. If this is 'Nothing',
-- Echidna won't collect coverage information (and will go faster)
, seed :: Maybe Int
-- ^ Seed used for the generation of random transactions
, dictFreq :: Float
-- ^ Frequency for the use of dictionary values in the random transactions
, corpusDir :: Maybe FilePath
-- ^ Directory to load and save lists of transactions
, mutConsts :: MutationConsts
}

-- | State of a particular Echidna test. N.B.: \"Solved\" means a falsifying call sequence was found.
data TestState = Open Int -- ^ Maybe solvable, tracking attempts already made
| Large Int [Tx] -- ^ Solved, maybe shrinable, tracking shrinks tried + best solve
| Passed -- ^ Presumed unsolvable
| Solved [Tx] -- ^ Solved with no need for shrinking
| Failed ExecException -- ^ Broke the execution environment
deriving Show

instance Eq TestState where
(Open i) == (Open j) = i == j
(Large i l) == (Large j m) = i == j && l == m
Passed == Passed = True
(Solved l) == (Solved m) = l == m
_ == _ = False

instance ToJSON TestState where
toJSON s = object $ ("passed", toJSON passed) : maybeToList desc where
(passed, desc) = case s of Open _ -> (True, Nothing)
Passed -> (True, Nothing)
Large _ l -> (False, Just ("callseq", toJSON l))
Solved l -> (False, Just ("callseq", toJSON l))
Failed e -> (False, Just ("exception", toJSON $ show e))

type Corpus = DS.Set (Integer, [Tx])

-- | The state of a fuzzing campaign.
data Campaign = Campaign { _tests :: [(SolTest, TestState)]
-- ^ Tests being evaluated
, _coverage :: CoverageMap
-- ^ Coverage captured (NOTE: we don't always record this)
, _gasInfo :: Map Text (Int, [Tx])
-- ^ Worst case gas (NOTE: we don't always record this)
, _genDict :: GenDict
-- ^ Generation dictionary
, _newCoverage :: Bool
-- ^ Flag to indicate new coverage found
, _corpus :: Corpus
-- ^ Set of transactions with maximum coverage
, _ncallseqs :: Int
-- ^ Number of times the callseq is called
}

instance ToJSON Campaign where
toJSON (Campaign ts co gi _ _ _ _) = object $ ("tests", toJSON $ mapMaybe format ts)
: ((if co == mempty then [] else [
("coverage",) . toJSON . mapKeys (("0x" ++) . (`showHex` "") . keccak) $ DF.toList <$> co]) ++
[(("maxgas",) . toJSON . toList) gi | gi /= mempty]) where
format (Right _, Open _) = Nothing
format (Right (n, _), s) = Just ("assertion in " <> n, toJSON s)
format (Left (n, _), s) = Just (n, toJSON s)

makeLenses ''Campaign

instance Has GenDict Campaign where
hasLens = genDict

defaultCampaign :: Campaign
defaultCampaign = Campaign mempty mempty mempty defaultDict False mempty 0

-- | Given a 'Campaign', checks if we can attempt any solves or shrinks without exceeding
-- the limits defined in our 'CampaignConf'.
isDone :: (MonadReader x m, Has CampaignConf x) => Campaign -> m Bool
isDone c | null (view tests c) = do tl <- view (hasLens . to testLimit)
q <- view (hasLens . to seqLen)
return $ view ncallseqs c * q >= tl
isDone (view tests -> ts) = view (hasLens . to (liftM3 (,,) testLimit shrinkLimit stopOnFail))
isDone c | null (view tests c) = do
tl <- view (hasLens . testLimit)
q <- view (hasLens . seqLen)
return $ view ncallseqs c * q >= tl
isDone (view tests -> ts) = view (hasLens . to (liftM3 (,,) _testLimit _shrinkLimit _stopOnFail))
<&> \(tl, sl, sof) -> let res (Open i) = if i >= tl then Just True else Nothing
res Passed = Just True
res (Large i _) = if i >= sl then Just False else Nothing
Expand All @@ -167,12 +79,12 @@ isSuccess (view tests -> ts) =
updateTest :: ( MonadCatch m, MonadRandom m, MonadReader x m
, Has SolConf x, Has TestConf x, Has TxConf x, Has CampaignConf x)
=> VM -> Maybe (VM, [Tx]) -> (SolTest, TestState) -> m (SolTest, TestState)
updateTest v (Just (v', xs)) (n, t) = view (hasLens . to testLimit) >>= \tl -> (n,) <$> case t of
updateTest v (Just (v', xs)) (n, t) = view (hasLens . testLimit) >>= \tl -> (n,) <$> case t of
Open i | i >= tl -> pure Passed
Open i -> catch (evalStateT (checkETest n) v' <&> bool (Large (-1) xs) (Open (i + 1)))
(pure . Failed)
_ -> snd <$> updateTest v Nothing (n,t)
updateTest v Nothing (n, t) = view (hasLens . to shrinkLimit) >>= \sl -> (n,) <$> case t of
updateTest v Nothing (n, t) = view (hasLens . shrinkLimit) >>= \sl -> (n,) <$> case t of
Large i x | i >= sl -> pure $ Solved x
Large i x -> if length x > 1 || any canShrinkTx x
then Large (i + 1) <$> evalStateT (shrinkSeq (checkETest n) x) v
Expand Down Expand Up @@ -230,9 +142,9 @@ execTxOptC t = do
og <- hasLens . coverage <<.= mempty
res <- execTxWith vmExcept (usingCoverage $ pointCoverage (hasLens . coverage)) t
let vmr = getResult $ fst res
-- Update the coverage map with the proper binary according to the vm result
-- Update the coverage map with the proper binary according to the vm result
hasLens . coverage %= mapWithKey (\ _ s -> DS.map (\(i,_) -> (i, vmr)) s)
-- Update the global coverage map with the union of the result just obtained
-- Update the global coverage map with the union of the result just obtained
hasLens . coverage %= unionWith DS.union og
grew <- (== LT) . comparing coveragePoints og <$> use (hasLens . coverage)
when grew $ do
Expand All @@ -242,18 +154,18 @@ execTxOptC t = do

-- | Given a list of transactions in the corpus, save them discarding reverted transactions
addToCorpus :: (MonadState s m, Has Campaign s) => Int -> [(Tx, (VMResult, Int))] -> m ()
addToCorpus n res = unless (null rtxs) $ hasLens . corpus %= DS.insert (toInteger n, rtxs) where
rtxs = map fst res
addToCorpus n res = unless (null rtxs) $ hasLens . corpus %= DS.insert (toInteger n, rtxs)
where rtxs = fst <$> res

seqMutators :: (MonadRandom m) => MutationConsts -> m (Int -> Corpus -> [Tx] -> m [Tx])
seqMutators (c1, c2, c3) = fromList
seqMutators (c1, c2, c3) = fromList
[(cnm , fromInteger c1),
(mut False, fromInteger c2),
(mut True , fromInteger c3)]
where -- Use the generated random transactions
cnm _ _ = return
-- Use the generated random transactions
where cnm _ _ = return
mut flp ql ctxs gtxs = do
let somePercent = if (fst . DS.findMax) ctxs > 1 -- if the corpus already contains new elements
let somePercent = if (fst . DS.findMax) ctxs > 1 -- if the corpus already contains new elements
then 1 + (DS.size ctxs `div` 20) -- then take 5% of its size
else DS.size ctxs -- otherwise, take all of it
rtxs <- fromList $ map (\(i, txs) -> (txs, fromInteger i)) $ take somePercent $ DS.toDescList ctxs
Expand All @@ -266,7 +178,7 @@ randseq :: ( MonadCatch m, MonadRandom m, MonadReader x m, MonadState y m
=> Int -> Map Addr Contract -> World -> m [Tx]
randseq ql o w = do
ca <- use hasLens
cs <- mutConsts <$> view hasLens
cs <- view $ hasLens . mutConsts
let ctxs = ca ^. corpus
p = ca ^. ncallseqs
if length ctxs > p then -- Replay the transactions in the corpus, if we are executing the first iterations
Expand All @@ -289,10 +201,10 @@ callseq :: ( MonadCatch m, MonadRandom m, MonadReader x m, MonadState y m
callseq v w ql = do
-- First, we figure out whether we need to execute with or without coverage optimization and gas info,
-- and pick our execution function appropriately
coverageEnabled <- isJust . knownCoverage <$> view hasLens
coverageEnabled <- isJust <$> view (hasLens . knownCoverage)
let ef = if coverageEnabled then execTxOptC else execTx
old = v ^. env . EVM.contracts
gasEnabled <- estimateGas <$> view hasLens
gasEnabled <- view $ hasLens . estimateGas
-- Then, we get the current campaign state
ca <- use hasLens
-- Then, we generate the actual transaction in the sequence
Expand All @@ -305,15 +217,14 @@ callseq v w ql = do
-- and construct a set to union to the constants table
diffs = H.fromList [(AbiAddressType, S.fromList $ AbiAddress <$> diff)]
-- Save the global campaign state (also vm state, but that gets reset before it's used)
hasLens .= snd s
-- Update the gas estimation
hasLens .= snd s -- Update the gas estimation
when gasEnabled $ hasLens . gasInfo %= updateGasInfo res []
-- If there is new coverage, add the transaction list to the corpus
when (s ^. _2 . newCoverage) $ addToCorpus (s ^. _2 . ncallseqs + 1) res
-- Reset the new coverage flag
hasLens . newCoverage .= False
-- Keep track of the number of calls to `callseq`
hasLens . ncallseqs += 1
hasLens . ncallseqs += 1
-- Now we try to parse the return values as solidity constants, and add then to the 'GenDict'
types <- use $ hasLens . rTypes
let results = parse (map (\(t, (vr, _)) -> (t, vr)) res) types
Expand Down Expand Up @@ -342,9 +253,9 @@ campaign :: ( MonadCatch m, MonadRandom m, MonadReader x m
-> m Campaign
campaign u v w ts d txs = do
let d' = fromMaybe defaultDict d
c <- fromMaybe mempty <$> view (hasLens . to knownCoverage)
g <- view (hasLens . to seed)
b <- view (hasLens . to _benchmarkMode)
c <- fromMaybe mempty <$> view (hasLens . knownCoverage)
g <- view (hasLens . seed)
b <- view (hasLens . benchmarkMode)
let g' = mkStdGen $ fromMaybe (d' ^. defSeed) g
execStateT (evalRandT runCampaign g') (Campaign ((,Open (-1)) <$> if b then [] else ts) c mempty d' False (DS.fromList $ map (1,) txs) 0) where
step = runUpdate (updateTest v Nothing) >> lift u >> runCampaign
Expand Down
2 changes: 1 addition & 1 deletion lib/Echidna/Config.hs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import qualified Data.ByteString as BS
import qualified Data.List.NonEmpty as NE
import qualified Data.Yaml as Y

import Echidna.Campaign
import Echidna.Types.Campaign
import Echidna.Solidity
import Echidna.Test
import Echidna.Transaction
Expand Down
2 changes: 1 addition & 1 deletion lib/Echidna/Output/JSON.hs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
module Echidna.Output.JSON where

import Echidna.ABI (ppAbiValue)
import qualified Echidna.Campaign as C
import qualified Echidna.Types.Campaign as C
import Echidna.Solidity (SolTest)
import Echidna.Transaction (Tx(..), TxCall(..), TxResult)
import Data.Aeson hiding (Error)
Expand Down
1 change: 1 addition & 0 deletions lib/Echidna/Types.hs
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
module Echidna.Types where
107 changes: 107 additions & 0 deletions lib/Echidna/Types/Campaign.hs
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
{-# LANGUAGE MultiParamTypeClasses #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE TemplateHaskell #-}
{-# LANGUAGE TupleSections #-}

module Echidna.Types.Campaign where

import Control.Lens
import Data.Aeson (ToJSON(..), object)
import Data.Foldable (toList)
import Data.Has (Has(..))
import Data.Map (Map, mapKeys)
import Data.Maybe (mapMaybe, maybeToList)
import Data.Set (Set)
import Data.Text (Text)
import EVM.Keccak (keccak)
import Numeric (showHex)

import Echidna.ABI (GenDict, defaultDict)
import Echidna.Exec (CoverageMap, ExecException)
import Echidna.Solidity (SolTest)
import Echidna.Transaction (Tx)

type MutationConsts = (Integer, Integer, Integer)

-- | Configuration for running an Echidna 'Campaign'.
data CampaignConf = CampaignConf { _testLimit :: Int
-- ^ Maximum number of function calls to execute while fuzzing
, _stopOnFail :: Bool
-- ^ Whether to stop the campaign immediately if any property fails
, _estimateGas :: Bool
-- ^ Whether to collect gas usage statistics
, _seqLen :: Int
-- ^ Number of calls between state resets (e.g. \"every 10 calls,
-- reset the state to avoid unrecoverable states/save memory\"
, _shrinkLimit :: Int
-- ^ Maximum number of candidate sequences to evaluate while shrinking
, _knownCoverage :: Maybe CoverageMap
-- ^ If applicable, initially known coverage. If this is 'Nothing',
-- Echidna won't collect coverage information (and will go faster)
, _seed :: Maybe Int
-- ^ Seed used for the generation of random transactions
, _dictFreq :: Float
-- ^ Frequency for the use of dictionary values in the random transactions
, _corpusDir :: Maybe FilePath
-- ^ Directory to load and save lists of transactions
, _mutConsts :: MutationConsts
}
makeLenses ''CampaignConf

-- | State of a particular Echidna test. N.B.: \"Solved\" means a falsifying call sequence was found.
data TestState = Open Int -- ^ Maybe solvable, tracking attempts already made
| Large Int [Tx] -- ^ Solved, maybe shrinable, tracking shrinks tried + best solve
| Passed -- ^ Presumed unsolvable
| Solved [Tx] -- ^ Solved with no need for shrinking
| Failed ExecException -- ^ Broke the execution environment
deriving Show

instance Eq TestState where
(Open i) == (Open j) = i == j
(Large i l) == (Large j m) = i == j && l == m
Passed == Passed = True
(Solved l) == (Solved m) = l == m
_ == _ = False

instance ToJSON TestState where
toJSON s = object $ ("passed", toJSON passed) : maybeToList desc where
(passed, desc) = case s of Open _ -> (True, Nothing)
Passed -> (True, Nothing)
Large _ l -> (False, Just ("callseq", toJSON l))
Solved l -> (False, Just ("callseq", toJSON l))
Failed e -> (False, Just ("exception", toJSON $ show e))

type Corpus = Set (Integer, [Tx])

-- | The state of a fuzzing campaign.
data Campaign = Campaign { _tests :: [(SolTest, TestState)]
-- ^ Tests being evaluated
, _coverage :: CoverageMap
-- ^ Coverage captured (NOTE: we don't always record this)
, _gasInfo :: Map Text (Int, [Tx])
-- ^ Worst case gas (NOTE: we don't always record this)
, _genDict :: GenDict
-- ^ Generation dictionary
, _newCoverage :: Bool
-- ^ Flag to indicate new coverage found
, _corpus :: Corpus
-- ^ List of transactions with maximum coverage
, _ncallseqs :: Int
-- ^ Number of times the callseq is called
}
makeLenses ''Campaign

instance ToJSON Campaign where
toJSON (Campaign ts co gi _ _ _ _) = object $ ("tests", toJSON $ mapMaybe format ts)
: ((if co == mempty then [] else [
("coverage",) . toJSON . mapKeys (("0x" <>) . (`showHex` "") . keccak) $ toList <$> co]) ++
[(("maxgas",) . toJSON . toList) gi | gi /= mempty]) where
format (Right _, Open _) = Nothing
format (Right (n, _), s) = Just ("assertion in " <> n, toJSON s)
format (Left (n, _), s) = Just (n, toJSON s)

instance Has GenDict Campaign where
hasLens = genDict

defaultCampaign :: Campaign
defaultCampaign = Campaign mempty mempty mempty defaultDict False mempty 0
Loading

0 comments on commit bc25142

Please sign in to comment.